tablassert 13.0.0__tar.gz → 15.0.0__tar.gz

{tablassert-13.0.0 → tablassert-15.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tablassert
-Version: 13.0.0
+Version: 15.0.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Science/Research

{tablassert-13.0.0 → tablassert-15.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tablassert"
-version = "13.0.0"
+version = "15.0.0"
 description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
 authors = [
     { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }

{tablassert-13.0.0 → tablassert-15.0.0}/src/tablassert/agent.py

@@ -1299,11 +1299,11 @@ def _biolink_report(nodes: Path, edges: Path) -> dict[str, object]:
     Wraps ``biolink.validate_kgx`` (the same check ``tablassert validate-kgx`` runs) into the
     flat, JSON-safe keys ``build_and_audit`` returns, plus the ``_notes`` list the caller
     folds into its own. ``biolink_valid_pct`` excludes the known-pending fields Tablassert
-    emits on purpose (``approval_ids`` as a translator-ingest pass-through and the KGX
-    denormalized carryovers -- the set is derived, and emptied itself of ``effect_size`` /
-    ``effect_type`` when biolink-model 4.4.4 shipped them as real Association slots) so the
-    scored number reflects the agent's decisions rather than a deliberate gap;
-    ``biolink_valid_pct_strict`` keeps that gap visible.
+    emits on purpose (the KGX denormalized carryovers -- the set is derived, and emptied
+    itself of ``approval_ids`` when the curated pass-through override was removed and of
+    ``effect_size`` / ``effect_type`` when biolink-model 4.4.4 shipped them as real
+    Association slots) so the scored number reflects the agent's decisions rather than a
+    deliberate gap; ``biolink_valid_pct_strict`` keeps that gap visible.
 
     Never raises: an unreadable or unparseable artifact degrades to ``None`` metrics and a
     note, exactly like the coverage measurement above it.
@@ -2349,19 +2349,13 @@ qualifier and evidence slot the specific class declared. build_and_audit reports
   `effect_type`), `q value` / `padj` -> `adjusted_p_value`. Names nothing claims
   (`fold_change` alone, `z_score`, `lfsr`, `standard_error`, free-form notes) are folded
   into `supporting_text`. Prefer `p_value`, `adjusted_p_value`, `effect_size`,
-  `effect_type`, `has_evidence`. For FDA application numbers, `approval_ids` is a deliberate
-  translator-ingest pass-through: keep the pipe-joined value as a scalar and do not add
-  `split_by`.
+  `effect_type`, `has_evidence`.
 - MULTIVALUED slots (`has_evidence` and friends) take a real JSON array, never a joined string:
   `split_by` is the ONLY multivalued encoding — there is no literal-list method. INSPECT the
   column's cells first (read_table shows them); the separator they ACTUALLY use — `|`, `,`, or
   `;` — is the one you declare: `{method: column, encoding: <letter>, split_by: "<separator>"}`.
   A SINGLE-value cell gets NO `split_by`: its scalar wraps into a one-element array, the correct
   shape. Cells that DO join multiple values but OMIT `split_by` ship as one unusable joined blob.
-- `approval_ids` is a deliberate translator-ingest pass-through extra, EXEMPT from the
-  validity score: a `biolink_valid_pct` below 1.0 is never caused by it. (`effect_size` /
-  `effect_type` were exempt only until biolink-model 4.4.4 shipped them as real Association
-  slots; they now validate like any other slot.)
 - QUALIFIERS: enum-ranged qualifiers take a literal TOKEN, never a CURIE
   (`object_direction_qualifier: increased`, not a UMLS id), and `species_context_qualifier` is
   disabled — never author it as a qualifier or annotation.

{tablassert-13.0.0 → tablassert-15.0.0}/src/tablassert/biolink.py

@@ -396,11 +396,6 @@ def resolve_association_class(category: str, predicate: str) -> type[Any]:
 #   - ``taxon`` -- a node property; no species-context edge is synthesized from it.
 TABLASERT_EDGE_EXTRAS: frozenset[str] = frozenset(
     [
-        # FDA application numbers from translator-ingests (DAKP precedent): the ingest
-        # emits them as a pipe-joined scalar (e.g. ``"011111|022222"``) with no
-        # ``split_by``, so the curated extra carries the scalar to the final edge
-        # verbatim instead of folding it into ``supporting_text``.
-        "approval_ids",
         "broad_synonym",
         "equivalent_identifiers",
         "evidence_direction",
@@ -574,9 +569,9 @@ against the resolved class is done by ``lib.prune_to_class()``.
 KNOWN_PENDING_EDGE_FIELDS: frozenset[str] = TABLASERT_EDGE_EXTRAS - frozenset(_association_model_fields())
 """Curated edge extras the installed Biolink Model does not (yet) declare on any association.
 
-Tablassert emits these deliberately -- ``approval_ids`` as a translator-ingest pass-through,
-plus the KGX denormalized carryovers (``synonym``, ``xref``, ``relation``, ...) -- so a Biolink
-class rejects them as ``extra_forbidden`` even though the build is behaving as designed. The
+Tablassert emits these deliberately -- the KGX denormalized carryovers (``synonym``,
+``xref``, ``relation``, ...) -- so a Biolink class rejects them as ``extra_forbidden`` even
+though the build is behaving as designed. The
 statistical ``effect_size`` / ``effect_type`` pair is no longer pending since
 biolink-model 4.4.4 shipped PR #1774.
 :func:`is_pending_problem` uses this set
@@ -643,6 +638,40 @@ def is_multivalued(cls: type[Any], field: str) -> bool:
     return any(get_origin(arg) is list for arg in get_args(annotation))
 
 
+@cache
+def _retrieval_source_id_required() -> bool:
+    """Whether the installed model still requires the inherited source ``id`` field."""
+    retrieval_source: Any = getattr(_bm, "RetrievalSource", None)
+    id_field: Any = getattr(retrieval_source, "model_fields", {}).get("id")
+    return id_field is not None and id_field.is_required()
+
+
+def _validation_record(record: dict[str, Any], *, edge: bool) -> dict[str, Any]:
+    """Add only in-memory compatibility aliases needed by the installed Biolink model.
+
+    ``RetrievalSource`` in the currently pinned model still requires the inherited
+    ``Entity.id`` even though ``resource_id`` is the canonical provenance identifier
+    emitted by Tablassert. The alias is used solely for Pydantic validation; it never
+    changes the decoded KGX record or the files written by the pipeline.
+    """
+    if not edge:
+        return record
+    if not _retrieval_source_id_required():
+        return record
+    sources: Any = record.get("sources")
+    if not isinstance(sources, list):
+        return record
+    normalized: list[Any] = []
+    changed: bool = False
+    for source in sources:
+        if isinstance(source, dict) and "id" not in source and "resource_id" in source:
+            normalized.append({**source, "id": source["resource_id"]})
+            changed = True
+        else:
+            normalized.append(source)
+    return {**record, "sources": normalized} if changed else record
+
+
 def validate_record(record: dict[str, Any], *, edge: bool) -> list[str]:
     """Validate one KGX record against the Biolink class named by its ``category``.
 
@@ -659,7 +688,7 @@ def validate_record(record: dict[str, Any], *, edge: bool) -> list[str]:
     category: str = categories[0] if isinstance(categories, list) and categories else str(categories or "")
     cls: type[Any] = association_class(category) if edge else node_class(category)
     try:
-        cls(**record)
+        cls(**_validation_record(record, edge=edge))
     except ValidationError as error:
         return [f"{'.'.join(str(part) for part in item['loc']) or '?'}: {item['type']}" for item in error.errors()]
     return []
@@ -731,7 +760,7 @@ def validate_kgx(nodes_path: Path, edges_path: Path, limit: int = 20) -> dict[st
     Two pass rates are reported. ``valid`` is strict and drives ``ok`` (the CLI's
     non-zero exit). ``valid_excluding_pending`` additionally counts records whose *every*
     failure is a :func:`is_pending_problem` -- the score to optimize against, so a
-    deliberate gap like the curated ``approval_ids`` pass-through is not mistaken for a
+    deliberate gap like the KGX denormalized carryovers is not mistaken for a
     malformed record. The two converge as future model releases absorb curated extras.
 
     Args:

{tablassert-13.0.0 → tablassert-15.0.0}/src/tablassert/coerce.py

@@ -48,8 +48,6 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_signific
         null p-value → null qualifier. The qualifier may only be set when
         ``p_value``/``adjusted_p_value`` is populated (Biolink class rule).
     """
-    from rapidfuzz import fuzz
-
     names: list[str] = lf.collect_schema().names()
     # Same rigorous classification as ``coerce_pvalue_columns``: a column is a significance
     # source only when ``pvalue_target`` accepts it — NOT by naive substring — so non-p-value
@@ -66,11 +64,14 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_signific
     # bucket is present. Raw p-value is the canonical significance source; adjusted is the fallback.
     preferred: str = col if col in buckets else next(iter(buckets))
     candidates: list[str] = buckets[preferred]
-    reference: str = preferred.replace("_", " ")
-    # An existing canonical column always wins; fuzzy ranking only picks among aliases
-    # (same rule as ``coerce_pvalue_columns``).
-    chosen: str = preferred if preferred in candidates else max(candidates, key=lambda c: fuzz.ratio(c, reference))
+    # Same selection rule as ``coerce_pvalue_columns``: canonical wins, then a
+    # raw candidate beats a -log10 alias, then fuzzy ranking among what is left.
+    chosen: str = _best_candidate(candidates, preferred, preferred)
     expr: pl.Expr = pl.col(chosen).cast(pl.Float64, strict=False)
+    # A -log10(p) score column must be un-logged before banding, or the bands
+    # invert (a score of 8 means p = 1e-8, not p = 8.0 -> not_significant).
+    if is_neglog10_column(chosen):
+        expr = _unlog10(expr)
     band: pl.Expr = (
         pl.when(expr.is_null())
         .then(pl.lit(None, dtype=pl.String))
@@ -173,6 +174,54 @@ SIGNIFICANCE_FLAG_PATTERN: re.Pattern[str] = re.compile(
     """,
     re.IGNORECASE | re.VERBOSE,
 )
+# A -log10(p/q) score column: an explicit negation marker (negative / negated /
+# neg / -) before a log/log10 p-or-q token ("negative log p value" — the
+# mokg-v12 HOYER1 spelling, "negative log10 p value", "-log10(p)",
+# "neg log10 q value"). The trailing token must be a *complete* p/q-value
+# token (or a bare delimited P), so the [pq] cannot swallow the first letter
+# of an unrelated word ("negative log protein" stays out). A plain
+# "log10 p value" without a negation marker does NOT match: the sign
+# convention is ambiguous there, so those columns keep riding verbatim rather
+# than being un-logged on a guess.
+NEGLOG10_PVALUE_PATTERN: re.Pattern[str] = re.compile(
+    rf"""
+    (?<![A-Za-z0-9])
+    (?: negative | negated | neg | - )
+    [\s_.\-()]*
+    log (?: 10 )?
+    [\s_.\-()]*
+    (?:
+        [pq] {_SEP} {_VALUE} {_NUMQUAL}    # complete p/q-value token ("log p value", "log q-value", "log pvalue")
+        | p (?! [A-Za-z0-9])               # bare P standing alone ("-log10(p)", "-log10 P")
+    )
+    """,
+    re.IGNORECASE | re.VERBOSE,
+)
+
+
+def is_neglog10_column(name: str) -> bool:
+    """Return True when a column reports a -log10(p/q) score instead of the raw p/q value.
+
+    Args:
+        name: Raw source column name.
+
+    Returns:
+        True when the name carries an explicit negation marker before a
+        log/log10 p-or-q token, else False. Plain ``log``/``log10`` spellings
+        without a marker are deliberately excluded (sign convention
+        ambiguous), as is everything without a log token at all.
+    """
+    return bool(NEGLOG10_PVALUE_PATTERN.search(name))
+
+
+def _unlog10(expr: pl.Expr) -> pl.Expr:
+    """Convert a -log10 score expression back to its original scale (10**-x).
+
+    Float64 underflow floors extreme scores at ``0.0`` — indistinguishable
+    from ``p ~ 0`` in practice, and the significance band is identical either
+    way. Nulls stay null.
+    """
+    return pl.lit(10.0) ** (-expr)
 
 
 def pvalue_target(name: str) -> str | None:
@@ -215,10 +264,37 @@ def pvalue_target(name: str) -> str | None:
     return "adjusted_p_value" if is_adjusted else "p_value"
 
 
+def _best_candidate(candidates: list[str], target: str, chosen: str) -> str:
+    """Pick the column a numeric p/q-value slot should read from.
+
+    A raw (non-neglog) candidate is always preferred over a -log10 alias of
+    the same statistic: the raw column holds the p/q value itself, while the
+    alias needs un-logging. Fuzzy ranking only runs when no raw candidate
+    exists (or only neglog candidates do), so a short raw name ("P", "FDR")
+    can no longer lose to a long -log10 alias it would then be un-logged over.
+
+    Args:
+        candidates: Column names bucketed onto ``target`` by ``pvalue_target``.
+        target: Canonical slot name (``p_value`` / ``adjusted_p_value``).
+        chosen: The canonical name when it is itself among the candidates
+            (the existing canonical-wins rule), else a sentinel that is not.
+
+    Returns:
+        The winning column name.
+    """
+    from rapidfuzz import fuzz
+
+    pool: list[str] = [c for c in candidates if not is_neglog10_column(c)] or candidates
+    return chosen if chosen in pool else max(pool, key=lambda c: fuzz.ratio(c, target.replace("_", " ")))
+
+
 def coerce_pvalue_columns(lf: pl.LazyFrame) -> pl.LazyFrame:
     """Rename p-value-like columns to Biolink KGX-compliant ``p_value`` / ``adjusted_p_value``.
 
     Picks a single best fuzzy match per target when multiple candidates exist.
+    A chosen column that reports a -log10(p/q) score (see
+    :func:`is_neglog10_column`) is un-logged (``p = 10**-x``) as it is renamed,
+    so the slot receives the p-value the model types it as.
 
     Args:
         lf: Source LazyFrame.
@@ -226,9 +302,6 @@ def coerce_pvalue_columns(lf: pl.LazyFrame) -> pl.LazyFrame:
     Returns:
         LazyFrame with the chosen columns renamed (no-op if no candidates).
     """
-    # Picks a single best fuzzy match per target when multiple candidates exist.
-    from rapidfuzz import fuzz
-
     names: list[str] = lf.collect_schema().names()
     buckets: dict[str, list[str]] = {}
     for n in names:
@@ -237,14 +310,23 @@ def coerce_pvalue_columns(lf: pl.LazyFrame) -> pl.LazyFrame:
             buckets.setdefault(target, []).append(n)
 
     renames: dict[str, str] = {}
+    unlog_targets: list[str] = []
     for target, candidates in buckets.items():
-        reference: str = target.replace("_", " ")
-        # An existing canonical column always wins; fuzzy ranking only picks among aliases.
-        chosen: str = target if target in candidates else max(candidates, key=lambda c: fuzz.ratio(c, reference))
+        # An existing canonical column always wins; a raw candidate beats a
+        # -log10 alias before fuzzy ranking has any say.
+        chosen: str = _best_candidate(candidates, target, target)
         if chosen != target:
             renames[chosen] = target
+        # A -log10(p) score must be un-logged when it lands on the numeric slot.
+        if is_neglog10_column(chosen):
+            unlog_targets.append(target)
 
-    return lf.rename(renames) if renames else lf
+    if not renames and not unlog_targets:
+        return lf
+    out: pl.LazyFrame = lf.rename(renames) if renames else lf
+    if unlog_targets:
+        out = out.with_columns([_unlog10(pl.col(t).cast(pl.Float64, strict=False)).alias(t) for t in unlog_targets])
+    return out
 
 
 # --- Study-size fragments ----------------------------------------------------

{tablassert-13.0.0 → tablassert-15.0.0}/src/tablassert/errors.py

@@ -22,6 +22,7 @@ TablassertErrorCodes = Literal[
     "encoding-bad-remove-entry",
     "graph-bad-infores",
     "override-bad-publication",
+    "override-bad-sources",
     "override-bad-upstream-infores",
     "override-bad-upstream-urls",
     "provenance-bad-pmc-id",

{tablassert-13.0.0 → tablassert-15.0.0}/src/tablassert/lib.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import math
 import operator
 import re
@@ -37,6 +38,7 @@ from tablassert.coerce import (
     coerced_target,
     effect_size_target,
     effect_type_target,
+    is_neglog10_column,
     pvalue_target,
     sig,
     study_size_target,
@@ -44,7 +46,7 @@ from tablassert.coerce import (
 from tablassert.enums import EncodingMethods, Files, InformationResources, Repositories, Tokens
 from tablassert.fullmap import ResolveSpec, fullmap_db_path, resolve, resolve_batch
 from tablassert.log import cat
-from tablassert.models import Encoding, NodeEncoding, Qualifier, RIGConfig, Section
+from tablassert.models import EDGE_ID_PLACEHOLDER, Encoding, NodeEncoding, Qualifier, RIGConfig, Section
 from tablassert.nlp import level_one, level_two
 from tablassert.qc import fullmap_audit
 from tablassert.rig import (
@@ -81,6 +83,7 @@ __all__ = [
     "effect_size_target",
     "effect_type_target",
     "infores",
+    "is_neglog10_column",
     "normalize_biolink_category",
     "predicate_options",
     "pvalue_target",
@@ -219,7 +222,7 @@ def predicate_options(subject_category: str, object_category: str) -> frozenset[
     return legal_predicates(derived_edge_category(subject_category, object_category))
 
 
-def edge_category(lf: pl.LazyFrame, predicate: str | None = None) -> pl.LazyFrame:
+def edge_category(lf: pl.LazyFrame, predicate: str | None = None, category_override: dict[str, str] | None = None) -> pl.LazyFrame:
     """Add the derived ``category`` column using native polars replace operations.
 
     The ``(subject role, object role)`` lookup alone routinely produces a category
@@ -230,10 +233,18 @@ def edge_category(lf: pl.LazyFrame, predicate: str | None = None) -> pl.LazyFram
     by :func:`biolink.resolve_association_class`, which walks up the association
     hierarchy only as far as the predicate requires.
 
+    ``category_override`` pins the association class per *object* category (the
+    config's ``statement.category_override``): a row whose raw object category --
+    before the role rollup that merges e.g. ``Disease`` and ``PhenotypicFeature``
+    into one pair key -- appears in the map carries the pinned class instead of the
+    pair-derived one. Pinned classes pass through the same predicate reconciliation.
+
     Args:
         lf: Source LazyFrame with ``subject category`` and ``object category``
             columns (biolink-prefixed).
         predicate: Section predicate CURIE used to reconcile the derived category.
+        category_override: ``{object category (bare name): biolink: CURIE}`` map
+            applied per row in place of the pair lookup.
 
     Returns:
         LazyFrame with a new list-typed ``category`` column containing the
@@ -249,12 +260,20 @@ def edge_category(lf: pl.LazyFrame, predicate: str | None = None) -> pl.LazyFram
         # small raw-category -> resolved-category remap resolved once at plan time.
         edge_lookup = {k: f"biolink:{resolve_association_class(v, predicate).__name__}" for k, v in edge_lookup.items()}
         default = f"biolink:{resolve_association_class(default, predicate).__name__}"
+        if category_override:
+            category_override = {k: f"biolink:{resolve_association_class(v, predicate).__name__}" for k, v in category_override.items()}
     names: list[str] = lf.collect_schema().names()
     subject_col: str = "subject_category" if "subject_category" in names else "subject category"
     object_col: str = "object_category" if "object_category" in names else "object category"
     sr: pl.Expr = pl.col(subject_col).str.replace("biolink:", "").replace(cat_role).fill_null("")
     or_: pl.Expr = pl.col(object_col).str.replace("biolink:", "").replace(cat_role).fill_null("")
-    return lf.with_columns(pl.concat_list(pl.concat_str([sr, pl.lit("|"), or_]).replace_strict(edge_lookup, default=default)).alias("category"))
+    derived: pl.Expr = pl.concat_str([sr, pl.lit("|"), or_]).replace_strict(edge_lookup, default=default)
+    if category_override:
+        # Key on the RAW object category: the role rollup above collapses exactly the
+        # distinction (Disease vs PhenotypicFeature) the override exists to preserve.
+        hit: pl.Expr = pl.col(object_col).str.replace("biolink:", "").replace_strict(category_override, default=None, return_dtype=pl.String)
+        derived = pl.when(hit.is_not_null()).then(hit).otherwise(derived)
+    return lf.with_columns(pl.concat_list(derived).alias("category"))
 
 
 def prune_to_class(lf: pl.LazyFrame) -> pl.LazyFrame:
@@ -354,16 +373,12 @@ def value(lf: pl.LazyFrame, col: str, x: object) -> pl.LazyFrame:
 def _retrieval_source(resource_id: str, resource_role: str, upstream: list[str] | None = None, urls: list[str] | None = None) -> pl.Expr:
     """Build one ``RetrievalSource`` struct expression.
 
-    Every entry declares the same five fields so that :func:`retrieval_sources` can
+    Every entry declares the same four fields so that :func:`retrieval_sources` can
     ``concat_list`` them into a single ``list[struct]`` column; absent list fields are
     typed nulls, which the Rust null-stripper removes from the emitted JSON.
     """
     empty: pl.Expr = pl.lit(None, dtype=pl.List(pl.String))
     return pl.struct(
-        # `id` duplicates `resource_id`, but RetrievalSource inherits `id` from
-        # `entity` and the generated Pydantic classes require it, so omitting it
-        # fails KGX validation. Stays until biolink-model #1706/#1731 land.
-        pl.lit(resource_id).alias("id"),
         pl.lit(resource_id).alias("resource_id"),
         pl.lit(resource_role).alias("resource_role"),
         (pl.concat_list([pl.lit(x) for x in upstream]) if upstream else empty).alias("upstream_resource_ids"),
@@ -558,7 +573,12 @@ def inline_supporting_study(lf: pl.LazyFrame, study_id: str, study_name: str | N
 
 
 def retrieval_sources(
-    lf: pl.LazyFrame, primary: str, upstream: list[str], urls: list[str], upstream_urls: dict[str, list[str]] | None = None
+    lf: pl.LazyFrame,
+    primary: str,
+    upstream: list[str],
+    urls: list[str],
+    upstream_urls: dict[str, list[str]] | None = None,
+    explicit: list[dict[str, Any]] | None = None,
 ) -> pl.LazyFrame:
     """Add the Biolink ``sources`` retrieval-provenance column.
 
@@ -576,18 +596,32 @@ def retrieval_sources(
     its own ``source_record_urls`` and the primary entry emits none (the primary is
     the transforming resource, not a downloadable record).
 
+    When ``explicit`` is given, exactly those entries are emitted, in order, and
+    ``primary``/``upstream``/``urls``/``upstream_urls`` are ignored. Each entry
+    template carries ``resource_id``, ``resource_role``, and optional
+    ``upstream_resource_ids``/``source_record_urls``; ``source_record_urls`` values
+    may contain the literal ``{edge_id}`` placeholder, which is NOT resolved here
+    (the edge id is only assigned at the final dedup stage) but in a post-dedup
+    sweep of the final edges NDJSON.
+
     Args:
         lf: Source LazyFrame.
         primary: Infores CURIE of the primary knowledge source.
         upstream: Infores CURIEs of upstream/supporting data sources.
         urls: Source record URLs for the primary entry (ignored when ``upstream_urls`` is set).
         upstream_urls: Optional per-upstream source record URLs keyed by infores CURIE.
+        explicit: Optional explicit entry templates emitted verbatim, in order.
 
     Returns:
         LazyFrame with a ``sources`` ``list[struct]`` column appended.
     """
-    if upstream_urls is not None:
-        entries: list[pl.Expr] = [_retrieval_source(primary, "primary_knowledge_source", upstream)]
+    if explicit is not None:
+        entries: list[pl.Expr] = [
+            _retrieval_source(entry["resource_id"], entry["resource_role"], entry.get("upstream_resource_ids"), entry.get("source_record_urls"))
+            for entry in explicit
+        ]
+    elif upstream_urls is not None:
+        entries = [_retrieval_source(primary, "primary_knowledge_source", upstream)]
         entries.extend(_retrieval_source(x, "supporting_data_source", urls=upstream_urls.get(x)) for x in upstream)
     else:
         entries = [_retrieval_source(primary, "primary_knowledge_source", upstream, urls)]
@@ -1159,7 +1193,7 @@ class Tcode(Section):
             [
                 op
                 for x in self.annotations
-                for op in [*self.encoding(x, x.annotation.lower()), *([(split_list, (x.annotation.lower(), x.split_by))] if x.split_by else [])]
+                for op in [*self.encoding(x, x.annotation), *([(split_list, (x.annotation, x.split_by))] if x.split_by else [])]
             ]
             if self.annotations
             else None,
@@ -1257,6 +1291,12 @@ class Tcode(Section):
             if override and override.upstream_source_record_urls is not None
             else None
         )
+        # An explicit `sources` template replaces the derived primary/upstream
+        # emission entirely; the model already forbids combining it with
+        # `upstream_resource_ids`/`upstream_source_record_urls`.
+        explicit_sources: list[dict[str, Any]] | None = (
+            [entry.model_dump(exclude_none=True) for entry in override.sources] if override and override.sources is not None else None
+        )
         knowledge_level = override.knowledge_level if override else self.provenance.knowledge_level
         agent_type = override.agent_type if override else self.provenance.agent_type
         publication_values = override.publications if override else [publication_curie(self.provenance.repo, self.provenance.publication or "")]
@@ -1273,17 +1313,22 @@ class Tcode(Section):
         study_name: str | None = sheet if sheet is not None else Path(self.source.local).name
         if study_name == study_id:
             study_name = None
+        # `use_enum_values` stores the override's enum keys/values as plain strings.
+        category_override: dict[str, str] | None = (
+            {obj: f"biolink:{pinned}" for obj, pinned in self.statement.category_override.items()} if self.statement.category_override else None
+        )
         return [
             (value, ("predicate", "biolink:" + self.statement.predicate)),
-            (edge_category, ("biolink:" + self.statement.predicate,)),
+            (edge_category, ("biolink:" + self.statement.predicate, category_override)),
             (value, ("knowledge_level", knowledge_level)),
             (value, ("agent_type", agent_type)),
             # Retrieval provenance lives only in the nested `sources` list (Biolink
             # RetrievalSource); current translator-ingests emits no flat
             # `primary_knowledge_source` scalar, so neither do we. A per-upstream URL
             # mapping (override.upstream_source_record_urls) re-homes the record URLs
-            # from the primary entry onto the matching supporting entries.
-            (retrieval_sources, (primary_knowledge_source, upstream_ids, [str(u) for u in self.source.url], upstream_urls))
+            # from the primary entry onto the matching supporting entries; an explicit
+            # `sources` template (override.sources) replaces the whole derivation.
+            (retrieval_sources, (primary_knowledge_source, upstream_ids, [str(u) for u in self.source.url], upstream_urls, explicit_sources))
             if primary_knowledge_source
             else None,
             (publications, (publication_values,)) if publication_values else None,
@@ -1524,6 +1569,44 @@ def dedup_stream(p_in: Path, is_edges: bool) -> None:
     p_in.unlink()
 
 
+def _resolve_edge_id_placeholders(edges_path: Path) -> None:
+    """Resolve ``{edge_id}`` placeholders in a final edges NDJSON file.
+
+    The edge ``id`` is a deterministic content hash assigned by the Rust deduper
+    after subgraphs are written, so explicit ``override.sources`` record URLs
+    cannot embed it during the polars build: the literal placeholder is what
+    gets hashed, and this post-dedup sweep substitutes each record's own id into
+    every string inside every ``sources[].source_record_urls`` list. The pass is
+    skipped entirely when no line contains the marker (cheap substring precheck,
+    no full parse), leaving the file byte-identical.
+
+    Args:
+        edges_path: Path to the deduplicated ``*.edges.ndjson`` file.
+
+    Returns:
+        ``None``; rewrites ``edges_path`` in place via a temp file when any
+        placeholder was resolved.
+    """
+    tmp_path: Path = edges_path.with_name(edges_path.name + ".placeholder.tmp")
+    resolved: bool = False
+    with edges_path.open("r", encoding="utf-8") as src, tmp_path.open("w", encoding="utf-8") as dst:
+        for line in src:
+            if EDGE_ID_PLACEHOLDER not in line:
+                dst.write(line)
+                continue
+            record: dict[str, Any] = json.loads(line)
+            for source in record.get("sources") or []:
+                urls: list[str] | None = source.get("source_record_urls")
+                if urls:
+                    source["source_record_urls"] = [url.replace(EDGE_ID_PLACEHOLDER, record["id"]) for url in urls]
+            dst.write(json.dumps(record, ensure_ascii=False) + "\n")
+            resolved = True
+    if resolved:
+        tmp_path.replace(edges_path)
+    else:
+        tmp_path.unlink()
+
+
 def fold_unknown_to_supporting_text(lf: pl.LazyFrame) -> pl.LazyFrame:
     """Fold any non-Biolink edge column into ``supporting_text`` as ``col: value`` strings.
 
@@ -1635,6 +1718,12 @@ def _write_ndjson(
     ``compile_graph``; each commented phase boundary below is a hook point for
     the US-009 ``on_phase`` progress callback.
 
+    Dedup assigns each edge ``id`` as the deterministic content hash of the
+    PRE-resolution record -- the literal ``{edge_id}`` placeholder in explicit
+    ``override.sources`` record URLs is what gets hashed -- and the placeholder
+    sweep that follows substitutes the assigned id into the final edges file
+    only, keeping ids deterministic.
+
     Args:
         subnodes: Per-section node LazyFrames from ``_collect_subframes``.
         subedges: Per-section edge LazyFrames from ``_collect_subframes``.
@@ -1669,6 +1758,9 @@ def _write_ndjson(
     if on_phase is not None:
         on_phase("dedup")
     dedup_stream(edges_tmp, is_edges=True)
+    # The deduper hashes the record WITH the literal `{edge_id}` placeholder still
+    # in place, so edge ids stay deterministic regardless of this resolution pass.
+    _resolve_edge_id_placeholders(edges_tmp.with_suffix(""))
     dedup_stream(nodes_tmp, is_edges=False)
 
     # Phase: rig. Summaries come from the FINAL deduplicated KGX files, and the

{tablassert-13.0.0 → tablassert-15.0.0}/src/tablassert/models.py

@@ -17,9 +17,11 @@ from tablassert.biolink import (
     UNSATISFIABLE_EDGE_FIELDS,
     AgentTypes,
     Categories,
+    EdgeCategories,
     KnowledgeLevels,
     Predicates,
     Qualifiers,
+    resolve_association_class,
 )
 from tablassert.coerce import coerced_target
 from tablassert.enums import (
@@ -435,6 +437,14 @@ class Statement(TablaBase):
     object: NodeEncoding = Field(..., description="Object node encoding and mapping configuration.")
     predicate: Predicates = Field(Predicates.RELATED_TO, description="Predicate connecting subject and object nodes.")
     qualifiers: list[Qualifier] | None = Field(None, description="Optional qualifier nodes attached to the statement.")
+    category_override: dict[Categories, EdgeCategories] | None = Field(
+        None,
+        description=(
+            "Optional per-object-category association-class override: maps a resolved object category (bare name, e.g. `Disease`) "
+            "to the association category its rows carry (e.g. `EntityToDiseaseAssociation`), used in place of the derived "
+            "(subject, object) pair lookup. Rows whose object category is absent from the map derive as before."
+        ),
+    )
 
     @model_validator(mode="after")
     def reject_duplicate_qualifiers(self: Self) -> Self:
@@ -458,6 +468,30 @@ class Statement(TablaBase):
             seen.add(key)
         return self
 
+    @model_validator(mode="after")
+    def warn_when_an_override_is_demoted_by_the_predicate(self: Self) -> Self:
+        """Warn when a pinned association class cannot carry the section predicate.
+
+        A ``category_override`` value still passes through
+        :func:`biolink.resolve_association_class` at build time: when the pinned class
+        restricts its ``predicate`` slot and the section predicate is not in it, the
+        emitted category silently walks up to an ancestor -- taking every subclass-only
+        slot the author pinned the class *for* with it. The value is never lost (the
+        reconciliation keeps the edge valid), so this is a warning, not an error.
+        """
+        for obj_category, pinned in (self.category_override or {}).items():
+            # `use_enum_values` stores both sides as plain strings.
+            resolved: str = resolve_association_class(f"biolink:{pinned}", f"biolink:{self.predicate}").__name__
+            if resolved != pinned:
+                warnings.warn(
+                    f"category_override pins `{pinned}` for `{obj_category}` objects, but that class does not accept "
+                    f"predicate `{self.predicate}`, so those rows are emitted as `{resolved}` instead -- any slots only "
+                    f"`{pinned}` declares are pruned off the edge.",
+                    BiolinkRelocationWarning,
+                    stacklevel=2,
+                )
+        return self
+
 
 def validate_infores_curie(value: str, code: TablassertErrorCodes) -> str:
     """Validate an ``infores:`` CURIE used for Biolink knowledge-source fields."""
@@ -466,16 +500,99 @@ def validate_infores_curie(value: str, code: TablassertErrorCodes) -> str:
     return value
 
 
+EDGE_ID_PLACEHOLDER: str = "{edge_id}"
+"""Placeholder for the final edge id inside ``override.sources`` record URLs.
+
+The edge ``id`` is a deterministic content hash computed during the final dedup
+stage -- after subgraphs are written -- so a per-edge URL cannot embed it during
+the table build. The literal placeholder is emitted as-is and resolved against
+the deduplicated ``*.edges.ndjson`` in a post-dedup sweep.
+"""
+
+RESOURCE_ROLES: tuple[str, ...] = ("primary_knowledge_source", "aggregator_knowledge_source", "supporting_data_source")
+"""Valid Biolink ``ResourceRoleEnum`` values for retrieval ``sources`` entries.
+
+Kept as literals instead of importing ``ResourceRoleEnum`` from the generated
+``biolink_model`` Pydantic classes so config validation stays cheap; any other
+value fails KGX validation downstream.
+"""
+
+
+class SourceOverride(TablaBase):
+    """One explicit retrieval-``sources`` entry template for manual provenance.
+
+    Each entry becomes one Biolink ``RetrievalSource`` struct on the edge's
+    ``sources`` list, replacing the derived primary/upstream emission entirely.
+    """
+
+    resource_id: str = Field(description="Infores CURIE of this retrieval source entry.", examples=["infores:my-source"])
+    resource_role: str = Field(
+        description="Biolink ResourceRoleEnum value for this entry.",
+        examples=["primary_knowledge_source", "aggregator_knowledge_source", "supporting_data_source"],
+    )
+    upstream_resource_ids: list[str] | None = Field(
+        None, description="Upstream infores CURIEs carried by this entry.", examples=[["infores:my-upstream"]]
+    )
+    source_record_urls: list[str] | None = Field(
+        None,
+        description=f"Source record URLs carried by this entry; `{EDGE_ID_PLACEHOLDER}` is replaced with the final edge id after dedup.",
+        examples=[["https://example.org/edge?id={edge_id}"]],
+    )
+
+    @field_validator("resource_id", mode="after")
+    @classmethod
+    def infores_resource_id(cls, value: str) -> str:
+        return validate_infores_curie(value, "override-bad-sources")
+
+    @field_validator("resource_role", mode="after")
+    @classmethod
+    def biolink_resource_role(cls, value: str) -> str:
+        if value not in RESOURCE_ROLES:
+            raise TablassertValidationError(f"`resource_role` must be one of {list(RESOURCE_ROLES)}, got {value!r}.", code="override-bad-sources")
+        return value
+
+    @field_validator("upstream_resource_ids", mode="after")
+    @classmethod
+    def infores_upstream_resource_ids(cls, values: list[str] | None) -> list[str] | None:
+        if values is None:
+            return None
+        for value in values:
+            validate_infores_curie(value, "override-bad-sources")
+        return values
+
+    @field_validator("source_record_urls", mode="after")
+    @classmethod
+    def url_or_edge_id_template(cls, values: list[str] | None) -> list[str] | None:
+        # Typed as plain str (not HttpUrl) so the `{edge_id}` placeholder survives
+        # validation; after removing placeholder occurrences the rest must still be
+        # an absolute http(s) URL.
+        if values is None:
+            return None
+        for value in values:
+            stripped: str = value.replace(EDGE_ID_PLACEHOLDER, "")
+            if not stripped.startswith(("https://", "http://")):
+                raise TablassertValidationError(
+                    f"`source_record_urls` entries must be http(s) URLs (optionally containing `{EDGE_ID_PLACEHOLDER}`), got {value!r}.",
+                    code="override-bad-sources",
+                )
+        return values
+
+
 class ManualProvenance(TablaBase):
     """Manually-specified provenance for non-PMID/PMC source graphs.
 
     When present under :class:`Provenance`, these values replace the legacy
     repo/publication-derived provenance while keeping the same KL/AT defaults.
     The primary ``sources`` entry (``resource_role: primary_knowledge_source``)
-    always derives from the graph-level ``rig.source_info.infores_id``; manual
-    infores CURIEs belong in ``upstream_resource_ids``.
+    derives from the graph-level ``rig.source_info.infores_id`` unless an
+    explicit ``sources`` template is given; manual infores CURIEs otherwise
+    belong in ``upstream_resource_ids``.
     """
 
+    sources: list[SourceOverride] | None = Field(
+        None,
+        description="Explicit retrieval-`sources` entry templates replacing the derived primary/upstream emission entirely; mutually exclusive with `upstream_resource_ids` and `upstream_source_record_urls`, which it subsumes.",
+    )
     upstream_resource_ids: list[str] = Field(
         default_factory=list,
         description="Manual upstream source infores CURIEs emitted instead of the repo-derived source map; the sanctioned place for manual infores.",
@@ -515,6 +632,26 @@ class ManualProvenance(TablaBase):
                 )
         return values
 
+    @model_validator(mode="after")
+    def sources_template_is_coherent(self: Self) -> Self:
+        if self.sources is None:
+            return self
+        if self.upstream_resource_ids or self.upstream_source_record_urls is not None:
+            raise TablassertValidationError(
+                "`sources` is mutually exclusive with `upstream_resource_ids` and `upstream_source_record_urls`; the explicit template subsumes both.",
+                code="override-bad-sources",
+            )
+        if not self.sources:
+            raise TablassertValidationError("`sources` must contain at least one entry when set.", code="override-bad-sources")
+        resource_ids: list[str] = [entry.resource_id for entry in self.sources]
+        if len(set(resource_ids)) != len(resource_ids):
+            raise TablassertValidationError("`sources` entries must have unique `resource_id` values.", code="override-bad-sources")
+        if not any(entry.resource_role in ("primary_knowledge_source", "aggregator_knowledge_source") for entry in self.sources):
+            raise TablassertValidationError(
+                "`sources` must include at least one `primary_knowledge_source` or `aggregator_knowledge_source` entry.", code="override-bad-sources"
+            )
+        return self
+
     @model_validator(mode="after")
     def upstream_urls_match_resource_ids(self: Self) -> Self:
         if self.upstream_source_record_urls is None:
@@ -571,7 +708,13 @@ class Annotation(Encoding):
     @field_validator("annotation", mode="after")
     @classmethod
     def clean_annotation(cls, annotation: str) -> str:
-        return annotation.lower().strip()
+        cleaned: str = annotation.strip()
+        lowered: str = cleaned.lower()
+        # Allow-listed slots may carry uppercase (Biolink's ``FDA_regulatory_approvals``):
+        # any casing the author declares canonicalizes onto the allow-listed spelling
+        # verbatim, so the emitted edge field preserves its case exactly instead of being
+        # lowercased into an unknown name.
+        return next((field for field in ALLOWED_EDGE_FIELDS if field.lower() == lowered), lowered)
 
     @model_validator(mode="after")
     def reject_disabled_annotations(self: Self) -> Self: