gem-mapping-studio 0.2.2__py3-none-any.whl

forome/gem/umls/build_umls_crosswalk.py

@@ -0,0 +1,384 @@
+#!/usr/bin/env python3
+"""Build the GEM -> UMLS crosswalk by resolving each dimension/value to a
+UMLS concept via the UTS REST API.
+
+This is the harness. It reads the curated inventory
+(``mapping/dimensions_inventory.yaml``), queries UMLS for each entry through a
+tiered search (exact -> normalized -> words), assigns an honest status
+(``mapped`` / ``review`` / ``unmapped`` / ``pending``), and writes a
+machine-readable crosswalk to ``mapping/umls_crosswalk.yaml`` plus a coverage
+summary. The LaTeX table is rendered separately by
+``mapping/render_crosswalk_tex.py`` from that YAML.
+
+Honesty by construction:
+* Nothing is mapped without a UMLS query actually returning a concept.
+* Without an API key the harness runs but every entry is ``pending`` (queried
+  nothing); it does not invent CUIs.
+* A query that finds candidates but no faithful exact/normalized match is
+  ``review`` (candidates recorded for a curator), never silently ``mapped``.
+
+Usage:
+    # 1. verify the inventory still covers the SHACL enumerations
+    python3 mapping/build_umls_crosswalk.py --check-inventory
+
+    # 2. real run (needs a UMLS license; key in UMLS_API_KEY or --api-key)
+    UMLS_API_KEY=... python3 mapping/build_umls_crosswalk.py
+
+    # 3. offline run with no key -> all entries 'pending' (scaffold mode)
+    python3 mapping/build_umls_crosswalk.py --no-key
+
+    # 4. offline run against test fixtures (used by the unit tests)
+    python3 mapping/build_umls_crosswalk.py --fixtures mapping/fixtures \
+        --out mapping/umls_crosswalk.demo.yaml
+
+Requirements: ``pyyaml``; ``requests`` for live runs; ``rdflib`` for
+``--check-inventory``.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import unicodedata
+from pathlib import Path
+
+import yaml
+
+from forome.gem.umls import semantic_types as stylib
+from forome.gem.umls._paths import DATA_DIR, SCHEMA_DIR
+
+BASE = SCHEMA_DIR.parent
+INVENTORY = DATA_DIR / "dimensions_inventory.yaml"
+ADJUDICATIONS = DATA_DIR / "adjudications.yaml"
+SHACL = SCHEMA_DIR / "genetic_evidence.shacl.ttl"
+DEFAULT_OUT = DATA_DIR / "umls_crosswalk.yaml"
+CACHE_DIR = DATA_DIR / "cache"
+
+# SHACL enum members deliberately NOT expected in the inventory:
+#   - the sentinel, declared a member of every dimension class
+#   - the legacy method synonyms (migrate to the renamed values)
+ENUM_EXCLUDE = {
+    "NOT_APPLICABLE_OR_OMITTED",
+    "IN_VIVO_EXPERIMENT", "IN_VITRO_EXPERIMENT", "BIOINFORMATICS_PREDICTION",
+}
+
+
+def _norm(s: str | None) -> str:
+    if not s:
+        return ""
+    s = s.lower().strip()
+    # Fold accents to their base letters (NFD decomposition + drop combining
+    # marks) before stripping non-ASCII, so e.g. "Über" -> "uber", not "ber".
+    s = unicodedata.normalize("NFD", s)
+    s = "".join(c for c in s if unicodedata.category(c) != "Mn")
+    s = re.sub(r"[^a-z0-9 ]+", " ", s)
+    return re.sub(r"\s+", " ", s).strip()
+
+
+# --------------------------------------------------------------------------
+# Inventory loading: flatten the nested YAML into a list of entries.
+# --------------------------------------------------------------------------
+
+def load_entries(inventory_path: Path = INVENTORY) -> list[dict]:
+    doc = yaml.safe_load(inventory_path.read_text())
+    entries: list[dict] = []
+    for dim, body in (doc.get("dimensions") or {}).items():
+        axis = body.get("axis")
+        if axis:
+            entries.append(_entry(dim, None, "axis", axis))
+        for v in body.get("values") or []:
+            entries.append(_entry(dim, v["token"], "value", v))
+        for v in body.get("common_values") or []:
+            entries.append(_entry(dim, v["token"], "common_value", v))
+    return entries
+
+
+def _entry(dim: str, token, kind: str, src: dict) -> dict:
+    return {
+        "dimension": dim,
+        "token": token,
+        "kind": kind,
+        "query": src["query"],
+        "expect": src.get("expect", "uncertain"),
+        "sab": src.get("sab"),
+        "inventory_note": src.get("note"),
+    }
+
+
+# --------------------------------------------------------------------------
+# Resolution: tiered UMLS search with honest status assignment.
+# --------------------------------------------------------------------------
+
+def resolve(client, query: str, sab: str | None, live: bool,
+            stys: str | None = None) -> dict:
+    if not live:
+        return {"status": "pending"}
+
+    def tier(use_sab):
+        s = sab if use_sab else None
+        for st in ("exact", "normalizedString", "words"):
+            cands = client.search(query, search_type=st, sabs=s, semantic_types=stys)
+            if cands:
+                return st, cands
+        return None, []
+
+    used_sab = bool(sab)
+    st, cands = tier(use_sab=True)
+    if not cands and sab:
+        st, cands = tier(use_sab=False)
+        used_sab = False
+
+    if not cands:
+        return {"status": "unmapped"}
+
+    top = cands[0]
+    # Honesty by construction: 'mapped' requires the returned concept name to
+    # actually match the query (normalized), regardless of which search tier
+    # produced it. The search_type label alone is NOT a guarantee that the
+    # top hit is faithful, so an 'exact'-tier hit with a mismatched name is
+    # demoted to 'review', not silently mapped.
+    exactish = _norm(top.get("name")) == _norm(query)
+    status = "mapped" if exactish else "review"
+    return {
+        "status": status,
+        "cui": top["cui"],
+        "matched_name": top["name"],
+        "root_source": top["root_source"],
+        "semantic_types": top["semantic_types"],
+        "search_type": st,
+        "sab_filtered": used_sab,
+        "sty_filtered": bool(stys),
+        "candidates": [
+            {k: c[k] for k in ("cui", "name", "root_source", "semantic_types")}
+            for c in cands[:5]
+        ],
+    }
+
+
+def _adj_key(dimension: str, token) -> str:
+    return f"{dimension}/{token if token is not None else '(axis)'}"
+
+
+def load_adjudications(path: Path | None = None) -> dict:
+    """Curator decisions keyed by 'dimension/token' (or 'dimension/(axis)').
+    ``path`` defaults to the CURRENT ``ADJUDICATIONS`` (resolved at call time, so
+    tests/workspaces that repoint it are honoured)."""
+    path = ADJUDICATIONS if path is None else path
+    if not path.is_file():
+        return {}
+    doc = yaml.safe_load(path.read_text()) or {}
+    return doc.get("adjudications", {}) or {}
+
+
+def apply_adjudication(entry: dict, adj: dict, client=None) -> dict:
+    """Apply a curator decision. Honest by construction: an accepted CUI is used
+    only if it was among the query's candidates OR the harness can fetch it live
+    from UMLS (confirming it is a real concept). The harness never fabricates a
+    concept the curator merely named. A curator may also accept a concept found
+    via the adjudication UI's live search; such a CUI is fetched here."""
+    if adj.get("unmapped"):
+        return {**entry, "status": "unmapped", "curated": True,
+                "curator_note": adj.get("note")}
+    cui = adj.get("accept")
+    if cui:
+        for c in (entry.get("candidates") or []):
+            if c["cui"] == cui:
+                return {**entry, "status": "mapped", "curated": True,
+                        "cui": c["cui"], "matched_name": c["name"],
+                        "root_source": c["root_source"],
+                        "semantic_types": c["semantic_types"],
+                        "curator_note": adj.get("note")}
+        # Not among the query candidates: fetch it live to confirm it exists.
+        concept = client.get_concept(cui) if client is not None else None
+        if concept and concept.get("cui") == cui:
+            return {**entry, "status": "mapped", "curated": True, "fetched": True,
+                    "cui": cui, "matched_name": concept["name"],
+                    "root_source": concept.get("root_source", "MTH"),
+                    "semantic_types": concept.get("semantic_types", []),
+                    "curator_note": adj.get("note")}
+        # Cannot confirm the concept: refuse to fabricate it.
+        return {**entry, "status": "review", "curated": False,
+                "curator_error": f"accepted CUI {cui} not in candidates and not "
+                                 f"confirmable via UMLS"}
+    return entry
+
+
+def build(client, live: bool, inventory_path: Path = INVENTORY,
+          adjudications: dict | None = None,
+          only_dims: set | None = None) -> dict:
+    """Resolve the whole inventory. Each dimension AXIS maps to a UMLS semantic
+    type (from adjudication accept_sty, else the inventory's semantic_type), and
+    that type's subtree constrains the search for the dimension's VALUE concepts
+    -- so axis and values reconcile by construction (a value is only ever mapped
+    within its axis's semantic branch).
+
+    ``only_dims`` restricts resolution to those dimensions (a scoped rebuild);
+    the returned doc then contains only their entries, and the caller is
+    responsible for merging into a full crosswalk."""
+    if adjudications is None:
+        adjudications = load_adjudications()
+    doc = yaml.safe_load(inventory_path.read_text())
+    out_entries = []
+    for dim, body in (doc.get("dimensions") or {}).items():
+        if only_dims is not None and dim not in only_dims:
+            continue
+        axis = body.get("axis")
+        axis_adj = adjudications.get(_adj_key(dim, None)) or {}
+        if axis_adj.get("unmapped"):
+            axis_sty = None                                   # curator: axis untyped
+        else:
+            axis_sty = axis_adj.get("accept_sty") or (axis or {}).get("semantic_type")
+        stys = stylib.filter_param(axis_sty)                  # value-search filter
+
+        if axis is not None:
+            e = _entry(dim, None, "axis", axis)
+            if not live:
+                e["status"] = "pending"
+            elif axis_sty:
+                sty = stylib.get(axis_sty)
+                if sty:
+                    e.update({"status": "mapped",
+                              "curated": bool(axis_adj.get("accept_sty")),
+                              "sty_tui": sty["tui"], "sty_name": sty["name"],
+                              "sty_tree": sty.get("tree_number"),
+                              "curator_note": axis_adj.get("note")})
+                else:
+                    e.update({"status": "review",
+                              "curator_error": f"semantic type {axis_sty} not in "
+                              "semantic_types.yaml (run fetch_semantic_network.py)"})
+            else:
+                e["status"] = "unmapped"
+                if axis_adj.get("unmapped"):
+                    e.update({"curated": True, "curator_note": axis_adj.get("note")})
+            out_entries.append(e)
+
+        for kind, seq in (("value", "values"), ("common_value", "common_values")):
+            for v in body.get(seq) or []:
+                e = _entry(dim, v["token"], kind, v)
+                res = resolve(client, e["query"], e.get("sab"), live, stys=stys)
+                entry = {**e, **res}
+                adj = adjudications.get(_adj_key(dim, v["token"]))
+                if adj and live:
+                    entry = apply_adjudication(entry, adj, client=client)
+                out_entries.append(entry)
+
+    counts = {s: 0 for s in ("mapped", "review", "unmapped", "pending")}
+    for e in out_entries:
+        counts[e["status"]] += 1  # KeyError surfaces any unexpected status
+    counts["curated"] = sum(1 for e in out_entries if e.get("curated"))
+    counts["axis_typed"] = sum(1 for e in out_entries if e.get("sty_tui"))
+    return {
+        "meta": {
+            "generated_by": "mapping/build_umls_crosswalk.py",
+            "source_inventory": "mapping/dimensions_inventory.yaml",
+            "live": live,
+            "total": len(out_entries),
+            "counts": counts,
+        },
+        "entries": out_entries,
+    }
+
+
+# --------------------------------------------------------------------------
+# Inventory <-> SHACL drift check.
+# --------------------------------------------------------------------------
+
+def shacl_enum_members() -> set[str]:
+    import rdflib
+    g = rdflib.Graph().parse(str(SHACL))
+    GEM = "https://w3id.org/genetic-evidence-model/"
+    enum_classes = {GEM + c for c in (
+        "KnowledgeDomain", "Method", "TargetType", "Resolution", "Credibility",
+        "PhenotypeScale", "VariantAscertainment", "MeasurementTarget",
+        "GeneRelation", "KnockoutType", "Penetrance")}
+    rdf_type = rdflib.RDF.type
+    members = set()
+    for s, _, o in g.triples((None, rdf_type, None)):
+        if str(o) in enum_classes and str(s).startswith(GEM):
+            members.add(str(s)[len(GEM):])
+    return members - ENUM_EXCLUDE
+
+
+# Dimensions whose values are intentionally NOT in the SHACL enumerations
+# (open / not schema-enumerated): the harness expects inventory tokens here
+# that have no SHACL member.
+OPEN_DIMS = {"subdomain", "mode_of_inheritance"}
+
+
+def check_inventory() -> int:
+    members = shacl_enum_members()
+    entries = load_entries()
+    inv_tokens = {str(e["token"]) for e in entries if e["token"] is not None}
+    missing = sorted(members - inv_tokens)
+    # Extra inventory tokens are expected for the open dimensions; any OTHER
+    # extra (a token that is neither a SHACL member nor in an open dimension)
+    # is surfaced as a warning so a stray/typo'd token does not hide.
+    open_tokens = {str(e["token"]) for e in entries
+                   if e["token"] is not None and e["dimension"] in OPEN_DIMS}
+    extra = sorted((inv_tokens - members) - open_tokens)
+    if missing:
+        print("INVENTORY DRIFT: SHACL enum members not covered by the inventory:")
+        for m in missing:
+            print(f"  - {m}")
+        return 1
+    print(f"Inventory covers all {len(members)} SHACL enum members "
+          f"(excluding sentinel + legacy synonyms).")
+    if extra:
+        print(f"WARNING: {len(extra)} inventory token(s) outside both the SHACL "
+              f"enums and the open dimensions ({', '.join(sorted(OPEN_DIMS))}):")
+        for e in extra:
+            print(f"  - {e}")
+    return 0
+
+
+# --------------------------------------------------------------------------
+# CLI
+# --------------------------------------------------------------------------
+
+def _make_client(args):
+    if args.check_inventory:
+        return None, False
+    if args.fixtures:
+        from forome.gem.umls.uts_client import FixtureClient
+        return FixtureClient(Path(args.fixtures)), True
+    key = args.api_key or os.environ.get("UMLS_API_KEY", "")
+    if key and not args.no_key:
+        from forome.gem.umls.uts_client import UTSClient
+        return UTSClient(key, cache_dir=CACHE_DIR), True
+    from forome.gem.umls.uts_client import NullClient
+    return NullClient(), False
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--api-key", help="UMLS UTS API key (else $UMLS_API_KEY).")
+    ap.add_argument("--no-key", action="store_true",
+                    help="Force scaffold mode: run with no UMLS query (all 'pending').")
+    ap.add_argument("--fixtures", help="Directory of canned UTS responses (offline tests).")
+    ap.add_argument("--out", default=str(DEFAULT_OUT), help="Output crosswalk YAML path.")
+    ap.add_argument("--check-inventory", action="store_true",
+                    help="Only verify the inventory still covers the SHACL enums; do not query.")
+    args = ap.parse_args(argv)
+
+    if args.check_inventory:
+        return check_inventory()
+
+    client, live = _make_client(args)
+    doc = build(client, live)
+    Path(args.out).write_text(
+        yaml.safe_dump(doc, sort_keys=False, allow_unicode=True, width=100))
+
+    c = doc["meta"]["counts"]
+    print(f"Wrote {args.out}")
+    print(f"  total={doc['meta']['total']}  mapped={c['mapped']}  "
+          f"review={c['review']}  unmapped={c['unmapped']}  pending={c['pending']}"
+          f"  (curated={c.get('curated', 0)})")
+    if not live:
+        print("  (scaffold mode: no UMLS key supplied -> all entries 'pending'. "
+              "Set UMLS_API_KEY and rerun to resolve concepts.)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

forome/gem/umls/classify_credibility_sweep.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Classify the credibility-sweep candidates against the adequacy criteria.
+
+Applies the rule families (regular expressions over the concept's preferred
+name) that assign each candidate the criterion it fails -- A denotation,
+B granularity, C set membership, D domain sense -- with a one-line reason.
+Together with ``gem-umls-sweep`` and ``data/umls/sweeps/credibility.yaml``
+this fully regenerates the curated results from any licensed UMLS copy; the
+results themselves are not redistributed (they carry UMLS-derived strings).
+
+Usage:
+    gem-umls-classify-credibility RESULTS.yaml [--check]
+
+``--check`` reports the classification without writing. A candidate no rule
+matches is left unclassified (``fails: null``) and reported -- extend RULES
+rather than forcing a code.
+"""
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from collections import Counter
+
+import yaml
+
+# (criterion, regex on the preferred name, one-line reason). Order matters:
+# specific families come before generic degree words.
+RULES: list[tuple[str, str, str]] = [
+ ("C", r"^(Definite|Probable|Possible|Probably|Possibly|Definitely|Unlikely|Certain)(\s*\(.*\))?$|^(Probable|Possible|Definite) diagnosis$",
+  "epistemic-modality qualifier (SNOMED diagnostic-certainty family: definite/probable/possible) — the nearest UMLS construct; denotes the likelihood that a proposition holds, not the defeasibility of trust in an evidence item; a three-point ladder with no intensified top, bound in its sources to diagnosis assertions (also D)"),
+ ("D", r"Related to Intervention",
+  "adverse-event causality attribution ladder (NCI: definitely/probably/possibly related) — degree of belief about causation of one event, not warrant for a research finding"),
+ ("D", r"TNM|degree of certainty of|Qualifier for TNM",
+  "SNOMED/TNM certainty factor (C-factor): certainty of a staging classification for one patient, a diagnostic-assertion qualifier, not evidence credibility"),
+ ("D", r"Level of Diagnostic Certainty|Diagnosis Confidence Level|UHDRS",
+  "certainty of a clinical diagnosis (rating-scale item), not warrant for a research finding (also B: names the dimension, not a level)"),
+ ("D", r"Level of evidence.*(Bld|Tiss|Molecular|Blood)",
+  "LOINC molecular-pathology 'level of evidence' (AMP/ASCO/CAP actionability tier for a variant's clinical significance): a clinical-actionability tier bound to a variant report, not curator warrant for an evidence item (also B: the attribute, not a level)"),
+ ("D", r"Metabolite Identification Confidence",
+  "metabolomics identification-confidence levels (Schymanski 1–5): an ordered epistemic ladder, but about confidence in a structure identification — domain-bound (D) and defined by analytical evidence type, not defeasibility"),
+ ("D", r"^Current level of confidence I can|^Current level of confidence|confidence:Find:Pt|Overall level of confidence with|I am confident|How confident|confident (that|about|in)|self[- ]?efficacy",
+  "patient-reported self-efficacy questionnaire item (LOINC/PROMIS): confidence in performing an activity, not warrant about evidence"),
+ ("D", r"^(Very High|High|Moderate|Low) Confidence$|^(Very |Somewhat |Not at all |Extremely |Fairly |Quite |Slightly )?Confident$|^Not Confident",
+  "NCI Clinical or Research Assessment Answer — a questionnaire response option for a respondent's confidence (Female Sexual Function Index subset), not an epistemic grade of evidence, despite matching GEM's four-level shape"),
+ ("D", r"self[- ]?esteem|Self Confidence|^Confidence$|Lack of confidence|Loss of confidence|confidence in (self|abilities)|Euphoric",
+  "psychological trait/state, not warrant about evidence"),
+ ("A", r"^Level of Evidence( (I|II|III|IV|V)[A-Za-z0-9]*)?$|Level of Evidence [IVX]+|^Evidence Level|^Level of evidence$",
+  "NCI/PDQ level-of-evidence: a study-design hierarchy (RCT > cohort > case series), not degree of warrant; mapping to it would collapse GEM's method facet into credibility (also C for the GEM scale)"),
+ ("A", r"Level of Confidence \(statistical\)|confidence interval|confidence limit|confidence coefficient|^Statistical",
+  "statistical quantity (interval/coefficient), not a degree of belief in evidence"),
+ ("A", r"probability|likelihood|risk score|risk category|score$|Score\b|Scale$|Index$|staging category",
+  "a quantitative score/probability or risk/staging category of a clinical measurand, not epistemic warrant"),
+ ("D", r"[Gg]rade|GRADE",
+  "'grade' in a non-epistemic sense (tumour, school, product grade)"),
+ ("D", r"[Cc]redib",
+  "credibility of a health-information source or witness as perceived by a patient/clinician (LOINC/CHV/NIC), not the curator's warrant for an evidence item"),
+ ("D", r"\[X\]|alcohol|[Ee]vidence of|evidence-based practice",
+  "clinical 'evidence of <condition>' sense (presence of a sign) or a care activity, not evidence strength"),
+ ("A", r"^(High|Very high|Very High|Extremely high|Abnormally high|Highest|Low|Very low|Lowest|Moderate|\(\+\) Moderate|Medium|Minimal|Maximal|Maximum|Minimum|Mild|Moderation|Strong|Weak|Weakness|Markedly increased|Increased|Decreased|Normal|Intermediate)(\s*\(.*\))?$",
+  "generic degree/qualifier value (SNOMED qualifier-value branch, IS-A Increased/Decreased): magnitude of a measurand, not epistemic degree"),
+ ("A", r"^(A )?(Medium|High|Low|Very high|Very low|Moderate|Minimal|Strong|Weak)( Amount| Level| Degree| Dose| Intensity| Frequency| Quality| Risk| Grade| Priority| Severity| Strength| Concentration| Pressure| Temperature| Volume| Density| Power| Speed| Altitude| Titer)?( of .*)?$",
+  "degree of a measured/graded property, not warrant about evidence"),
+ ("D", r"[Mm]edia\b|Tunica|Lowing|Lower \(action\)|spatial qualifier|Body Site Modifier|Culture Media",
+  "lexical collision (medium/media, lower) — unrelated sense"),
+ ("A", r"\b(high|low|moderate|medium|very high|very low|minimal|strong|weak|extremely high|highest|mild|maximal)\b",
+  "the qualifier appears as a magnitude/degree of a clinical or physical property (measurand degree), not as a degree of epistemic warrant"),
+]
+
+
+def classify(name: str) -> tuple[str, str] | None:
+    for code, rx, why in RULES:
+        if re.search(rx, name or "", re.I):
+            return code, why
+    return None
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="gem-umls-classify-credibility",
+                                 description=__doc__.splitlines()[0])
+    ap.add_argument("results", help="a gem-umls-sweep results YAML to classify")
+    ap.add_argument("--check", action="store_true",
+                    help="report only; do not write")
+    args = ap.parse_args(argv)
+    doc = yaml.safe_load(open(args.results))
+    dist: Counter = Counter()
+    residual = []
+    for c in doc.get("candidates") or []:
+        hit = classify(c.get("name"))
+        if hit is None:
+            residual.append(c)
+            continue
+        c["fails"], c["why"] = hit
+        dist[hit[0]] += 1
+    print(f"classified {sum(dist.values())} of "
+          f"{len(doc.get('candidates') or [])}: {dict(dist)}")
+    for c in residual:
+        print(f"  UNCLASSIFIED {c.get('cui')} {c.get('name')!r} "
+              f"{c.get('root_source')}")
+    if residual:
+        print("extend RULES for the unclassified candidates; nothing written"
+              if not args.check else "")
+        return 1
+    if not args.check:
+        yaml.safe_dump(doc, open(args.results, "w"), sort_keys=False,
+                       allow_unicode=True, width=110)
+        print(f"written: {args.results}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

forome/gem/umls/fetch_semantic_network.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Fetch the full UMLS Semantic Network (all semantic types + tree numbers) and
+render it as a hierarchical tree.
+
+Writes:
+  mapping/semantic_types.yaml        - the reference data (tui, name, tree, def)
+  mapping/semantic_network_tree.md   - the tree, annotated with where each GEM
+                                       dimension axis currently *derives* to
+                                       (the modal semantic type of its mapped
+                                       values), so untyped axes are visible.
+
+Use this to decide which dimension axes have a faithful UMLS semantic type and
+which are untyped (unmapped). Requires UMLS_API_KEY (env or .envrc).
+
+Usage:
+    python3 mapping/fetch_semantic_network.py
+    python3 mapping/fetch_semantic_network.py --max 220   # TUI scan upper bound
+"""
+from __future__ import annotations
+
+import argparse
+import re
+from collections import Counter, defaultdict
+
+import yaml
+
+from forome.gem.umls import build_umls_crosswalk as H
+from forome.gem.umls import adjudicate_ui as UI  # reuse api_key()
+from forome.gem.umls.uts_client import UTSClient
+from forome.gem.umls._paths import DATA_DIR
+
+TYPES_YAML = DATA_DIR / "semantic_types.yaml"
+TREE_MD = DATA_DIR / "semantic_network_tree.md"
+CROSSWALK = DATA_DIR / "umls_crosswalk.yaml"
+
+
+def tree_sort_key(tn: str):
+    key = []
+    for p in (tn or "").split("."):
+        m = re.match(r"([A-Za-z]*)(\d+)", p)
+        key.append((m.group(1), int(m.group(2))) if m else ("", 0))
+    return key
+
+
+def axis_derivations() -> dict:
+    """semantic-type name -> [dimensions whose mapped values are modally that type]."""
+    if not CROSSWALK.is_file():
+        return {}
+    doc = yaml.safe_load(CROSSWALK.read_text())
+    per = defaultdict(Counter)
+    for e in doc["entries"]:
+        if e["kind"] in ("value", "common_value") and e["status"] == "mapped":
+            for st in (e.get("semantic_types") or []):
+                per[e["dimension"]][st] += 1
+    name_to_axes = defaultdict(list)
+    for dim, cnt in per.items():
+        if cnt:
+            name_to_axes[cnt.most_common(1)[0][0]].append(dim)
+    return name_to_axes
+
+
+def fetch_all(client, upper: int) -> list:
+    types = []
+    for n in range(1, upper + 1):
+        st = client.get_semantic_type(f"T{n:03d}")
+        if st and st.get("tree_number"):
+            types.append(st)
+    types.sort(key=lambda s: tree_sort_key(s["tree_number"]))
+    return types
+
+
+def _is_type(s) -> bool:
+    return (s["tree_number"] or "").startswith(("A", "B"))
+
+
+def render_tree(types: list, derivations: dict) -> str:
+    sem = [s for s in types if _is_type(s)]          # semantic TYPES (A/B)
+    rel = [s for s in types if not _is_type(s)]       # relations (R)
+    lines = ["# UMLS Semantic Network — semantic types tree",
+             "",
+             f"_{len(sem)} semantic types (plus {len(rel)} relations, listed at "
+             "the end). Generated by mapping/fetch_semantic_network.py from the "
+             "live UTS API._",
+             "",
+             "Dimension **axes** map to a semantic **type** (A/B branches below); "
+             "dimension **values** map to Metathesaurus concepts. GEM axes are "
+             "annotated at the type their mapped values currently *derive* to "
+             "(modal). Axes with no typed values are untyped/unmapped candidates "
+             "(e.g. `target_type`, `resolution`, `credibility`, "
+             "`variant_ascertainment`).",
+             "", "## Semantic types", "", "```"]
+    for s in sem:
+        depth = len((s["tree_number"] or "").split("."))
+        indent = "  " * (depth - 1)
+        axes = derivations.get(s["name"])
+        tag = ("   <== GEM axis: " + ", ".join(sorted(axes))) if axes else ""
+        lines.append(f"{indent}{s['tree_number']:<12} {s['name']}  ({s['tui']}){tag}")
+    lines += ["```", "", "## Relations (for reference)", "", "```"]
+    for s in rel:
+        depth = len((s["tree_number"] or "").split("."))
+        lines.append(f"{'  ' * (depth - 1)}{s['tree_number']:<10} {s['name']}  ({s['tui']})")
+    lines.append("```")
+    return "\n".join(lines) + "\n"
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--max", type=int, default=220, help="TUI scan upper bound")
+    args = ap.parse_args(argv)
+
+    key = UI.api_key()
+    if not key:
+        print("ERROR: no UMLS_API_KEY (env or .envrc).")
+        return 1
+    client = UTSClient(key, cache_dir=H.CACHE_DIR)
+    print(f"Fetching semantic types T001..T{args.max:03d} (cached after first run)...")
+    types = fetch_all(client, args.max)
+    TYPES_YAML.write_text(yaml.safe_dump(
+        {"count": len(types), "source": "UMLS Semantic Network (UTS API)",
+         "types": types}, sort_keys=False, allow_unicode=True, width=200))
+    tree = render_tree(types, axis_derivations())
+    TREE_MD.write_text(tree)
+    print(f"Wrote {TYPES_YAML.name} ({len(types)} types) and {TREE_MD.name}.")
+    # top-level summary to stdout
+    tops = [s for s in types if len((s["tree_number"] or "").split(".")) <= 2]
+    print("\nTop-level branches:")
+    for s in tops:
+        print(f"  {s['tree_number']:<8} {s['name']} ({s['tui']})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())