gem-mapping-studio 0.2.2__py3-none-any.whl

forome/gem/extraction/extract_annotations.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Extract PDF highlights and callouts using PyMuPDF.
+
+For each highlight, PyMuPDF can give us the covered text directly via
+page.get_textbox(rect) or via the annotation's get_text() method. It also
+gives us better handling of Popup-linked callout notes.
+"""
+import sys
+import json
+import fitz  # PyMuPDF
+
+
+def rect_from_quadpoints(quads):
+    """Given the 8-float quadpoints, return a fitz.Rect bounding box."""
+    if not quads:
+        return None
+    xs = [quads[i] for i in range(0, len(quads), 2)]
+    ys = [quads[i] for i in range(1, len(quads), 2)]
+    return fitz.Rect(min(xs), min(ys), max(xs), max(ys))
+
+
+def main(pdf_path, output_json=None):
+    doc = fitz.open(pdf_path)
+    results = []
+
+    for page_num, page in enumerate(doc, start=1):
+        annot = page.first_annot
+        while annot is not None:
+            info = annot.info or {}
+            atype = annot.type  # (type_id, type_name)
+            type_name = atype[1] if atype else "?"
+            contents = info.get("content", "") or ""
+            author = info.get("title", "") or ""
+
+            entry = {
+                "page": page_num,
+                "type": type_name,
+                "author": author,
+                "note": contents.strip() if contents else "",
+                "rect": list(annot.rect),
+            }
+
+            if type_name == "Highlight":
+                # Try multiple extraction strategies and pick the best
+                texts = []
+
+                # 1. Quadpoints-based: iterate the vertices in groups of 4
+                try:
+                    verts = annot.vertices
+                    if verts:
+                        # vertices come as list of tuples; each highlight quad
+                        # uses 4 points
+                        for i in range(0, len(verts), 4):
+                            quad = verts[i:i+4]
+                            if len(quad) == 4:
+                                xs = [p[0] for p in quad]
+                                ys = [p[1] for p in quad]
+                                r = fitz.Rect(min(xs), min(ys),
+                                              max(xs), max(ys))
+                                t = page.get_textbox(r).strip()
+                                if t:
+                                    texts.append(t)
+                except Exception:
+                    pass
+
+                # 2. Fallback: use the annotation rect
+                if not texts:
+                    try:
+                        t = page.get_textbox(annot.rect).strip()
+                        if t:
+                            texts.append(t)
+                    except Exception:
+                        pass
+
+                highlighted = " ".join(texts).strip()
+                # Clean up the weird line breaks and excess whitespace
+                highlighted = " ".join(highlighted.split())
+                entry["highlighted_text"] = highlighted
+
+            results.append(entry)
+            annot = annot.next
+
+    # Filter out noise: empty highlights with no note
+    cleaned = []
+    for r in results:
+        if r["type"] == "Highlight":
+            if not r.get("highlighted_text") and not r.get("note"):
+                continue  # skip truly empty
+        cleaned.append(r)
+
+    if output_json:
+        with open(output_json, "w") as f:
+            json.dump(cleaned, f, indent=2)
+        print(f"Wrote {len(cleaned)} annotations to {output_json}")
+    else:
+        for r in cleaned:
+            print(f"\n[Page {r['page']}] {r['type']} (author: {r['author']})")
+            if r.get("note"):
+                print(f"  NOTE: {r['note']}")
+            if r.get("highlighted_text"):
+                print(f"  TEXT: {r['highlighted_text']}")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1],
+         output_json=sys.argv[2] if len(sys.argv) > 2 else None)

forome/gem/extraction/extract_annotations_pypdf.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Extract PDF annotations (highlights and text-note callouts) from a PDF.
+
+For each annotation on each page, report:
+  - annotation subtype (Highlight, Text, Popup, etc.)
+  - page number (1-indexed)
+  - the author of the annotation if present
+  - the 'Contents' field (what the annotator typed as a note)
+  - the quadpoints / rect (location on page)
+  - the text actually under the highlight, extracted by coordinates
+
+This matters because a 'Highlight' annotation stores WHERE it is, not WHAT it
+covers; the covered text has to be recovered from the page's text content
+using the quadpoints.
+"""
+
+import sys
+import pypdf
+from pypdf.generic import ArrayObject, NumberObject
+
+
+def extract_text_under_quads(page, quadpoints):
+    """Given a page and the Highlight's /QuadPoints, extract the text under
+    each quad. QuadPoints come in groups of 8: (x1,y1, x2,y2, x3,y3, x4,y4)
+    describing the four corners of each rectangle covered by the highlight.
+
+    We use pypdf's visitor-based text extraction to collect only text whose
+    position falls inside any quad rectangle.
+    """
+    if not quadpoints:
+        return ""
+
+    # Group quadpoints into (x_min, y_min, x_max, y_max) rects.
+    rects = []
+    q = [float(v) for v in quadpoints]
+    for i in range(0, len(q), 8):
+        xs = q[i:i+8:2]
+        ys = q[i+1:i+8:2]
+        rects.append((min(xs), min(ys), max(xs), max(ys)))
+
+    collected = []
+
+    def visitor(text, cm, tm, fontDict, fontSize):
+        # tm is the text matrix; its e,f are the x,y position.
+        try:
+            x = tm[4]
+            y = tm[5]
+        except Exception:
+            return
+        for (xmin, ymin, xmax, ymax) in rects:
+            # Allow a small slack in y because tm gives the baseline, not the
+            # character box.
+            if xmin - 1 <= x <= xmax + 1 and ymin - 4 <= y <= ymax + 4:
+                if text and text.strip():
+                    collected.append(text)
+                return
+
+    try:
+        page.extract_text(visitor_text=visitor)
+    except Exception as e:
+        return f"<extraction error: {e}>"
+
+    return " ".join(collected).strip()
+
+
+def main(pdf_path):
+    reader = pypdf.PdfReader(pdf_path)
+    print(f"PDF: {pdf_path}")
+    print(f"Pages: {len(reader.pages)}")
+    print()
+
+    # First pass: find all Popup annotations, keyed by IndirectRef of the
+    # Popup object itself. A Highlight often has a /Popup pointer to a
+    # Popup annotation which in turn has /Contents (the actual callout text).
+    popups_by_ref = {}
+    for page_idx, page in enumerate(reader.pages):
+        if "/Annots" not in page:
+            continue
+        for annot_ref in page["/Annots"]:
+            annot = annot_ref.get_object()
+            subtype = annot.get("/Subtype")
+            if subtype == "/Popup":
+                popups_by_ref[annot_ref.idnum] = annot
+
+    annot_count = 0
+    for page_idx, page in enumerate(reader.pages):
+        if "/Annots" not in page:
+            continue
+
+        for annot_ref in page["/Annots"]:
+            annot = annot_ref.get_object()
+            subtype = annot.get("/Subtype", "?")
+
+            # We are most interested in Highlight and Text (callout/sticky).
+            # Popups are auxiliary; they'll be read via their parent Highlight.
+            if subtype not in ("/Highlight", "/Text", "/Underline",
+                               "/Squiggly", "/StrikeOut"):
+                continue
+
+            annot_count += 1
+            contents = annot.get("/Contents", "")
+            author   = annot.get("/T", "")
+            quads    = annot.get("/QuadPoints", None)
+            rect     = annot.get("/Rect", None)
+
+            # If Highlight has a Popup with Contents, prefer that text
+            popup_ref = annot.get("/Popup")
+            popup_contents = ""
+            if popup_ref is not None:
+                try:
+                    popup = popup_ref.get_object()
+                    popup_contents = popup.get("/Contents", "") or ""
+                except Exception:
+                    pass
+
+            note_text = contents or popup_contents or ""
+
+            highlighted_text = ""
+            if subtype == "/Highlight" and quads is not None:
+                highlighted_text = extract_text_under_quads(page, quads)
+
+            print(f"--- Annotation #{annot_count} (page {page_idx+1}) ---")
+            print(f"  subtype: {subtype}")
+            if author:
+                print(f"  author: {author}")
+            if note_text:
+                print(f"  note/callout: {note_text!r}")
+            if highlighted_text:
+                print(f"  highlighted text: {highlighted_text!r}")
+            elif subtype == "/Text":
+                # Sticky note — no highlighted text, but has a location
+                if rect:
+                    print(f"  location (rect): {list(rect)}")
+            print()
+
+    print(f"Total annotations processed: {annot_count}")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])

forome/gem/extraction/yaml_to_rdf.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Convert a Genetic Evidence Model annotation YAML to RDF (Turtle).
+
+The emitted RDF is what the SHACL shapes in ``schema/genetic_evidence.shacl.ttl``
+validate. Every ``GeneticEvidence`` item becomes a ``gem:GeneticEvidence``
+node carrying its dimension values (as ``gem:`` IRIs for categorical
+dimensions, literals for free-text ones) and its assertions, each with a
+``gem:sourceSpan`` blank node.
+
+Design notes
+------------
+* Field-name aliases are normalized: ``resolution``/``target_resolution`` and
+  source-span ``key_phrase``/``phrase`` (the v0 -> v1 protocol rename).
+* Legacy method synonyms are mapped to canonical values
+  (``IN_VIVO_EXPERIMENT`` -> ``IN_VIVO`` etc.) so the shapes need only the
+  canonical enumeration.
+* The explicit escape value ``not_applicable_or_omitted`` (used when a
+  curator considered a dimension and chose not to fill it) is emitted as
+  ``gem:NOT_APPLICABLE_OR_OMITTED``; the shapes accept it as "addressed".
+
+Usage:
+    python3 extraction/yaml_to_rdf.py annotations/jossin2017.yaml [out.ttl]
+    python3 extraction/yaml_to_rdf.py annotations/*.yaml --stdout
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import yaml
+
+GEM = "https://w3id.org/genetic-evidence-model/"
+NA = "NOT_APPLICABLE_OR_OMITTED"
+
+# Categorical dimensions: YAML key -> gem: property local name. Values become
+# gem:<VALUE> IRIs (drawn from the enumerations declared in the SHACL file).
+CATEGORICAL = {
+    "knowledge_domain": "knowledgeDomain",
+    "method": "method",
+    "target_type": "targetType",
+    "resolution": "resolution",
+    "target_resolution": "resolution",   # v1 alias
+    "credibility": "credibility",
+    "phenotype_scale": "phenotypeScale",
+    "variant_ascertainment": "variantAscertainment",
+    "measurement_target": "measurementTarget",
+    "gene_relation": "geneRelation",
+    "knockout_type": "knockoutType",
+    "mode_of_inheritance": "modeOfInheritance",
+    "penetrance": "penetrance",
+}
+
+# Free-text dimensions emitted as plain literals.
+LITERAL_DIMS = {
+    "organism": "organism",
+    "target": "target",
+    "specificity_of_phenotype": "specificityOfPhenotype",
+    "subdomain": "subdomain",   # enum values contain spaces ("Candidate Gene Study")
+}
+
+# A value safe to use as a Turtle IRI local name (no spaces/punctuation).
+import re as _re
+_TOKEN = _re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+# Boolean dimensions.
+BOOLEAN_DIMS = {
+    "mendelian_segregation": "mendelianSegregation",
+    "exact_variant": "exactVariant",
+    "genetic_background_considered": "geneticBackgroundConsidered",
+}
+
+# Legacy method synonyms -> canonical value (dimensions.md "Backward compatibility").
+METHOD_SYNONYMS = {
+    "IN_VIVO_EXPERIMENT": "IN_VIVO",
+    "IN_VITRO_EXPERIMENT": "IN_VITRO",
+    "BIOINFORMATICS_PREDICTION": "BIOINFORMATICS_INFERENCE",
+}
+
+
+def esc(text: str) -> str:
+    """Escape a string for a Turtle long-literal (triple-quoted)."""
+    return str(text).replace("\\", "\\\\").replace('"""', '\\"\\"\\"')
+
+
+def _values(raw):
+    """Normalise a YAML dimension value into a list of scalar strings."""
+    if raw is None:
+        return []
+    if isinstance(raw, list):
+        return [v for v in raw if v is not None]
+    if isinstance(raw, dict):
+        # e.g. hpo_terms: {annotated_as: not_applicable_or_omitted, ...}
+        if raw.get("annotated_as"):
+            return [raw["annotated_as"]]
+        return []
+    return [raw]
+
+
+def _iri_value(prop: str, val) -> str:
+    """Map a scalar dimension value to a gem: IRI local name."""
+    s = str(val).strip()
+    if s.lower() == "not_applicable_or_omitted":
+        return NA
+    if prop == "method":
+        s = METHOD_SYNONYMS.get(s, s)
+    return s
+
+
+def item_triples(item: dict, idx: int) -> list[str]:
+    """Return a list of Turtle blocks for one GeneticEvidence item."""
+    raw_id = str(item.get("id", f"item{idx}"))
+    node = f"gem:item_{raw_id.replace('.', '_').replace('-', '_')}"
+    item_props = [f'gem:itemId "{esc(raw_id)}"']
+    extra_blocks = []  # assertion + source-span nodes, declared separately
+
+    for key, prop in CATEGORICAL.items():
+        for v in _values(item.get(key)):
+            tok = _iri_value(prop, v)
+            if _TOKEN.match(tok):
+                item_props.append(f"gem:{prop} gem:{tok}")
+            else:  # value is not a clean enum token; keep it as a literal
+                item_props.append(f'gem:{prop} """{esc(v)}"""')
+
+    for key, prop in LITERAL_DIMS.items():
+        if item.get(key) not in (None, ""):
+            item_props.append(f'gem:{prop} """{esc(item[key])}"""')
+
+    for key, prop in BOOLEAN_DIMS.items():
+        if item.get(key) is not None:
+            b = "true" if item[key] in (True, "true", "True") else "false"
+            item_props.append(f"gem:{prop} {b}")
+
+    for a_i, a in enumerate(item.get("assertions", []) or []):
+        an = f"{node}_a{a_i}"
+        span = a.get("source_span") or {}
+        phrase = span.get("phrase", span.get("key_phrase"))
+        page = span.get("page")
+        a_props = [f'gem:assertionId "{esc(a.get("id", f"{raw_id}.A{a_i}"))}"']
+        if a.get("statement"):
+            a_props.append(f'gem:statement """{esc(a["statement"])}"""')
+        if phrase is not None or page is not None:
+            sp_props = []
+            if page is not None:
+                sp_props.append(f"gem:page {int(page)}")
+            if phrase is not None:
+                sp_props.append(f'gem:phrase """{esc(phrase)}"""')
+            extra_blocks.append(
+                f"{an}_span a gem:SourceSpan ;\n    "
+                + " ;\n    ".join(sp_props) + " .")
+            a_props.append(f"gem:sourceSpan {an}_span")
+        extra_blocks.append(
+            f"{an} a gem:GeneticEvidenceAssertion ;\n    "
+            + " ;\n    ".join(a_props) + " .")
+        item_props.append(f"gem:assertion {an}")
+
+    item_block = (f"{node} a gem:GeneticEvidence ;\n    "
+                  + " ;\n    ".join(item_props) + " .")
+    return [item_block] + extra_blocks
+
+
+def convert(path: Path) -> str:
+    data = yaml.safe_load(path.read_text())
+    triples = [f"@prefix gem: <{GEM}> .",
+               f"@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .",
+               f"# Generated from {path.name} by extraction/yaml_to_rdf.py", ""]
+    for idx, item in enumerate(data.get("evidence", []) or []):
+        triples.extend(item_triples(item, idx))
+        triples.append("")
+    return "\n".join(triples) + "\n"
+
+
+def main(argv):
+    args = [a for a in argv if not a.startswith("--")]
+    to_stdout = "--stdout" in argv
+    if not args:
+        print(__doc__)
+        return 2
+    for src in args:
+        p = Path(src)
+        ttl = convert(p)
+        if to_stdout or len(args) == 1 and len(argv) == 1:
+            sys.stdout.write(ttl)
+        else:
+            out = p.with_suffix(".ttl")
+            out.write_text(ttl)
+            print(f"wrote {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

forome/gem/umls/_paths.py

@@ -0,0 +1,58 @@
+"""Locate the repository's data and schema directories.
+
+The UMLS crosswalk tooling is a repo-run data project: its inputs/outputs live
+in ``data/umls/`` and the schema in ``schema/`` at the repo root, discovered by
+walking up from this module. ``GEM_DATA_DIR`` overrides the data directory
+(used by tests / alternate checkouts).
+
+Standalone installs (``pip install forome-gem``, no repo checkout) fall back
+to the synced reference copies packaged in ``forome.gem._reference`` for the
+Semantic Network and the SHACL schema, and to the current working directory
+as the default workspace (the Studio treats an empty directory as a fresh
+workspace).
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+_HERE = Path(__file__).resolve()
+
+
+def _repo_root() -> Path | None:
+    """The repo checkout this module runs from, or None (standalone install)."""
+    for cand in _HERE.parents:
+        if (cand / "data" / "umls").is_dir() and (cand / "schema").is_dir():
+            return cand
+    for cand in _HERE.parents:
+        if (cand / "pyproject.toml").is_file():
+            return cand
+    return None
+
+
+_REPO = _repo_root()
+_PACKAGED = _HERE.parent.parent / "_reference"   # forome/gem/_reference
+
+# Kept for repo-mode consumers; in standalone mode it points at the package's
+# parent purely so derived paths stay well-defined (they will not exist).
+REPO_ROOT = _REPO if _REPO is not None else _HERE.parents[6] if len(_HERE.parents) > 6 else _HERE.parent
+
+# The workspace: inventory / crosswalk / adjudications live here. ``GEM_DATA_DIR``
+# points the tooling at any mapping directory (which may be empty, e.g. when
+# building axes from scratch). Standalone default: the current directory.
+if "GEM_DATA_DIR" in os.environ:
+    DATA_DIR = Path(os.environ["GEM_DATA_DIR"])
+elif _REPO is not None:
+    DATA_DIR = _REPO / "data" / "umls"
+else:
+    DATA_DIR = Path.cwd()
+
+# Canonical reference data (the UMLS Semantic Network) resolves to the repo's
+# copy when in a checkout — independent of the workspace, so type exploration
+# works even when GEM_DATA_DIR is an empty directory — and to the packaged
+# synced copy otherwise.
+REFERENCE_DIR = (_REPO / "data" / "umls") if _REPO is not None else _PACKAGED
+# The SHACL schema and dimensions.md: repo ``schema/`` or the packaged copies
+# (the packaged layout is flat: the same filenames, no ``schema/`` prefix).
+SCHEMA_DIR = (_REPO / "schema") if _REPO is not None else _PACKAGED
+PAPER_SECTIONS = REPO_ROOT / "paper" / "sections"