verdy 0.1.0__py3-none-any.whl

verdy/__init__.py

@@ -0,0 +1,34 @@
+"""VERDY: predict matches for your ENA IDs or free-text terms in the ontologies you choose.
+
+Part of VERDANT: Vocabulary Enrichment & Resolution of Diversity Annotations via Neural Technology.
+
+Input (a CSV / TSV / Excel file, a text file or a Python list) holds ENA IDs (samples,
+runs, projects) and/or free text (species, habitats, parameters, measurements). ENA IDs
+are looked up in ENA for their organism and environment fields; every name is searched in
+each ontology you choose; out come its exact and most related terms.
+
+    import verdy
+
+    matches = verdy.match("my_input.csv", ontologies=["natura2000", "gold", "envo", "ncbitaxon"])
+    verdy.save(matches, "my_input.matches.csv")        # or .tsv / .sssom.tsv / .json / .xlsx
+    verdy.match.unresolved                             # ENA IDs ENA did not know
+    verdy.unmatched("my_input.csv", matches, ["envo"]) # names an ontology found nothing for
+
+    verdy.ONTOLOGIES                       # the ready ones; any OLS4 ontology ID works too
+    verdy.search("air temperature", "cf")  # one name in one ontology, its best terms
+    verdy.resolve("SAMD00132352")          # what ENA says about an ID
+
+From the shell:  verdy my_input.csv -o natura2000,gold,envo --out my_input.matches.csv
+"""
+
+from verdy.match import MATCH_TYPES, match, read_terms, resolve_terms, unmatched
+from verdy.resolve import SAMPLE_FIELDS, resolve
+from verdy.output import save, to_dataframe, to_sssom, to_table
+from verdy.sources import ONTOLOGIES, ontology_info, search
+
+__version__ = "0.1.0"
+
+predict = match          # the same: IDs or names in, predicted matches out
+
+__all__ = ["match", "predict", "resolve", "resolve_terms", "SAMPLE_FIELDS", "search", "read_terms", "unmatched", "save", "to_table", "to_sssom", "to_dataframe",
+           "ONTOLOGIES", "MATCH_TYPES", "ontology_info", "__version__"]

verdy/__main__.py

@@ -0,0 +1,60 @@
+"""Command line:  verdy TERMS -o envo,pato,cf [--column NAME] [--out FILE]
+
+    verdy my_terms.csv -o envo,ncbitaxon,gbif --out my_terms.matches.csv
+    verdy my_terms.csv -o pato,cf,uo                       # print a table instead
+    verdy --list                                           # the ready ontologies
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from verdy.match import match, read_terms, resolve_terms, unmatched
+from verdy.output import save, to_table
+from verdy.sources import ONTOLOGIES
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="verdy", description="Match your terms to the ontologies you choose.")
+    ap.add_argument("terms", nargs="?", help="CSV / TSV / Excel with an ID or name column, or a text file with one per line")
+    ap.add_argument("-o", "--ontologies", help="comma-separated IDs: see --list; any OLS4 ontology ID works too")
+    ap.add_argument("--column", help="the column holding the terms (found by its header otherwise)")
+    ap.add_argument("--out", type=Path, help="write here: .csv, .tsv, .sssom.tsv, .json or .xlsx (default: print)")
+    ap.add_argument("--per-ontology", type=int, default=3, help="at most this many matches per term and ontology (3)")
+    ap.add_argument("--min-score", type=float, default=0.34, help="least word overlap for a related match (0.34)")
+    ap.add_argument("--list", action="store_true", help="list the ready ontologies")
+    args = ap.parse_args(argv)
+    if args.list:
+        for k, v in ONTOLOGIES.items():
+            print(f"{k:11} {v}")
+        return 0
+    if not args.terms or not args.ontologies:
+        ap.error("give a TERMS file and --ontologies (or --list)")
+    try:
+        terms = read_terms(args.terms, args.column)
+        matches = match(terms, args.ontologies, per_ontology=args.per_ontology,
+                        min_score=args.min_score,
+                        progress=lambda d, t: print(f"\r{d} of {t} lookups", end="", file=sys.stderr))
+    except ValueError as exc:
+        ap.error(str(exc))
+    print(file=sys.stderr)
+    unresolved = match.unresolved
+    if unresolved:
+        print(f"nothing found for {len(unresolved)} IDs: {', '.join(unresolved[:20])}"
+              + (" ..." if len(unresolved) > 20 else ""), file=sys.stderr)
+    if args.out:
+        save(matches, args.out)
+        print(f"{len(matches)} matches for {len({m['term_id'] for m in matches})} of {len(terms)} inputs -> {args.out}",
+              file=sys.stderr)
+    else:
+        sys.stdout.write(to_table(matches, "\t"))
+    for o, missing in unmatched(resolve_terms(terms)[0], matches, args.ontologies).items():
+        if missing:
+            print(f"nothing in {o} for: {', '.join(missing)}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

verdy/match.py

@@ -0,0 +1,197 @@
+"""Predict matches for your IDs or terms in the ontologies you choose.
+
+    import verdy
+    matches = verdy.match("my_samples.csv", ontologies=["natura2000", "gold", "envo"])
+    verdy.save(matches, "my_samples.matches.csv")
+
+1. An ENA ID (sample, run or project) is looked up in ENA for its sample's organism
+   (scientific_name), broad_scale_environmental_context, local_environmental_context,
+   environmental_medium, ...; any other value is free text and is searched as it is.
+2. Each name is searched in each chosen ontology on its own:
+
+    exactMatch     a term's label or synonym is the name (plurals and 'sp.' aside)
+    closeMatch     most of their words are shared (overlap >= 0.6)
+    relatedMatch   some words are shared (overlap >= ``min_score``)
+
+Up to ``per_ontology`` matches per term and ontology; when there is an exact one, only it.
+In the taxonomies (NCBITaxon, GBIF) only exact names count: a near name is another taxon.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import re
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from verdy.resolve import SAMPLE_FIELDS, kind, resolve
+from verdy.sources import TAXONOMIES, ontology_info, query_text, search
+
+NAME_COLUMNS = ("name", "label", "term", "parameter", "measurement", "variable", "scientific_name", "scientificname",
+                "species", "taxon", "habitat", "biome", "text", "value", "description")
+ID_COLUMNS = ("id", "subject_id", "identifier", "accession", "sample_accession", "run_accession", "study_accession",
+              "sample_id", "sample", "key", "source", "code")
+CATEGORY_COLUMNS = ("category", "group", "type", "kind", "field")
+MATCH_TYPES = ("skos:exactMatch", "skos:closeMatch", "skos:relatedMatch")
+
+
+# --- your terms ---------------------------------------------------------------------
+
+def _slug(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", str(text).lower()).strip("_")[:60] or "term"
+
+
+def _pick(columns, wanted) -> str | None:
+    low = {str(c).strip().lower(): c for c in columns}
+    return next((low[w] for w in wanted if w in low), None)
+
+
+def read_terms(data, column: str | None = None) -> list[dict]:
+    """``[{"id", "label", "category"}]`` from a CSV / TSV / Excel file, a text file with one
+    value per line, a list of strings or dicts, or a pandas DataFrame. The column read is
+    given with ``column=``, or found by its header: a name column (name, label, term,
+    parameter, measurement, species, ...) if there is one, else an ID column (id, accession,
+    sample_accession, run_accession, ...). Repeats once."""
+    if hasattr(data, "to_dict") and hasattr(data, "columns"):            # pandas DataFrame
+        rows = data.fillna("").astype(str).to_dict("records")
+    elif isinstance(data, (list, tuple)):
+        rows = [r if isinstance(r, dict) else {"name": r} for r in data]
+    else:
+        path = Path(data)
+        if path.suffix.lower() in (".xlsx", ".xls"):
+            import pandas as pd
+            rows = pd.read_excel(path, dtype=str).fillna("").to_dict("records")
+        else:
+            text = path.read_text(encoding="utf-8-sig")
+            lines = [ln for ln in text.splitlines() if ln.strip() and not ln.startswith("#")]
+            delim = "\t" if path.suffix.lower() in (".tsv", ".txt") or text.count("\t") > text.count(",") else ","
+            first = lines[0] if lines else ""
+            header = _pick(next(csv.reader([first], delimiter=delim)), NAME_COLUMNS + ID_COLUMNS) if lines else None
+            if column is None and header is None and (delim not in first or path.suffix.lower() == ".txt"):
+                rows = [{"name": ln.strip()} for ln in lines]                 # one term per line, no header
+            else:
+                rows = list(csv.DictReader(io.StringIO("\n".join(lines)), delimiter=delim))
+    if not rows:
+        return []
+    cols = list(rows[0])
+    id_col, cat_col = _pick(cols, ID_COLUMNS), _pick(cols, CATEGORY_COLUMNS)
+    name_col = column or _pick(cols, NAME_COLUMNS) or id_col
+    if name_col is None:      # the first column with words in it
+        name_col = next((c for c in cols if c != id_col and any(re.search(r"[A-Za-z]{3}", str(r.get(c) or "")) for r in rows[:20])), None)
+    if name_col is None or name_col not in cols:
+        raise ValueError(f"No name column found; give one with column= (columns: {', '.join(map(str, cols))}).")
+    out, seen = [], set()
+    for r in rows:
+        label = str(r.get(name_col) or "").strip()
+        if not label or label.lower() == "nan":
+            continue
+        tid = (str(r.get(id_col) or "").strip() if id_col and id_col != name_col else "") or _slug(label)
+        if tid in seen:
+            continue
+        seen.add(tid)
+        out.append({"id": tid, "label": label, "category": (str(r.get(cat_col) or "").strip() or None) if cat_col else None})
+    return out
+
+
+# --- IDs to names --------------------------------------------------------------------
+
+def resolve_terms(terms: list[dict], fields=SAMPLE_FIELDS, workers: int = 6) -> tuple[list[dict], list[str]]:
+    """Terms whose value is an ID become one term per name found for it (``id`` the ID,
+    ``category`` the field(s) it came from); the IDs nothing was found for come second."""
+    ids = [t for t in terms if kind(t["label"]) != "name"]
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        found = dict(zip((t["label"] for t in ids), pool.map(lambda t: resolve(t["label"], fields), ids)))
+    out, unresolved = [], []
+    for t in terms:
+        if t["label"] not in found:
+            out.append(t)
+            continue
+        names = found[t["label"]]
+        if not names:
+            unresolved.append(t["label"] + (" (no answer from its source)" if names is None else ""))
+            continue
+        merged: dict[tuple[str, str], list[str]] = {}
+        for field, value in names:
+            sample, _, field = field.rpartition(":")         # a project's values come per sample
+            merged.setdefault((sample or t["label"], value), []).append(field)
+        for (tid, value), fs in merged.items():
+            out.append({"id": tid, "label": value, "category": "|".join(dict.fromkeys(fs)),
+                        **({"input": t["label"]} if tid != t["label"] else {})})
+    return out, unresolved
+
+
+# --- matching ---------------------------------------------------------------------------
+
+def _ontologies(ontologies) -> list[str]:
+    if isinstance(ontologies, str):
+        ontologies = re.split(r"[,\s]+", ontologies)
+    return list(dict.fromkeys(o.strip().lower() for o in ontologies or [] if o and o.strip()))
+
+
+def _matches(term: dict, ontology: str, short: str, hits: list[dict], per_ontology: int, min_score: float) -> list[dict]:
+    exact = [h for h in hits if h["exact"]]
+    if ontology in TAXONOMIES or exact:
+        chosen = exact[:1]
+    else:
+        chosen = [h for h in hits if h["score"] >= min_score][:per_ontology]
+    out = []
+    for rank, h in enumerate(chosen, 1):
+        if h["exact"]:
+            kind, conf = "skos:exactMatch", 1.0 if h["same"] else 0.95
+        elif h["score"] >= 0.6:
+            kind, conf = "skos:closeMatch", round(0.5 + 0.4 * h["score"], 3)
+        else:
+            kind, conf = "skos:relatedMatch", round(0.3 + 0.4 * h["score"], 3)
+        out.append({"term_id": term["id"], "term": term["label"], "category": term.get("category"),
+                    "match_type": kind, "ontology": short, "match_id": h["id"], "match_curie": h["curie"],
+                    "match_label": h["label"], "confidence": conf, "word_overlap": h["score"], "rank": rank,
+                    "definition": h.get("definition"), "searched_for": query_text(term["label"])})
+    return out
+
+
+def match(data, ontologies, *, column: str | None = None, per_ontology: int = 3, min_score: float = 0.34,
+          fields=SAMPLE_FIELDS, progress=None, workers: int = 6) -> list[dict]:
+    """Matches for every ID or term in ``data`` in each of ``ontologies``: one dict per match
+    with term_id (your ID, or the term), term (the name searched), category (the field it
+    came from), match_type, ontology, match_id, match_curie, match_label, confidence,
+    word_overlap, rank, definition, searched_for.
+
+    ``ontologies``: a list or a comma-separated string of IDs (``verdy.ONTOLOGIES`` or any
+    OLS4 ontology). ``fields``: which ENA sample fields to search for a sample, run or project.
+    ``progress(done, total)`` is called as names are looked up. IDs nothing was found for
+    are in ``match.unresolved`` after the call."""
+    chosen = _ontologies(ontologies)
+    if not chosen:
+        raise ValueError("Choose at least one ontology, e.g. ontologies=['envo', 'ncbitaxon'].")
+    terms = data if isinstance(data, list) and data and isinstance(data[0], dict) and {"id", "label"} <= set(data[0]) \
+        else read_terms(data, column)
+    terms, match.unresolved = resolve_terms(terms, fields, workers)
+    shorts = {}
+    for o in chosen:
+        info = ontology_info(o)
+        if info is None:
+            raise ValueError(f"Unknown ontology “{o}”: use one of verdy.ONTOLOGIES or an OLS4 ontology ID.")
+        shorts[o] = info["short"]
+
+    tasks = [(t, o) for o in chosen for t in terms]
+    out, done = [], 0
+    with ThreadPoolExecutor(max_workers=2 if "wikidata" in chosen else workers) as pool:
+        for t, o, hits in pool.map(lambda task: (*task, search(task[0]["label"], task[1])), tasks):
+            done += 1
+            if progress:
+                progress(done, len(tasks))
+            if hits:
+                out.extend(_matches(t, o, shorts[o], hits, per_ontology, min_score))
+    return out
+
+
+def unmatched(data, matches: list[dict], ontologies) -> dict[str, list[str]]:
+    """Per ontology, the terms that got no match in it."""
+    terms = data if isinstance(data, list) and data and isinstance(data[0], dict) and "label" in data[0] else read_terms(data)
+    got = {(m["term_id"], m["ontology"].lower()) for m in matches}
+    out = {}
+    for o in _ontologies(ontologies):
+        short = (ontology_info(o) or {}).get("short", o).lower()
+        out[o] = [t["label"] for t in terms if (t["id"], short) not in got]
+    return out

verdy/output.py

@@ -0,0 +1,87 @@
+"""Matches out: a table (CSV / TSV), SSSOM, JSON, or a pandas DataFrame."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import re
+from pathlib import Path
+
+COLUMNS = ["term_id", "term", "category", "match_type", "ontology", "match_curie", "match_label", "confidence",
+           "word_overlap", "rank", "match_id", "definition", "searched_for"]
+PREFIXES = {"ENVO": "http://purl.obolibrary.org/obo/ENVO_", "NCBITaxon": "http://purl.obolibrary.org/obo/NCBITaxon_",
+            "PATO": "http://purl.obolibrary.org/obo/PATO_", "OBA": "http://purl.obolibrary.org/obo/OBA_",
+            "UO": "http://purl.obolibrary.org/obo/UO_", "CHEBI": "http://purl.obolibrary.org/obo/CHEBI_",
+            "OBI": "http://purl.obolibrary.org/obo/OBI_", "UBERON": "http://purl.obolibrary.org/obo/UBERON_",
+            "PO": "http://purl.obolibrary.org/obo/PO_", "GO": "http://purl.obolibrary.org/obo/GO_",
+            "GBIF": "https://www.gbif.org/species/", "wikidata": "http://www.wikidata.org/entity/",
+            "CF": "http://vocab.nerc.ac.uk/standard_name/", "GOLD": "https://w3id.org/gold.vocab/",
+            "GOLD.path": "https://w3id.org/gold.path/", "n2k": "https://w3id.org/verdy/natura2000/"}   # local: Natura 2000 has no URIs of its own
+
+
+def to_table(matches: list[dict], sep: str = ",") -> str:
+    buf = io.StringIO()
+    w = csv.DictWriter(buf, fieldnames=COLUMNS, delimiter=sep, extrasaction="ignore", lineterminator="\n")
+    w.writeheader()
+    w.writerows(matches)
+    return buf.getvalue()
+
+
+def _subject(term_id: str) -> str:
+    """Your ID as a CURIE: ENA accessions with their registry prefix, other IDs as they are,
+    free text under term:."""
+    t = str(term_id)
+    if re.match(r"^SAM[NED][A-Z]?\d+$", t):
+        return "biosample:" + t
+    if re.match(r"^([EDS]R[RXSP]|PRJ[EDN][A-Z])\d+$", t):
+        return "ena.embl:" + t
+    if re.match(r"^[A-Za-z][\w.]*:\S+$", t) or t.startswith("http"):
+        return t
+    if re.match(r"^Q\d+$", t):
+        return "wikidata:" + t
+    return "term:" + re.sub(r"\s+", "_", t)
+
+
+def to_sssom(matches: list[dict], name: str = "matches") -> str:
+    """SSSOM TSV: lexical matches with their confidence, subjects as ``term:<term_id>``."""
+    used = {m["match_curie"].split(":", 1)[0] for m in matches if ":" in m["match_curie"]}
+    header = ["# curie_map:", "#   term: https://w3id.org/verdy/term/",
+              "#   biosample: https://www.ebi.ac.uk/biosamples/samples/", "#   ena.embl: https://www.ebi.ac.uk/ena/browser/view/"] + \
+        [f"#   {p}: {u}" for p, u in PREFIXES.items() if p in used] + [
+        "#   skos: http://www.w3.org/2004/02/skos/core#", "#   semapv: https://w3id.org/semapv/vocab/",
+        f"# mapping_set_id: https://w3id.org/verdy/{re.sub(r'[^A-Za-z0-9_.-]+', '-', name)}.sssom.tsv",
+        "# mapping_set_description: Lexical matches found by VERDY; not reviewed."]
+    fields = ["subject_id", "subject_label", "subject_category", "predicate_id", "object_id", "object_label",
+              "mapping_justification", "confidence", "object_source", "mapping_tool", "comment"]
+    buf = io.StringIO()
+    buf.write("\n".join(header) + "\n")
+    w = csv.DictWriter(buf, fieldnames=fields, delimiter="\t", lineterminator="\n")
+    w.writeheader()
+    for m in matches:
+        w.writerow({"subject_id": _subject(m["term_id"]), "subject_label": m["term"],
+                    "subject_category": m.get("category") or "", "predicate_id": m["match_type"],
+                    "object_id": m["match_curie"], "object_label": m["match_label"] or "",
+                    "mapping_justification": "semapv:LexicalMatching", "confidence": m["confidence"],
+                    "object_source": m["ontology"], "mapping_tool": "verdy",
+                    "comment": f"word overlap {m['word_overlap']}; rank {m['rank']}"})
+    return buf.getvalue()
+
+
+def to_dataframe(matches: list[dict]):
+    import pandas as pd
+    return pd.DataFrame(matches, columns=COLUMNS)
+
+
+def save(matches: list[dict], path) -> Path:
+    """By the extension: .csv, .tsv, .sssom.tsv, .json (or .xlsx with pandas)."""
+    path = Path(path)
+    if path.name.endswith(".sssom.tsv"):
+        path.write_text(to_sssom(matches, path.name[: -len(".sssom.tsv")]), encoding="utf-8")
+    elif path.suffix.lower() == ".json":
+        path.write_text(json.dumps(matches, indent=1, ensure_ascii=False), encoding="utf-8")
+    elif path.suffix.lower() == ".xlsx":
+        to_dataframe(matches).to_excel(path, index=False)
+    else:
+        path.write_text(to_table(matches, "\t" if path.suffix.lower() == ".tsv" else ","), encoding="utf-8")
+    return path

verdy/resolve.py

@@ -0,0 +1,85 @@
+"""ENA IDs to the names worth searching for (ENA Portal API).
+
+    resolve("SAMD00132352") -> [("scientific_name", "air metagenome"),
+                                ("broad_scale_environmental_context", "urban biome"), ...]
+    resolve("ERR10002200")  -> the run's sample, as above
+    resolve("PRJEB54740")   -> every sample of the project, as "<sample>:<field>"
+
+Samples (SAMN/SAMEA/SAMD..., ERS/SRS/DRS...), runs and experiments (ERR/SRR/DRR...,
+ERX/SRX/DRX...) and projects (PRJEB/PRJNA/PRJDB..., ERP/SRP/DRP...). Anything else is
+free text and is searched as it is.
+"""
+
+from __future__ import annotations
+
+import re
+
+from verdy import sources
+from verdy.sources import Cache
+
+ENA_API = "https://www.ebi.ac.uk/ena/portal/api"
+# ENA sample fields that say what a sample is; ``fields=`` chooses others (see ENA's returnFields).
+SAMPLE_FIELDS = ("scientific_name", "broad_scale_environmental_context", "local_environmental_context",
+                 "environmental_medium", "environment_biome", "environment_feature", "environment_material",
+                 "isolation_source", "host_scientific_name", "host_body_site", "tissue_type")
+EMPTY = {"", "missing", "not applicable", "not collected", "not provided", "na", "n/a", "unknown", "none"}
+
+SAMPLE = re.compile(r"^(SAM[NED][A-Z]?\d+|[EDS]RS\d+)$", re.I)
+RUN = re.compile(r"^[EDS]R[RX]\d+$", re.I)
+PROJECT = re.compile(r"^(PRJ[EDN][A-Z]\d+|[EDS]RP\d+)$", re.I)
+
+
+def kind(identifier: str) -> str:
+    """What sort of ID this is, or 'name' when it is not one."""
+    s = str(identifier or "").strip()
+    for name, pattern in (("sample", SAMPLE), ("run", RUN), ("project", PROJECT)):
+        if pattern.match(s):
+            return name
+    return "name"
+
+
+def ena_sample(accession: str, fields=SAMPLE_FIELDS) -> list[tuple[str, str]] | None:
+    """The sample's (field, value) pairs from ENA; [] when ENA does not know it."""
+    acc = accession.upper()
+    fields = tuple(fields)
+
+    def fetch():
+        d = sources._json(f"{ENA_API}/search", result="sample", format="json", fields=",".join(fields),
+                          query=f'sample_accession="{acc}" OR secondary_sample_accession="{acc}"')
+        return d if d is None else (d[0] if isinstance(d, list) and d else {})
+    row = Cache("resolve_ena_sample").get(f"{acc}|{','.join(fields)}", fetch)
+    if row is None:
+        return None
+    return [(f, str(row.get(f)).strip()) for f in fields if str(row.get(f) or "").strip().lower() not in EMPTY]
+
+
+def ena_samples_of(accession: str) -> list[str] | None:
+    """The sample accessions of an ENA run, experiment or project."""
+    acc = accession.upper()
+
+    def fetch():
+        d = sources._json(f"{ENA_API}/filereport", accession=acc, result="read_run", fields="sample_accession",
+                          format="json", limit=0)
+        return d if d is None else list(dict.fromkeys(r["sample_accession"] for r in d or [] if r.get("sample_accession")))
+    return Cache("resolve_ena_runs").get(acc, fetch)
+
+
+def resolve(identifier, fields=SAMPLE_FIELDS) -> list[tuple[str, str]] | None:
+    """``[(field, name), ...]`` to search for: an ENA ID's sample fields (None when ENA did not
+    answer, [] when it does not know the ID), or free text as it is."""
+    s = str(identifier or "").strip()
+    k = kind(s)
+    if k == "name":
+        return [("name", s)] if s else []
+    if k == "sample":
+        return ena_sample(s, fields)
+    if k in ("run", "project"):
+        samples = ena_samples_of(s)
+        if samples is None:
+            return None
+        out = []
+        for acc in samples:
+            got = ena_sample(acc, fields)
+            if got:
+                out += [(f"{acc}:{f}" if k == "project" else f, v) for f, v in got]
+        return out

verdy/sources.py

@@ -0,0 +1,403 @@
+"""Where the terms come from, and how a name is searched in each.
+
+Ontologies are chosen by ID. The ready ones are in ``ONTOLOGIES``; any other OLS4
+ontology ID works too (https://www.ebi.ac.uk/ols4/ontologies).
+
+* OLS4 ontologies (ENVO, NCBITaxon, PATO, OBA, UO, ...): OLS4's exact search on the name as
+  written, then its word searches on the name, its singular form and its key words (OLS4
+  only returns terms holding every word of a query).
+* ``gbif``: the GBIF Backbone's name matcher.
+* ``wikidata``: Wikidata's item search.
+* ``natura2000``: the Natura 2000 habitat types and species of the EEA's Article 17 report
+  (2019-2024), fetched from its public service once and searched locally; by name, or by
+  code ('1330 Atlantic salt meadows').
+* ``cf``: the CF Standard Names table (climate and forecast variables), read once and
+  searched locally.
+
+Every answer is cached on disk (``VERDY_CACHE``, default ``~/.cache/verdy``), so a
+second run on the same names makes no requests.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import threading
+import time
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import requests
+
+logger = logging.getLogger("verdy")
+
+CACHE = Path(os.environ.get("VERDY_CACHE", Path.home() / ".cache" / "verdy"))
+UA = {"User-Agent": "verdy/0.1 (ontology matcher)"}
+OLS = "https://www.ebi.ac.uk/ols4/api"
+CF_TABLE = "https://cfconventions.org/Data/cf-standard-names/current/src/cf-standard-name-table.xml"
+
+ONTOLOGIES = {
+    # environments, habitats, samples
+    "envo": "ENVO: environments, habitats, environmental materials",
+    "natura2000": "Natura 2000 habitat types and species (EEA Article 17 lists)",
+    "gold": "GOLD ecosystem classification",
+    # organisms
+    "ncbitaxon": "NCBI Taxonomy",
+    "gbif": "GBIF Backbone Taxonomy",
+    # parameters and measurements
+    "pato": "PATO: qualities (temperature, salinity, colour, ...)",
+    "oba": "OBA: biological attributes and traits",
+    "cf": "CF Standard Names: climate and forecast variables",
+    "uo": "UO: units of measurement",
+    "chebi": "ChEBI: chemical entities",
+    "obi": "OBI: investigations, assays, instruments",
+    # anatomy, plants, processes
+    "uberon": "Uberon: anatomy",
+    "po": "Plant Ontology",
+    "go": "Gene Ontology",
+    # general
+    "wikidata": "Wikidata",
+}
+NATURA = "natura2000"
+ART17 = "https://bio.discomap.eea.europa.eu/arcgis/rest/services/Article17/ART17_2019_2024_public/MapServer"
+SHORT = {NATURA: "Natura2000", "gbif": "GBIF", "wikidata": "Wikidata", "ncbitaxon": "NCBITaxon", "cf": "CF"}
+TAXONOMIES = {"ncbitaxon", "gbif"}      # a near name there is another taxon: exact matches only
+LOCAL = {"gbif", "wikidata", "cf", NATURA}
+
+_STOP = {"the", "and", "with", "of", "in", "on", "or", "by", "for", "to", "a", "an", "which", "are", "is", "sp", "spp",
+         "cf", "aff", "type", "types", "other", "including", "mainly", "where", "at", "per", "from"}
+
+
+# --- cache and requests ----------------------------------------------------------
+
+class Cache:
+    """One JSON file per source; a failed lookup (None) is not kept, so it is tried again."""
+    _locks: dict[str, threading.Lock] = {}
+
+    def __init__(self, name: str):
+        self.path = CACHE / f"{name}.json"
+        self.lock = Cache._locks.setdefault(str(self.path), threading.Lock())
+        try:
+            self.data = json.loads(self.path.read_text())
+        except (OSError, ValueError):
+            self.data = {}
+
+    def get(self, key, fetch):
+        if key in self.data:
+            return self.data[key]
+        value = fetch()
+        if value is None:
+            return None
+        with self.lock:
+            self.data[key] = value
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            tmp = self.path.with_name(self.path.name + f".{threading.get_ident()}.tmp")
+            tmp.write_text(json.dumps(self.data))
+            tmp.replace(self.path)
+        return value
+
+
+def _json(url, **params):
+    """GET JSON; {} for 404, None when the service did not answer (after two retries)."""
+    for attempt in range(3):
+        try:
+            r = requests.get(url, params=params, headers=UA, timeout=30)
+            if r.status_code == 404:
+                return {}
+            if r.status_code in (429, 500, 502, 503, 504) and attempt < 2:
+                wait = r.headers.get("Retry-After") or ""
+                time.sleep(min(10.0, float(wait)) if wait.isdigit() else 2 * (attempt + 1))
+                continue
+            r.raise_for_status()
+            return r.json()
+        except requests.Timeout:
+            if attempt < 2:
+                continue
+            logger.warning("No answer from %s", url)
+        except (requests.RequestException, ValueError) as exc:
+            logger.warning("Lookup failed for %s: %s", url, exc)
+            return None
+    return None
+
+
+# --- names and identifiers --------------------------------------------------------
+
+def canon(identifier) -> str:
+    """One spelling per term, so a CURIE and a URI of the same term compare equal."""
+    o = str(identifier or "").strip()
+    if m := re.search(r"/obo/([A-Za-z][A-Za-z0-9]*)_(\w+)$", o):
+        return f"{m.group(1)}:{m.group(2)}"
+    if m := re.search(r"gbif\.org/species/(\d+)", o):
+        return f"GBIF:{m.group(1)}"
+    if m := re.search(r"wikidata\.org/(?:entity|wiki)/(Q\d+)", o):
+        return f"wikidata:{m.group(1)}"
+    if m := re.search(r"w3id\.org/gold\.(path|vocab)/([\w.-]+)", o):
+        return f"{'GOLD.path' if m.group(1) == 'path' else 'GOLD'}:{m.group(2)}"
+    if m := re.search(r"vocab\.nerc\.ac\.uk/standard_name/([\w-]+)", o):
+        return f"CF:{m.group(1)}"
+    return o
+
+
+def query_text(label: str) -> str:
+    """The name as a search: without a leading code ('1330 Atlantic salt meadows'), a bracketed
+    note such as a unit ('air temperature [K]') or underscores ('air_temperature')."""
+    t = re.sub(r"^\d{3,}[0-9A-Z]*\s+\*?\s*(?=[A-Za-z])", "", str(label or "")).strip()
+    t = re.sub(r"\s*[\(\[][^)\]]*[\)\]]", "", t)
+    t = t.replace("_", " ")
+    return re.sub(r"\s+", " ", t).strip(" ,;")
+
+
+def singular(word: str) -> str:
+    w = word.lower()
+    if len(w) > 4 and w.endswith("ies"):
+        return w[:-3] + "y"
+    if len(w) > 4 and w.endswith(("sses", "ches", "shes", "xes")):
+        return w[:-2]
+    if len(w) > 3 and w.endswith("s") and not w.endswith(("ss", "us", "is")):
+        return w[:-1]
+    return w
+
+
+def words(text) -> set[str]:
+    return {singular(w) for w in re.findall(r"[a-z0-9]+", str(text or "").lower()) if w not in _STOP and len(w) > 1}
+
+
+def _norm(text) -> str:
+    return re.sub(r"[\s_]+", " ", str(text or "").lower()).strip(" .")
+
+
+def score(name: str, labels: list[str]) -> tuple[float, bool, bool]:
+    """How well the best of a term's labels fits the name (word overlap, 0..1); whether one
+    has the same words (plurals and 'sp.' aside); whether one is the name as written."""
+    nw = words(name)
+    # an abbreviation ('AIR' for a chemical) counts only when the name is written the same way
+    labels = [lab for lab in labels if lab and not (lab.isupper() and len(lab) <= 8 and lab != name.strip())]
+    # each label also without a bracketed part, and a path's last step ('Environmental > Air > Outdoor Air')
+    labels = list(dict.fromkeys(v for lab in labels if lab for v in (lab, query_text(lab), lab.split(">")[-1].strip()) if v))
+    best, exact, same = 0.0, False, any(_norm(lab) == _norm(name) for lab in labels)
+    for lab in labels:
+        lw = words(lab)
+        if not lw or not nw:
+            continue
+        if lw == nw:
+            best, exact = 1.0, True
+        best = max(best, len(nw & lw) / len(nw | lw))
+    return best, exact or same, same
+
+
+def _queries(name: str, taxon: bool) -> list[str]:
+    out = [name]
+    bare = re.sub(r"\b(spp?|cf|aff)\.?(\s|$)", " ", name).strip()
+    if bare and bare != name:
+        out.append(bare)
+    sing = " ".join(singular(w) for w in name.split())
+    if sing.lower() != name.lower():
+        out.append(sing)
+    if not taxon:
+        keys = [singular(w) for w in re.findall(r"[A-Za-z][A-Za-z-]+", name) if w.lower() not in _STOP and len(w) > 2]
+        if len(keys) > 2:
+            out += [f"{a} {b}" for a, b in zip(keys, keys[1:])][:4]
+        if len(keys) > 1:
+            out += sorted(set(keys), key=len, reverse=True)[:4]
+    seen, uniq = set(), []
+    for q in out:
+        if q.lower() not in seen:
+            seen.add(q.lower())
+            uniq.append(q)
+    return uniq
+
+
+# --- the sources ------------------------------------------------------------------
+
+def ontology_info(ontology: str) -> dict | None:
+    """Title and short name of a source; None when it is neither ready nor in OLS4."""
+    o = ontology.strip().lower()
+    if o in LOCAL:
+        return {"id": o, "title": ONTOLOGIES[o], "short": SHORT[o]}
+    if not re.fullmatch(r"[a-z0-9_.-]{1,40}", o):
+        return None
+
+    def fetch():
+        d = _json(f"{OLS}/ontologies/{o}")
+        return d if not d else {"title": (d.get("config") or {}).get("title"),
+                                "prefix": (d.get("config") or {}).get("preferredPrefix")}
+
+    d = Cache("ols_ontologies").get(o, fetch)
+    if not d:
+        return None
+    return {"id": o, "title": d.get("title") or ONTOLOGIES.get(o) or o, "short": SHORT.get(o) or d.get("prefix") or o.upper()}
+
+
+def _ols(name: str, ontology: str, cache: Cache) -> list[dict] | None:
+    taxon = ontology == "ncbitaxon"
+    docs, seen, failed = [], set(), False
+    for exact, q in [(True, name)] + [(False, q) for q in _queries(name, taxon)]:
+        res = cache.get(f"{ontology}|{'=' if exact else ''}{q.lower()}", lambda q=q, exact=exact: _json(
+            f"{OLS}/search", q=q, ontology=ontology, type="class", rows=10, exact=str(exact).lower(),
+            fieldList="iri,label,obo_id,synonym,description,is_defining_ontology,is_obsolete"))
+        if res is None:
+            failed = True
+            continue
+        for d in (res.get("response") or {}).get("docs") or []:
+            if d.get("iri") in seen or d.get("is_obsolete"):
+                continue
+            seen.add(d["iri"])
+            docs.append(d)
+        if taxon and docs:
+            break
+    if failed and not docs:
+        return None
+    if any(d.get("is_defining_ontology") for d in docs):          # its own terms, not imported ones;
+        docs = [d for d in docs if d.get("is_defining_ontology")]  # (some, like GOLD, mark none as its own)
+    hits = []
+    for d in docs:
+        s, exact, same = score(name, [d.get("label") or "", *(d.get("synonym") or [])])
+        if s > 0:
+            desc = d.get("description")
+            hits.append({"id": d["iri"], "curie": canon(d["iri"]) if "gold" in d["iri"] else d.get("obo_id") or canon(d["iri"]),
+                         "label": d.get("label"),
+                         "score": round(s, 3), "exact": exact, "same": same,
+                         "definition": (desc[0] if isinstance(desc, list) and desc else desc) or None})
+    return hits
+
+
+def _gbif(name: str, cache: Cache) -> list[dict] | None:
+    name = re.sub(r"\b(spp?|cf|aff)\.?(\s|$)", " ", name).strip()     # GBIF reads 'Elphidium sp.' as the genus
+    d = cache.get(name.lower(), lambda: _json("https://api.gbif.org/v1/species/match", name=name, verbose="true"))
+    if d is None:
+        return None
+    out = []
+    for c in [d, *(d.get("alternatives") or [])]:
+        if c.get("usageKey") and c.get("matchType") not in (None, "NONE", "HIGHERRANK"):
+            label = c.get("canonicalName") or c.get("scientificName")
+            s, exact, same = score(name, [label])
+            if not s and c.get("matchType") != "EXACT":
+                continue
+            out.append({"id": f"https://www.gbif.org/species/{c['usageKey']}", "curie": f"GBIF:{c['usageKey']}",
+                        "label": label, "score": round(s, 3), "exact": exact or c.get("matchType") == "EXACT",
+                        "same": same, "definition": " · ".join(x for x in (c.get("rank"), c.get("status")) if x) or None})
+    return out
+
+
+def _wikidata(name: str, cache: Cache) -> list[dict] | None:
+    d = cache.get(name.lower(), lambda: _json(
+        "https://www.wikidata.org/w/api.php", action="wbsearchentities", search=name, language="en", limit=7,
+        format="json", type="item"))
+    if d is None:
+        return None
+    out = []
+    for h in d.get("search") or []:
+        _, exact, same = score(name, [h.get("label") or "", *(h.get("aliases") or [])])   # an alias can make it exact,
+        s = 1.0 if exact else score(name, [h.get("label") or ""])[0]                     # only the label otherwise
+        if s > 0:
+            out.append({"id": f"http://www.wikidata.org/entity/{h['id']}", "curie": f"wikidata:{h['id']}",
+                        "label": h.get("label"), "score": round(s, 3), "exact": exact, "same": same,
+                        "definition": h.get("description")})
+    return out
+
+
+_cf_terms: list[dict] | None = None
+
+
+def cf_terms() -> list[dict]:
+    """The CF Standard Names table: name, its words as a label, units, description; aliases included."""
+    global _cf_terms
+    if _cf_terms is None:
+        path = CACHE / "cf-standard-name-table.xml"
+        if not path.exists():
+            r = requests.get(CF_TABLE, headers=UA, timeout=60)
+            r.raise_for_status()
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(r.content)
+        root = ET.fromstring(path.read_bytes())
+        terms = {}
+        for e in root.iter("entry"):
+            sid = e.get("id")
+            terms[sid] = {"id": sid, "labels": [sid.replace("_", " ")], "units": (e.findtext("canonical_units") or "").strip(),
+                          "description": re.sub(r"\s+", " ", e.findtext("description") or "").strip()}
+        for a in root.iter("alias"):
+            target = (a.findtext("entry_id") or "").strip()
+            if target in terms:
+                terms[target]["labels"].append(a.get("id").replace("_", " "))
+        _cf_terms = list(terms.values())
+    return _cf_terms
+
+
+def _cf(name: str) -> list[dict]:
+    out = []
+    for t in cf_terms():
+        s, exact, same = score(name, t["labels"])
+        if s >= 0.25:
+            out.append({"id": f"http://vocab.nerc.ac.uk/standard_name/{t['id']}/", "curie": f"CF:{t['id']}",
+                        "label": t["id"], "score": round(s, 3), "exact": exact, "same": same,
+                        "definition": (f"[{t['units']}] " if t["units"] else "") + t["description"][:200]})
+    return out
+
+
+def natura_terms() -> list[dict]:
+    """Natura 2000 habitat types and species, with codes, from the Article 17 layers (cached)."""
+    def fetch(layer, fields):
+        d = _json(f"{ART17}/{layer}/query", where="1=1", outFields=",".join(fields), returnDistinctValues="true",
+                  returnGeometry="false", f="json")
+        return None if d is None or "features" not in d else [f["attributes"] for f in d["features"]]
+
+    def build():
+        hab = fetch(5, ["reference_habitatlist.habitat_code", "reference_habitatlist.label"])
+        spe = fetch(6, ["reference_specieslist.species_code", "reference_specieslist.species_name",
+                        "reference_specieslist.taxonomical_group"])
+        if hab is None or spe is None:
+            return None
+        out = [{"code": a["reference_habitatlist.habitat_code"], "label": a["reference_habitatlist.label"],
+                "category": "habitat"} for a in hab if a.get("reference_habitatlist.habitat_code")]
+        out += [{"code": a["reference_specieslist.species_code"], "label": a["reference_specieslist.species_name"],
+                 "category": "species", "group": a.get("reference_specieslist.taxonomical_group")}
+                for a in spe if a.get("reference_specieslist.species_name")]
+        return out
+    terms = Cache("natura2000").get("art17_2019_2024", build)
+    if terms is None:
+        raise RuntimeError("The EEA Article 17 service did not answer; try again later.")
+    return terms
+
+
+_CODE = re.compile(r"^(?:n2k:|natura2000:)?(\d[0-9A-Z]{3})\b", re.I)
+
+
+def _natura(name: str, raw: str) -> list[dict]:
+    terms = natura_terms()
+    hit = lambda t, s, exact, same, note=None: {
+        "id": f"n2k:{t['code']}", "curie": f"n2k:{t['code']}", "label": f"{t['code']} {t['label']}" if t["category"] == "habitat" else t["label"],
+        "score": round(s, 3), "exact": exact, "same": same,
+        "definition": note or ("Annex I habitat type" if t["category"] == "habitat" else f"species · {t.get('group') or ''}")}
+    if (m := _CODE.match(raw.strip())) and (t := next((t for t in terms if t["code"] == m.group(1).upper()
+                                                       and t["category"] == "habitat"), None)):
+        return [hit(t, 1.0, True, True, "matched by its Annex I code")]
+    out = []
+    for t in terms:
+        s, exact, same = score(name, [t["label"]])
+        if s >= 0.25:
+            out.append(hit(t, s, exact, same))
+    return out
+
+
+def search(name: str, ontology: str, top: int = 5) -> list[dict] | None:
+    """The ontology's best terms for ``name``, best first (exact names first, then by word
+    overlap, then in the source's own order); None if the service did not answer."""
+    q = query_text(name)
+    if not q:
+        return []
+    if ontology == "gbif":
+        hits = _gbif(q, Cache("search_gbif"))
+    elif ontology == "wikidata":
+        hits = _wikidata(q, Cache("search_wikidata"))
+    elif ontology == "cf":
+        hits = _cf(q)
+    elif ontology == NATURA:
+        hits = _natura(q, name)
+    else:
+        hits = _ols(q, ontology, Cache(f"search_{ontology}"))
+    if hits is None:
+        return None
+    hits.sort(key=lambda h: (-h["same"], -h["exact"], -h["score"]))
+    return hits[:top]

verdy-0.1.0.dist-info/METADATA

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: verdy
+Version: 0.1.0
+Summary: VERDY: match your own terms (species, habitats, parameters, measurements) to the ontologies you choose: ENVO, NCBITaxon, GBIF, PATO, CF, Wikidata, any OLS4 ontology.
+Author: Jovana Dobreva
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/dobrevajovana/verdy
+Project-URL: Source, https://github.com/dobrevajovana/verdy
+Project-URL: Issues, https://github.com/dobrevajovana/verdy/issues
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31
+Provides-Extra: excel
+Requires-Dist: pandas>=2.0; extra == "excel"
+Requires-Dist: openpyxl>=3.1; extra == "excel"
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Dynamic: license-file
+
+<p align="center"><img src="https://raw.githubusercontent.com/dobrevajovana/verdy/main/logo.png" width="200" alt="VERDY logo: a green forest spirit with a flower crown, braided leafy hair and berry nodes linked by vines"></p>
+
+# VERDY
+
+[![Python package](https://github.com/dobrevajovana/verdy/actions/workflows/python-package.yml/badge.svg)](https://github.com/dobrevajovana/verdy/actions/workflows/python-package.yml)
+
+*Part of **VERDANT**: **V**ocabulary **E**nrichment & **R**esolution of **D**iversity **A**nnotations via **N**eural **T**echnology
+(BioHackathon Europe 2026).*
+
+**Meet VERDY, your warrior for ontology matching.** Give it your ENA IDs or free-text terms, choose the ontologies, and it predicts the matching terms in each.
+
+1. **Input:** a file (or Python list) of ENA IDs and/or free text. The IDs can be samples, runs or projects. The free text can be species, habitats, parameters or measurement names.
+2. **ENA lookup:** each ENA ID is looked up in [ENA](https://www.ebi.ac.uk/ena/portal/api) for its sample's organism and environment fields (`scientific_name`, `broad_scale_environmental_context`, `local_environmental_context`, `environmental_medium`, ...). Free text is used as it is.
+3. **Search:** every name is searched in each ontology you choose (for example `natura2000`, `gold`, `envo`).
+4. **Output:** a file with the exact match, or the most closely related terms, per name and ontology.
+
+```python
+import verdy
+
+matches = verdy.match("my_input.csv", ontologies=["natura2000", "gold", "envo", "ncbitaxon"])
+verdy.save(matches, "my_input.matches.csv")      # .csv  .tsv  .sssom.tsv  .json  .xlsx
+```
+
+```
+verdy my_input.csv -o natura2000,gold,envo --out my_input.matches.csv
+```
+
+## Install
+
+```
+pip install git+https://github.com/dobrevajovana/verdy.git
+pip install "verdy[excel] @ git+https://github.com/dobrevajovana/verdy.git"    # to read and write Excel
+```
+
+Python 3.10 or newer; the only dependency is `requests`.
+
+## Input
+
+`verdy.match` takes a CSV / TSV / Excel file, a text file with one value per line, a Python list or a pandas DataFrame.
+The column is found by its header (`name`, `term`, `parameter`, `species`, ..., or `id`, `accession`, `sample_accession`, `run_accession`)
+or given with `column="..."`. One file can mix both kinds of value:
+
+| value | what happens |
+|---|---|
+| `SAMD00132352`, `SAMEA...`, `SAMN...`, `ERS...` | an ENA sample: its fields are looked up in ENA and each value is searched |
+| `ERR9966617`, `SRR...`, `ERX...` | a run or experiment: its sample, as above |
+| `PRJEB54740`, `PRJNA...` | a project: all its samples, each row under the sample's own accession |
+| anything else | free text, searched as it is (units in brackets, underscores and a leading code are set aside) |
+
+`fields=` chooses other ENA sample fields to search (see ENA's `returnFields?result=sample`).
+After a run, `verdy.match.unresolved` lists the IDs ENA did not know.
+
+## The ontologies
+
+`verdy.ONTOLOGIES` (or `verdy --list`):
+
+| ID | What | For |
+|---|---|---|
+| `natura2000` | Natura 2000 habitat types and species (EEA Article 17 lists, fetched live) | habitats by name or code, protected species |
+| `envo` | Environment Ontology | habitats, environments, materials, environmental measurements |
+| `gold` | GOLD ecosystem classification | sample ecosystems |
+| `ncbitaxon` | NCBI Taxonomy | organisms |
+| `gbif` | GBIF Backbone Taxonomy | organisms |
+| `pato` | Phenotype And Trait Ontology | qualities: temperature, salinity, humidity, ... |
+| `oba` | Ontology of Biological Attributes | traits and measurable attributes |
+| `cf` | CF Standard Names | climate, weather and ocean variables |
+| `uo` | Units of measurement | units |
+| `chebi` | Chemical Entities of Biological Interest | chemicals |
+| `obi` | Ontology for Biomedical Investigations | assays, instruments, protocols |
+| `uberon`, `po`, `go` | anatomy, plants, biological processes | |
+| `wikidata` | Wikidata | anything with a name |
+
+**Any other ontology in [OLS4](https://www.ebi.ac.uk/ols4/ontologies) works as well:** pass its ID
+(for example `foodon`, `pco` or `ms`).
+
+## What comes out
+
+One row per match:
+
+| column | |
+|---|---|
+| `term_id`, `term`, `category` | your term |
+| `match_type` | `skos:exactMatch`: a label or synonym is the term (plurals and `sp.` aside) · `skos:closeMatch`: most words shared · `skos:relatedMatch`: some words shared |
+| `ontology`, `match_curie`, `match_id`, `match_label` | the matched term |
+| `confidence` | 1.0 exact as written, 0.95 exact up to plurals, then by word overlap |
+| `word_overlap`, `rank` | how many words they share (0..1), and the place among this term's matches |
+| `definition` | the term's definition (units first, for CF) |
+| `searched_for` | the term after cleaning |
+
+How the search works:
+* At most `per_ontology` matches (default 3) per term and ontology. When there is an exact match, only that one is given.
+* In the taxonomies (NCBITaxon, GBIF), only exact names count, because a near name is another taxon.
+* `verdy.unmatched(terms, matches, ontologies)` lists, per ontology, the terms it found nothing for.
+
+Every answer (ENA, OLS4, GBIF, Wikidata, the CF table, the Natura 2000 lists) is cached in `~/.cache/verdy` (or `VERDY_CACHE`), so a second run makes no requests.
+
+## Example
+
+`python examples/predict.py` runs [examples/input.csv](examples/input.csv), which has three ENA accessions and four free-text terms, against
+Natura 2000, GOLD, ENVO, NCBITaxon, PATO and CF. It writes [examples/input.matches.csv](examples/input.matches.csv). Part of the result:
+
+| input | name searched (from) | ontology | match |
+|---|---|---|---|
+| SAMD00132352 | urban biome (broad_scale_environmental_context) | ENVO | exactMatch ENVO:01000249 urban biome |
+| SAMD00132352 | air (local_environmental_context, environmental_medium) | ENVO · GOLD | exactMatch ENVO:00002005 air · GOLD:Air |
+| SAMD00132352 | air metagenome (scientific_name) | NCBITaxon | exactMatch NCBITaxon:655179 |
+| ERR9966617 | temperate forest | ENVO | exactMatch ENVO:01001805 temperate forest |
+| ERR9966617 | mountain | ENVO | exactMatch ENVO:00000081 mountain |
+| SAMEA110429224 | Human (host_scientific_name) | NCBITaxon | exactMatch NCBITaxon:9606 Homo sapiens |
+| Lynx lynx | | Natura 2000 · NCBITaxon | exactMatch n2k:1361 · NCBITaxon:13125 |
+| Atlantic salt meadows | | Natura 2000 | exactMatch n2k:1330 |
+| air_temperature | | CF · ENVO | exactMatch CF:air_temperature · ENVO:09200001 temperature of air |
+| salinity | | PATO | exactMatch PATO:0085001 |
+
+## One term at a time
+
+```python
+verdy.search("cloud liquid water content", "cf")    # the source's best terms, best first
+verdy.resolve("SAMD00132352")                      # what ENA says about an ID
+verdy.read_terms("my_input.csv")                    # what VERDY read from your file
+verdy.to_dataframe(matches)                         # pandas
+```
+
+## Tests
+
+```
+pip install -e ".[test]" && python -m pytest -q
+```
+
+The tests stub every web service, so they run offline.
+
+## Licence
+
+MIT, see [LICENSE](LICENSE).

verdy-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+verdy/__init__.py,sha256=r1pHqjoFqODwZQ_DbRQWJQ-UIC7MVqY3llqz9VVCK0g,1845
+verdy/__main__.py,sha256=l6FmcAIP0h3bmuxSlTmaKCe-hIE17s_cW72e-tprQQ4,2829
+verdy/match.py,sha256=8vl5fn6qBnF1CgKjk8LQPCw23idj20AxwnxCqEK9tcc,10199
+verdy/output.py,sha256=tu2GYfzHDSk4Vwzj1WZ6Q7jqlIK3IlqX7ZkgQYkrM-Y,4612
+verdy/resolve.py,sha256=vZUStrmmlHXH5TSvOlet5eBKCEY7uKMfq4Hnszpe51E,3733
+verdy/sources.py,sha256=sNG5WiF_fRQeZEV_jjW8YC3CTjH86_fB_2vpGfZV1DQ,17818
+verdy-0.1.0.dist-info/licenses/LICENSE,sha256=OuM4HpQk9KXk2ynEb-poNG1CsbY-bLTJEjXVv0JwCsk,1071
+verdy-0.1.0.dist-info/METADATA,sha256=qJup4i4DOzi95RhUGt1q0i_z70ntgRTNY_U6sId7wXM,7820
+verdy-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+verdy-0.1.0.dist-info/entry_points.txt,sha256=rXCx9wGEBZ1zxF9f4J0DCnvPgLLhzKA9lXe1H_aUf1c,46
+verdy-0.1.0.dist-info/top_level.txt,sha256=7oY2YvwyxHSA8l8KA1ULphZXZnYQ6wi3ls2z6zm3w7Q,6
+verdy-0.1.0.dist-info/RECORD,,

verdy-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

verdy-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+verdy = verdy.__main__:main

verdy-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jovana Dobreva
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

verdy-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+verdy