ref-verify 1.1.2__py3-none-any.whl

ref_verify/numeric_claim.py

@@ -0,0 +1,425 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+_NUMBER_PATTERN = r"\d+(?:,\d{3})*(?:\.\d+)?"
+_UNIT_PATTERN = (
+    r"(?:mg/ml|mg\/ml|g\/l|per\s+cent|percent|cycles?|patients?|subjects?|"
+    r"samples?|devices?|°c|degc|khz|mhz|mv|kv|ma|mg|ml|kg|mm|cm|nm|hz|"
+    r"%|v|a|c|g|l|m)"
+)
+_MEASUREMENT_PATTERN = re.compile(
+    rf"(?<![\d,])(?P<value>{_NUMBER_PATTERN})\s*(?P<unit>{_UNIT_PATTERN})(?=$|\W)",
+    re.IGNORECASE,
+)
+
+_STOPWORDS = {
+    "a",
+    "an",
+    "and",
+    "as",
+    "at",
+    "by",
+    "for",
+    "in",
+    "of",
+    "on",
+    "the",
+    "to",
+    "was",
+    "were",
+    "with",
+}
+
+_NON_SUBJECT_TERMS = {
+    "above",
+    "at",
+    "below",
+    "equal",
+    "exceed",
+    "exceeded",
+    "exceeding",
+    "exceeds",
+    "greater",
+    "higher",
+    "least",
+    "less",
+    "lower",
+    "more",
+    "most",
+    "no",
+    "not",
+    "than",
+    "under",
+    "up",
+}
+
+_UNSUPPORTED_FRAME_PATTERNS = (
+    r"\baccording to\b",
+    r"\bwhether\b",
+    r"\bnot true that\b",
+    r"\bfalse that\b",
+    r"\b(?:did|do|does|is|are|was|were|has|have|had) not\b",
+    r"\b(?:fail|fails|failed) to\b",
+    r"\bwithout\b",
+    r"\bnever\b",
+    r"\bno\b.*\b(?:observed|found|shown|showed|reported|measured|demonstrated|had)\b",
+    r"\bno (?:sample|samples|specimen|specimens|device|devices|case|cases)\b",
+    r"\bnone of (?:the )?(?:sample|samples|specimen|specimens|device|devices|case|cases)\b",
+    r"\bnone (?:show|shows|showed|had|has|have|observed|found|reported|reached|met|demonstrated)\b",
+    r"\b(?:previous|prior|earlier) (?:work|study|studies|research)\b",
+    r"\b(?:may|might|would|should)\b",
+    r"\b(?:appear|appears|appeared|appearing) to\b",
+    r"\b(?:seem|seems|seemed|seeming) to\b",
+    r"\b(?:the )?(?:paper|article|study|work) (?:report|reports|reported|reporting)\b",
+    r"\b(?:the )?authors? (?:report|reports|reported|found|finds|observed|observes|noted|notes|claim|claims)\b",
+    r"\breportedly\b",
+    r"\b(?:claim|claims|claimed|claiming)\b",
+    r"\b(?:suggest|suggests|suggested|suggesting)\b",
+    r"\b(?:indicate|indicates|indicated|indicating)\b",
+    r"\b(?:imply|implies|implied|implying)\b",
+)
+
+_SCOPE_OR_COMPARATIVE_SUFFIX_TERMS = {
+    "across",
+    "after",
+    "among",
+    "at",
+    "average",
+    "averaged",
+    "averages",
+    "before",
+    "during",
+    "except",
+    "following",
+    "for",
+    "from",
+    "in",
+    "longer",
+    "max",
+    "maximum",
+    "mean",
+    "median",
+    "min",
+    "minimum",
+    "more",
+    "only",
+    "then",
+    "typical",
+    "typically",
+    "under",
+    "unless",
+    "until",
+    "when",
+    "while",
+    "within",
+}
+
+_UNIT_TERMS = {
+    "a",
+    "c",
+    "cycle",
+    "cycles",
+    "degc",
+    "device",
+    "devices",
+    "g",
+    "hz",
+    "kg",
+    "khz",
+    "kv",
+    "l",
+    "m",
+    "ma",
+    "mg",
+    "mgml",
+    "mhz",
+    "ml",
+    "mm",
+    "mv",
+    "nm",
+    "patient",
+    "patients",
+    "per",
+    "percent",
+    "sample",
+    "samples",
+    "subject",
+    "subjects",
+    "v",
+}
+
+
+@dataclass(frozen=True)
+class NumericClaimResult:
+    status: str
+    reason: str
+    evidence: str
+
+
+@dataclass(frozen=True)
+class NumericExpression:
+    value: float
+    unit: str
+    comparator: str
+    subject_terms: set[str]
+    evidence: str
+
+
+def check_numeric_claim_support(abstract: str, claim: str) -> NumericClaimResult:
+    claim_expression = _extract_claim_expression(claim)
+    if claim_expression is None:
+        return NumericClaimResult(
+            status="NOT_NUMERIC",
+            reason="The claim does not contain a supported numeric expression.",
+            evidence="",
+        )
+
+    related_evidence = ""
+    for clause in _clauses(abstract):
+        if _has_unsupported_frame(clause):
+            related_evidence = related_evidence or clause
+            continue
+        evidence_expressions = _extract_evidence_expressions(clause)
+        if not evidence_expressions:
+            continue
+        if _subject_terms_match(claim_expression.subject_terms, clause):
+            related_evidence = related_evidence or clause
+        for evidence_expression in evidence_expressions:
+            if not _units_match(claim_expression.unit, evidence_expression.unit):
+                continue
+            if not _subject_terms_match(claim_expression.subject_terms, clause):
+                continue
+            if _evidence_entails_claim(
+                evidence_expression.value,
+                evidence_expression.comparator,
+                claim_expression.value,
+                claim_expression.comparator,
+            ):
+                return NumericClaimResult(
+                    status="SUPPORTED",
+                    reason="The abstract explicitly reports a matching numeric claim.",
+                    evidence=clause,
+                )
+            related_evidence = related_evidence or clause
+
+    return NumericClaimResult(
+        status="PARTIAL",
+        reason="The abstract contains numeric evidence, but not a clear subject-bound match.",
+        evidence=related_evidence or _best_numeric_clause(abstract),
+    )
+
+
+def _extract_claim_expression(claim: str) -> NumericExpression | None:
+    match = _MEASUREMENT_PATTERN.search(claim)
+    if not match:
+        return None
+    return NumericExpression(
+        value=_parse_value(match.group("value")),
+        unit=_normalize_unit(match.group("unit")),
+        comparator=_claim_comparator(claim[: match.start()]),
+        subject_terms=_subject_terms(claim[: match.start()]),
+        evidence=claim,
+    )
+
+
+def _extract_evidence_expressions(clause: str) -> list[NumericExpression]:
+    expressions: list[NumericExpression] = []
+    for match in _MEASUREMENT_PATTERN.finditer(clause):
+        expressions.append(
+            NumericExpression(
+                value=_parse_value(match.group("value")),
+                unit=_normalize_unit(match.group("unit")),
+                comparator=_evidence_comparator(
+                    clause[: match.start()],
+                    clause[match.end() :],
+                ),
+                subject_terms=_subject_terms(clause[: match.start()]),
+                evidence=clause,
+            )
+        )
+    return expressions
+
+
+def _clauses(value: str) -> list[str]:
+    clauses: list[str] = []
+    for sentence in re.split(r"(?<=[.!?])\s+|[.;:]\s+", value.strip()):
+        sentence = sentence.strip()
+        if not sentence:
+            continue
+        parts = [
+            part.strip()
+            for part in re.split(r",?\s+\b(?:and|but|while|whereas)\b\s+", sentence)
+            if part.strip()
+        ]
+        for part in parts:
+            clauses.extend(_split_numeric_comma_clauses(part))
+    return clauses
+
+
+def _split_numeric_comma_clauses(value: str) -> list[str]:
+    parts = [part.strip() for part in value.split(",") if part.strip()]
+    if len(parts) <= 1:
+        return [value.strip()]
+    if parts[0].lower().split()[:1] in (["after"], ["before"], ["under"], ["in"]):
+        return [value.strip()]
+    if sum(1 for part in parts if _MEASUREMENT_PATTERN.search(part)) >= 2:
+        return parts
+    return [value.strip()]
+
+
+def _best_numeric_clause(value: str) -> str:
+    for clause in _clauses(value):
+        if _MEASUREMENT_PATTERN.search(clause):
+            return clause
+    return value.strip()
+
+
+def _subject_terms_match(subject_terms: set[str], clause: str) -> bool:
+    if not subject_terms:
+        return False
+    clause_terms = {_stem(token) for token in _tokens(clause)}
+    return subject_terms <= clause_terms
+
+
+def _subject_terms(value: str) -> set[str]:
+    return {
+        _stem(token)
+        for token in _tokens(value)
+        if _stem(token) not in _NON_SUBJECT_TERMS and _stem(token) not in _UNIT_TERMS
+    }
+
+
+def _tokens(value: str) -> list[str]:
+    return [
+        token
+        for token in re.findall(r"[a-zA-Z°]+", value.lower().replace("/", ""))
+        if token not in _STOPWORDS
+    ]
+
+
+def _stem(token: str) -> str:
+    if token.endswith("s") and len(token) > 3:
+        return token[:-1]
+    return token
+
+
+def _parse_value(value: str) -> float:
+    return float(value.replace(",", ""))
+
+
+def _normalize_unit(value: str) -> str:
+    normalized = value.lower().replace(" ", "").replace("/", "")
+    if normalized in {"%", "percent"}:
+        return "%"
+    if normalized in {"°c", "degc", "c"}:
+        return "c"
+    if normalized in {"cycle", "cycles"}:
+        return "cycle"
+    if normalized in {"patient", "patients", "subject", "subjects"}:
+        return "person"
+    if normalized in {"sample", "samples", "device", "devices"}:
+        return normalized.rstrip("s")
+    return normalized.rstrip("s")
+
+
+def _units_match(left: str, right: str) -> bool:
+    return left == right
+
+
+def _claim_comparator(prefix: str) -> str:
+    normalized = prefix.lower()
+    if ">=" in normalized or "≥" in normalized:
+        return "gte"
+    if "<=" in normalized or "≤" in normalized:
+        return "lte"
+    if ">" in normalized:
+        return "gt"
+    if "<" in normalized:
+        return "lt"
+    if re.search(r"\b(?:more|greater|higher) than or equal to\b", normalized):
+        return "gte"
+    if re.search(r"\b(?:less|lower|fewer) than or equal to\b", normalized):
+        return "lte"
+    if re.search(r"\b(?:at least|not less than)\b", normalized):
+        return "gte"
+    if re.search(r"\b(?:at most|no more than)\b", normalized):
+        return "lte"
+    if re.search(r"\b(?:below|under|less than)\b", normalized):
+        return "lt"
+    if re.search(r"\b(?:above|over|greater than|more than|exceeded|exceeds)\b", normalized):
+        return "gt"
+    return "exact"
+
+
+def _evidence_comparator(prefix: str, suffix: str = "") -> str:
+    normalized = prefix.lower().rstrip()
+    normalized_suffix = suffix.lower().lstrip()
+    if normalized.endswith((">=", "≥")):
+        return "gte"
+    if normalized.endswith(("<=", "≤")):
+        return "lte"
+    if normalized.endswith(">"):
+        return "gt"
+    if normalized.endswith("<"):
+        return "lt"
+    if re.search(r"\b(?:at least|not less than)\s*$", normalized):
+        return "gte"
+    if re.search(r"\b(?:at most|no more than)\s*$", normalized):
+        return "lte"
+    if re.search(r"\b(?:below|under|less than)\s*$", normalized):
+        return "lt"
+    if re.search(r"\b(?:above|over|greater than|more than|exceeded|exceeds|reached|survived|maintained)\s*$", normalized):
+        return "exact"
+    if re.search(r"\bup to\s*$", normalized):
+        return "up_to"
+    if re.search(r"^\s*(?:or\s+)?(?:more|greater|higher|min|minimum)\b", normalized_suffix):
+        return "gte"
+    if re.search(r"^\s*(?:or\s+)?(?:less|fewer|lower|max|maximum)\b", normalized_suffix):
+        return "lte"
+    return "exact"
+
+
+def _has_unsupported_frame(value: str) -> bool:
+    normalized = " ".join(re.findall(r"[a-zA-Z0-9]+", value.lower()))
+    if any(re.search(pattern, normalized) for pattern in _UNSUPPORTED_FRAME_PATTERNS):
+        return True
+    for match in _MEASUREMENT_PATTERN.finditer(value):
+        prefix_tokens = [
+            token
+            for token in re.findall(r"[a-zA-Z]+", value[: match.start()].lower())
+        ]
+        if any(
+            token in {"average", "averaged", "mean", "median", "typical", "typically"}
+            for token in prefix_tokens
+        ):
+            return True
+        suffix_tokens = [
+            token
+            for token in re.findall(r"[a-zA-Z]+", value[match.end() :].lower())
+        ]
+        if any(token in _SCOPE_OR_COMPARATIVE_SUFFIX_TERMS for token in suffix_tokens):
+            return True
+    prefix_tokens = re.findall(r"[a-zA-Z]+", value.lower())[:2]
+    return any(token in {"after", "before", "under", "in"} for token in prefix_tokens)
+
+
+def _evidence_entails_claim(
+    evidence_value: float,
+    evidence_comparator: str,
+    claim_value: float,
+    claim_comparator: str,
+) -> bool:
+    if evidence_comparator == "up_to":
+        return claim_comparator in {"lt", "lte"} and evidence_value <= claim_value
+    if claim_comparator == "gt":
+        return evidence_value > claim_value
+    if claim_comparator == "gte":
+        return evidence_value >= claim_value
+    if claim_comparator == "lt":
+        return evidence_value < claim_value
+    if claim_comparator == "lte":
+        return evidence_value <= claim_value
+    return evidence_value == claim_value

ref_verify/pubmed.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import json
+import re
+import xml.etree.ElementTree as ET
+from typing import Any
+from urllib.error import HTTPError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from ref_verify import __version__
+from ref_verify.abstract_lookup import AbstractSourceError
+from ref_verify.doi_check import normalize_doi
+from ref_verify.models import PaperRecord
+
+
+class PubMedClient:
+    source_name = "pubmed"
+
+    def __init__(self, timeout: float = 20.0) -> None:
+        self.timeout = timeout
+
+    def fetch_record(self, doi: str) -> PaperRecord | None:
+        normalized = normalize_doi(doi)
+        pmids = self._search_pmids(normalized)
+        if not pmids:
+            raise AbstractSourceError("NOT_FOUND", "PubMed had no record for the DOI.")
+        if len(pmids) != 1:
+            raise AbstractSourceError("UNSUPPORTED", "PubMed returned multiple records for the DOI.")
+        request = Request(
+            "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?"
+            + urlencode({"db": "pubmed", "id": pmids[0], "retmode": "xml"}),
+            headers={
+                "User-Agent": (
+                    f"ref-verify/{__version__} "
+                    "(+https://github.com/Moonweave-Research/ref-verify)"
+                )
+            },
+        )
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                xml_payload = response.read().decode("utf-8")
+        except HTTPError as exc:
+            if exc.code == 404:
+                raise AbstractSourceError("NOT_FOUND", "PubMed had no record for the DOI.") from exc
+            raise
+        return parse_pubmed_article(xml_payload)
+
+    def _search_pmids(self, doi: str) -> list[str]:
+        request = Request(
+            "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?"
+            + urlencode(
+                {
+                    "db": "pubmed",
+                    "term": f"{doi}[AID]",
+                    "retmode": "json",
+                    "retmax": "2",
+                }
+            ),
+            headers={
+                "User-Agent": (
+                    f"ref-verify/{__version__} "
+                    "(+https://github.com/Moonweave-Research/ref-verify)"
+                )
+            },
+        )
+        with urlopen(request, timeout=self.timeout) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+        ids = payload.get("esearchresult", {}).get("idlist", [])
+        return [str(value) for value in ids if str(value).strip()]
+
+
+def parse_pubmed_article(xml_payload: str) -> PaperRecord | None:
+    root = ET.fromstring(xml_payload)
+    article = root.find(".//PubmedArticle")
+    if article is None:
+        return None
+
+    doi = _article_doi(article)
+    abstract = _abstract_text(article)
+    if not doi or not abstract:
+        return None
+
+    return PaperRecord(
+        doi=doi,
+        title=_text(article.find(".//ArticleTitle")) or "[title missing]",
+        authors=_authors(article),
+        year=_publication_year(article),
+        abstract=abstract,
+        source="PubMed",
+        journal=_text(article.find(".//Journal/Title")),
+        url=f"https://pubmed.ncbi.nlm.nih.gov/{_pmid(article)}/" if _pmid(article) else None,
+    )
+
+
+def _article_doi(article: ET.Element) -> str | None:
+    for element in article.findall(".//ArticleId"):
+        if element.attrib.get("IdType", "").lower() == "doi":
+            return _text(element)
+    for element in article.findall(".//ELocationID"):
+        if element.attrib.get("EIdType", "").lower() == "doi":
+            return _text(element)
+    return None
+
+
+def _abstract_text(article: ET.Element) -> str | None:
+    parts = []
+    for element in article.findall(".//Abstract/AbstractText"):
+        label = element.attrib.get("Label")
+        text = _flatten_text(element)
+        if not text:
+            continue
+        parts.append(f"{label}: {text}" if label else text)
+    if not parts:
+        return None
+    return re.sub(r"\s+", " ", " ".join(parts)).strip()
+
+
+def _authors(article: ET.Element) -> list[str]:
+    names: list[str] = []
+    for author in article.findall(".//AuthorList/Author"):
+        collective = _text(author.find("CollectiveName"))
+        if collective:
+            names.append(collective)
+            continue
+        last_name = _text(author.find("LastName"))
+        if last_name:
+            names.append(last_name)
+    return names
+
+
+def _publication_year(article: ET.Element) -> int | None:
+    for path in (".//ArticleDate/Year", ".//JournalIssue/PubDate/Year"):
+        value = _text(article.find(path))
+        if value and value.isdigit():
+            return int(value)
+    medline_date = _text(article.find(".//JournalIssue/PubDate/MedlineDate"))
+    if medline_date:
+        match = re.search(r"\b(19|20)\d{2}\b", medline_date)
+        if match:
+            return int(match.group(0))
+    return None
+
+
+def _pmid(article: ET.Element) -> str | None:
+    return _text(article.find(".//PMID"))
+
+
+def _text(element: ET.Element | None) -> str | None:
+    if element is None or element.text is None or not element.text.strip():
+        return None
+    return element.text.strip()
+
+
+def _flatten_text(element: ET.Element) -> str:
+    return "".join(element.itertext()).strip()

ref_verify/semantic_scholar.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib.error import HTTPError
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+from ref_verify import __version__
+from ref_verify.abstract_lookup import AbstractSourceError
+from ref_verify.doi_check import normalize_doi
+from ref_verify.models import PaperRecord
+
+
+class SemanticScholarClient:
+    source_name = "semantic_scholar"
+
+    def __init__(self, timeout: float = 20.0) -> None:
+        self.timeout = timeout
+
+    def fetch_record(self, doi: str) -> PaperRecord | None:
+        paper_id = quote(f"DOI:{normalize_doi(doi)}", safe=":")
+        fields = "title,authors,year,abstract,externalIds,url,venue"
+        request = Request(
+            f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}?fields={fields}",
+            headers={
+                "User-Agent": (
+                    f"ref-verify/{__version__} "
+                    "(+https://github.com/Moonweave-Research/ref-verify)"
+                )
+            },
+        )
+        try:
+            with urlopen(request, timeout=self.timeout) as response:
+                payload = json.loads(response.read().decode("utf-8"))
+        except HTTPError as exc:
+            if exc.code == 404:
+                raise AbstractSourceError("NOT_FOUND", "Semantic Scholar had no paper for the DOI.") from exc
+            raise
+        return parse_semantic_scholar_paper(payload)
+
+
+def parse_semantic_scholar_paper(payload: dict[str, Any]) -> PaperRecord | None:
+    abstract = _string_or_none(payload.get("abstract"))
+    if abstract is None:
+        return None
+    external_ids = payload.get("externalIds")
+    doi = ""
+    if isinstance(external_ids, dict):
+        doi = _string_or_none(external_ids.get("DOI")) or ""
+    if not doi:
+        return None
+
+    return PaperRecord(
+        doi=doi,
+        title=_string_or_none(payload.get("title")) or "[title missing]",
+        authors=[
+            name
+            for author in payload.get("authors", [])
+            if isinstance(author, dict)
+            if (name := _string_or_none(author.get("name")))
+        ],
+        year=_int_or_none(payload.get("year")),
+        abstract=abstract,
+        source="Semantic Scholar",
+        journal=_string_or_none(payload.get("venue")),
+        url=_string_or_none(payload.get("url")),
+    )
+
+
+def _string_or_none(value: Any) -> str | None:
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None
+
+
+def _int_or_none(value: Any) -> int | None:
+    if isinstance(value, int):
+        return value
+    return None