linkml-reference-validator 0.3.0rc1__tar.gz → 0.3.0rc3__tar.gz

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: linkml_reference_validator
-Version: 0.3.0rc1
+Version: 0.3.0rc3
 Summary: Validation of supporting text from references and publications
 Author-email: Chris Mungall <cjmungall@lbl.gov>
 License-Expression: Apache-2.0

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc3}/src/linkml_reference_validator/etl/fulltext/pmc.py

@@ -6,7 +6,7 @@ fetched from the PMC XML API (with an HTML fallback) and extracted via XMLExtrac
 
 import logging
 import time
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from Bio import Entrez  # type: ignore
 from bs4 import BeautifulSoup  # type: ignore
@@ -38,6 +38,51 @@ class TransientFullTextError(RuntimeError):
     """
 
 
+#: Classes PMC has used for the article body. ``main-article-body`` is what it
+#: serves now, on a ``<section>``; the two ``div`` classes are the older markup
+#: and are kept so entries cached against them still re-extract.
+#:
+#: Deliberately not ``body``, which PMC pairs with ``main-article-body`` on the
+#: same element. Matching it alone would match every page's ``<body>``, and the
+#: structural test is the only thing standing between this fetch and caching a
+#: bot-check interstitial served on an HTTP 200.
+#:
+#: Order is a priority order, not an alphabetical one: the first match wins, so
+#: the current wrapper is tried before the legacy ones. It matters for a legacy
+#: page carrying several ``tsec`` sections, where only the first is returned --
+#: the previous code behaved the same way, so this is a known limit rather than
+#: a regression, but reordering the tuple would change which section that is.
+PMC_ARTICLE_BODY_CLASSES = ("main-article-body", "article-body", "tsec")
+
+
+def find_pmc_article_body(soup: BeautifulSoup) -> Optional[Any]:
+    """Return PMC's article-body element, or None if the page has none.
+
+    Matches on class rather than element name: the container moved from ``div``
+    to ``section`` when PMC changed its markup, so pinning the tag is what broke
+    the previous version for every article.
+
+    Args:
+        soup: A parsed PMC article page
+
+    Returns:
+        The article-body element, or None
+
+    Examples:
+        >>> from bs4 import BeautifulSoup
+        >>> html = '<section class="body main-article-body"><p>Text.</p></section>'
+        >>> find_pmc_article_body(BeautifulSoup(html, "html.parser")) is not None
+        True
+        >>> find_pmc_article_body(BeautifulSoup("<body><p>x</p></body>", "html.parser")) is None
+        True
+    """
+    for class_name in PMC_ARTICLE_BODY_CLASSES:
+        found = soup.find(class_=class_name)
+        if found is not None:
+            return found
+    return None
+
+
 @FullTextProviderRegistry.register
 class PMCFullTextProvider(FullTextProvider):
     """Provide PMC full text for a reference identified by PMID/PMCID.
@@ -132,7 +177,8 @@ class PMCFullTextProvider(FullTextProvider):
 
         This is the one page fetch the OA providers' "a landing page is not full
         text" rule does not cover, and what makes it safe is the requirement
-        below: text is returned only from a ``div.article-body`` or ``div.tsec``.
+        below: text is returned only from an element carrying one of
+        :data:`PMC_ARTICLE_BODY_CLASSES`.
         A bot-check interstitial -- which PMC serves on an HTTP 200, so no status
         check sees it -- carries neither, so this yields ``None`` rather than
         caching the page. The ``oa_url`` fallback that was removed had no such
@@ -152,7 +198,7 @@ class PMCFullTextProvider(FullTextProvider):
             )
 
         soup = BeautifulSoup(response.content, "html.parser")
-        article_body = soup.find("div", class_="article-body") or soup.find("div", class_="tsec")
+        article_body = find_pmc_article_body(soup)
         if article_body:
             # The region is selected here, but the text comes out of the shared
             # extractor rather than a private copy of the paragraph walk, so

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc3}/src/linkml_reference_validator/etl/reference_fetcher.py

@@ -343,7 +343,18 @@ class ReferenceFetcher:
 
         # Check disk cache
         if not force_refresh:
-            cached = self._load_from_disk(normalized_reference_id)
+            cached = self._load_from_disk(
+                normalized_reference_id,
+                allow_stale=self.config.trust_cached_entries,
+                allow_stale_html=self.config.trust_cached_entries,
+            )
+            if cached and self.config.trust_cached_entries:
+                # Trusted: serve what is on disk. No staleness re-fetch and no
+                # full-text retry, so nothing can shorten the body or replace it
+                # with a bot-check page.
+                return self._remember(
+                    normalized_reference_id, FetchOutcome(content=cached)
+                )
             if cached:
                 # A record cached as abstract_only may predate full-text support, or
                 # reflect a prior transient failure. Give the chain one more chance
@@ -424,7 +435,11 @@ class ReferenceFetcher:
         if force_refresh:
             return FetchOutcome(content=None)
 
-        stale = self._load_from_disk(normalized_reference_id, allow_stale=True)
+        stale = self._load_from_disk(
+            normalized_reference_id,
+            allow_stale=True,
+            allow_stale_html=self.config.serve_stale_html_full_text,
+        )
         if stale is None:
             return FetchOutcome(content=None)
 
@@ -517,10 +532,17 @@ class ReferenceFetcher:
             cached.content_type,
         )
 
-        # May its text be served? Asked without the bypass, so stale HTML is
-        # withheld here exactly as _stale_fallback withholds it, and validation
-        # falls back to the freshly fetched abstract.
-        servable = self._load_from_disk(normalized_reference_id, allow_stale=True)
+        # May its text be served? Asked with the consumer's serve policy, the
+        # same one _stale_fallback uses. By default stale HTML is withheld and
+        # validation falls back to the freshly fetched abstract; a consumer that
+        # has opted in gets the cached body instead. This is the path a refresh
+        # normally takes -- the source still answers, with an abstract -- so it
+        # is the one most consumers see, not _stale_fallback.
+        servable = self._load_from_disk(
+            normalized_reference_id,
+            allow_stale=True,
+            allow_stale_html=self.config.serve_stale_html_full_text,
+        )
         return FetchOutcome(content=servable or fresh, served_stale=True)
 
     def _report_shrinking_refresh(
@@ -1528,8 +1550,11 @@ class ReferenceFetcher:
         ):
             logger.warning(
                 "Refusing stale HTML full text for %s: it may be a repository "
-                "landing page. Retry when the source serves full text again to "
-                "repair it.",
+                "landing page. A re-fetch repairs it where the source still "
+                "serves full text -- but not for a PMC-only article, which "
+                "answers a Python HTTP client with a bot-check interstitial on "
+                "an HTTP 200. Set serve_stale_html_full_text if this cache is "
+                "one you review.",
                 reference_id,
             )
             return None

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc3}/src/linkml_reference_validator/etl/sources/pmid.py

@@ -23,6 +23,7 @@ from Bio import Entrez  # type: ignore
 from bs4 import BeautifulSoup  # type: ignore
 import requests  # type: ignore
 
+from linkml_reference_validator.etl.fulltext.pmc import find_pmc_article_body
 from linkml_reference_validator.models import ReferenceContent, ReferenceValidationConfig
 from linkml_reference_validator.etl.extract.html import HTMLExtractor
 from linkml_reference_validator.etl.extract import MIN_FULLTEXT_CHARS
@@ -582,9 +583,7 @@ class PMIDSource(ReferenceSource):
             return None
 
         soup = BeautifulSoup(response.content, "html.parser")
-        article_body = soup.find("div", class_="article-body") or soup.find(
-            "div", class_="tsec"
-        )
+        article_body = find_pmc_article_body(soup)
 
         if article_body:
             # Region selected here, text extracted by the shared extractor, for

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc3}/src/linkml_reference_validator/models.py

@@ -510,6 +510,32 @@ class ReferenceValidationConfig(BaseModel):
             "when a metadata source does not already return full text."
         ),
     )
+    serve_stale_html_full_text: bool = Field(
+        default=False,
+        description=(
+            "If True, serve a stale full_text_html cache entry as evidence instead "
+            "of withholding it. Off by default: a pre-stamp HTML entry may be a "
+            "publisher landing page rather than the article, and the default "
+            "assumes a re-fetch can repair it. Set this only for a cache you "
+            "control and review -- for a PMC-only article the repair route is "
+            "unreachable (PMC answers a Python HTTP client with a bot-check "
+            "interstitial on an HTTP 200), so the guard withholds text that "
+            "cannot be recovered. Affects serving only, never overwriting, and "
+            "the entry is left un-stamped so a later run still retries."
+        ),
+    )
+    trust_cached_entries: bool = Field(
+        default=False,
+        description=(
+            "If True, use a cache entry as it is on disk and never re-fetch it "
+            "because an extractor stamp is missing or old. Off by default, so "
+            "caches still migrate. Set it for a committed, reviewed cache, where "
+            "a re-fetch is not an improvement but a risk: it can shorten a body, "
+            "cache a bot-check page, or replace text a curator quoted. Imperfect "
+            "cached text is a content problem to fix deliberately, not on every "
+            "validation run."
+        ),
+    )
     full_text_providers: list[str] = Field(
         default_factory=lambda: ["pmc", "epmc_preprint", "unpaywall", "openalex"],
         description=(

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc3}/tests/test_no_cache_downgrade.py

@@ -76,7 +76,21 @@ def _write_cache(cache_dir, reference_id, content_type, body, stamped=False):
 
 @pytest.fixture
 def fetcher(tmp_path):
-    return ReferenceFetcher(ReferenceValidationConfig(cache_dir=tmp_path, email="me@example.org"))
+    """A fetcher with the full-text chain off.
+
+    These tests are about what a *refresh that loses full text* does to the
+    cached file, so the chain has to be absent rather than merely expected to
+    fail. With it on they reach the network: the PMC HTML fallback fires and the
+    outcome depends on what PMC serves for PMC21078 that minute. They passed for
+    a while only because the fallback's container selector matched nothing at
+    all -- fixing that (dismech#12672) turned two of them red, which is the test
+    depending on a bug rather than the bug being load-bearing.
+    """
+    return ReferenceFetcher(
+        ReferenceValidationConfig(
+            cache_dir=tmp_path, email="me@example.org", fetch_full_text=False
+        )
+    )
 
 
 def _fetch_returning(fetcher, content_type, body):

linkml_reference_validator-0.3.0rc3/tests/test_pmc_modern_container.py

@@ -0,0 +1,75 @@
+"""PMC's current article markup must be recognised by the HTML fallback.
+
+The fallback looked for ``div.article-body`` or ``div.tsec``. PMC serves neither
+today: the article sits in ``<section class="body main-article-body">`` -- a
+different element *and* a different class. So the path declined for every PMC
+article, and the "retry when the source serves full text again to repair it"
+advice printed when a stale ``full_text_html`` entry is withheld pointed at a
+route that could never succeed (reported downstream as monarch-initiative/dismech#12672).
+
+The structural test is what makes this fetch safe -- a bot-check interstitial
+arrives on an HTTP 200 and carries no article container -- so widening the
+selector must not widen it to something an interstitial or a navigation page
+would satisfy.
+"""
+
+from __future__ import annotations
+
+import pytest
+from bs4 import BeautifulSoup
+
+from linkml_reference_validator.etl.fulltext.pmc import find_pmc_article_body
+
+BODY = "<p>" + ("Real article prose. " * 40) + "</p>"
+
+
+@pytest.mark.parametrize(
+    "markup,label",
+    [
+        (f'<section class="body main-article-body">{BODY}</section>', "current PMC"),
+        (f'<div class="article-body">{BODY}</div>', "legacy div.article-body"),
+        (f'<div class="tsec">{BODY}</div>', "legacy div.tsec"),
+        (f'<section class="main-article-body">{BODY}</section>', "class without 'body'"),
+    ],
+)
+def test_article_containers_are_found(markup, label):
+    soup = BeautifulSoup(markup, "html.parser")
+    assert find_pmc_article_body(soup) is not None, f"{label} should be recognised"
+
+
+@pytest.mark.parametrize(
+    "markup,label",
+    [
+        ("<html><body><p>Checking your browser before accessing.</p></body></html>",
+         "bot-check interstitial"),
+        ("<html><body><nav><a href='/'>Home</a></nav></body></html>", "navigation page"),
+        ("<html><body><div class='usa-banner'>An official website</div></body></html>",
+         "banner chrome only"),
+    ],
+)
+def test_pages_without_an_article_container_are_declined(markup, label):
+    soup = BeautifulSoup(markup, "html.parser")
+    assert find_pmc_article_body(soup) is None, f"{label} must not be accepted"
+
+
+def test_a_bare_body_element_is_not_an_article_container():
+    """``<body>`` is on every page, so matching class="body" alone would accept
+    an interstitial. Only the article-body classes count."""
+    soup = BeautifulSoup(f"<html><body>{BODY}</body></html>", "html.parser")
+    assert find_pmc_article_body(soup) is None
+
+
+def test_a_challenge_page_carrying_a_classed_body_is_still_declined():
+    """The interstitial arrives on an HTTP 200, so this selector is the only check.
+
+    PMC's challenge page carries a classed ``<body>``. None of its classes may
+    overlap the article-body list, or a bot-check page gets cached as full text.
+    """
+    markup = (
+        '<html><body class="usa-page bot-check">'
+        "<h1>Checking your browser before accessing</h1>"
+        "<p>Enable JavaScript and cookies to continue.</p>"
+        "</body></html>"
+    )
+    soup = BeautifulSoup(markup, "html.parser")
+    assert find_pmc_article_body(soup) is None

linkml_reference_validator-0.3.0rc3/tests/test_serve_stale_html_optin.py

@@ -0,0 +1,134 @@
+"""A consumer with a reviewed, committed cache can opt into serving stale HTML.
+
+Refusing stale ``full_text_html`` is right by default: a pre-fix entry may be a
+publisher landing page, and #12009 found exactly that stored as full text. But
+the refusal assumes the entry can be repaired by re-fetching, and for a
+PMC-only article it cannot -- PMC answers a Python HTTP client with a bot-check
+interstitial on an HTTP 200 regardless of headers, so the advertised repair route
+is unreachable.
+
+For a consumer whose cache is committed and reviewed, that turns a guard into
+data loss: every snippet quoting the body of any pre-stamp HTML entry fails, on
+files the pull request never touched (monarch-initiative/dismech#12672 --
+1,654 such files, ~294 of them with a snippet that is not in the abstract).
+
+``serve_stale_html_full_text`` lets that consumer say so explicitly. It changes
+only what may be *served*; nothing about overwriting, and nothing by default.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from linkml_reference_validator.etl.reference_fetcher import ReferenceFetcher
+from linkml_reference_validator.models import (
+    ReferenceContent,
+    ReferenceValidationConfig,
+)
+
+BODY_SENTENCE = "A sentence that exists only in the article body."
+
+
+def _cache(tmp_path):
+    (tmp_path / "PMID_30929739.md").write_text(
+        "---\nreference_id: PMID:30929739\ntitle: A paper\n"
+        f"content_type: full_text_html\n---\n\n## Content\n\n{BODY_SENTENCE}\n",
+        encoding="utf-8",
+    )
+
+
+def _fetch(tmp_path, **cfg):
+    _cache(tmp_path)
+    fetcher = ReferenceFetcher(
+        ReferenceValidationConfig(
+            cache_dir=tmp_path, email="me@example.org", fetch_full_text=False, **cfg
+        )
+    )
+
+    class _Source:
+        def fetch(self, identifier, config):
+            return None  # the source yields nothing: the stale-fallback path
+
+    with patch(
+        "linkml_reference_validator.etl.reference_fetcher"
+        ".ReferenceSourceRegistry.get_source",
+        return_value=_Source,
+    ):
+        return fetcher.fetch("PMID:30929739")
+
+
+def test_stale_html_is_withheld_by_default(tmp_path):
+    """The default must not change: an unrepaired entry is still not quotable."""
+    assert _fetch(tmp_path) is None
+
+
+def test_stale_html_is_served_when_the_consumer_opts_in(tmp_path):
+    content = _fetch(tmp_path, serve_stale_html_full_text=True)
+    assert content is not None, "opting in must make the cached body servable"
+    assert BODY_SENTENCE in (content.content or "")
+
+
+def test_opting_in_does_not_stamp_or_rewrite_the_entry(tmp_path):
+    """Serving is not migrating. The entry stays stale so a later run retries."""
+    _fetch(tmp_path, serve_stale_html_full_text=True)
+    text = (tmp_path / "PMID_30929739.md").read_text(encoding="utf-8")
+    assert "extractor_version" not in text
+    assert BODY_SENTENCE in text
+
+
+def _fetch_with_abstract_source(tmp_path, **cfg):
+    """The path a refresh normally takes: the source answers, with an abstract.
+
+    ``_stale_fallback`` needs the source to yield *nothing*. For a PMC-only
+    article PubMed still returns the abstract, so the refresh reaches
+    ``_preserve_cached_full_text`` instead: the overwrite is refused, and then a
+    separate question decides what may be served. That second question is the
+    one the flag has to answer, and it is the one dismech actually hits.
+    """
+    _cache(tmp_path)
+    fetcher = ReferenceFetcher(
+        ReferenceValidationConfig(
+            cache_dir=tmp_path, email="me@example.org", fetch_full_text=False, **cfg
+        )
+    )
+    fresh = ReferenceContent(
+        reference_id="PMID:30929739",
+        content="Just the abstract.",
+        content_type="abstract_only",
+        title="A paper",
+    )
+
+    class _Source:
+        def fetch(self, identifier, config):
+            return fresh
+
+    with patch(
+        "linkml_reference_validator.etl.reference_fetcher"
+        ".ReferenceSourceRegistry.get_source",
+        return_value=_Source,
+    ):
+        return fetcher.fetch("PMID:30929739")
+
+
+def test_refused_overwrite_serves_the_abstract_by_default(tmp_path):
+    content = _fetch_with_abstract_source(tmp_path)
+    assert content is not None
+    assert BODY_SENTENCE not in (content.content or ""), (
+        "by default the cached body stays withheld even though it was preserved"
+    )
+    assert "Just the abstract." in (content.content or "")
+
+
+def test_refused_overwrite_serves_the_cached_body_when_opted_in(tmp_path):
+    content = _fetch_with_abstract_source(tmp_path, serve_stale_html_full_text=True)
+    assert content is not None
+    assert BODY_SENTENCE in (content.content or "")
+
+
+def test_refused_overwrite_leaves_the_file_alone_either_way(tmp_path):
+    """Serving is not overwriting: the entry keeps its body and stays un-stamped."""
+    for flag in (False, True):
+        _fetch_with_abstract_source(tmp_path, serve_stale_html_full_text=flag)
+        text = (tmp_path / "PMID_30929739.md").read_text(encoding="utf-8")
+        assert BODY_SENTENCE in text, f"body lost with flag={flag}"
+        assert "extractor_version" not in text, f"entry stamped with flag={flag}"

linkml_reference_validator-0.3.0rc3/tests/test_trust_cached_entries.py

@@ -0,0 +1,81 @@
+"""A committed cache can be used as it is, without a re-fetch on every run.
+
+Staleness re-fetching is right for a cache the tool owns and can rebuild. For a
+cache that is committed and reviewed it inverts: the re-fetch is the risk, not
+the repair. It can shorten a body (dismech#12879), replace an entry with a
+bot-check page served on an HTTP 200 (dismech#12867), or withhold text that
+nothing can recover (dismech#12672).
+
+`trust_cached_entries` says use the file. Off by default, so caches still
+migrate for consumers that want that.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+
+from linkml_reference_validator.etl.reference_fetcher import ReferenceFetcher
+from linkml_reference_validator.models import ReferenceValidationConfig
+
+BODY = "A sentence a curator quoted, present only in this cached body."
+
+
+def _cache(tmp_path, content_type="full_text_html"):
+    (tmp_path / "PMID_20301575.md").write_text(
+        "---\nreference_id: PMID:20301575\ntitle: A paper\n"
+        f"content_type: {content_type}\n---\n\n## Content\n\n{BODY}\n",
+        encoding="utf-8",
+    )
+
+
+def _fetch(tmp_path, trust):
+    fetcher = ReferenceFetcher(
+        ReferenceValidationConfig(
+            cache_dir=tmp_path, email="me@example.org", trust_cached_entries=trust
+        )
+    )
+    calls = []
+
+    class _Source:
+        def fetch(self, identifier, config):
+            calls.append(identifier)
+            return None
+
+    with patch(
+        "linkml_reference_validator.etl.reference_fetcher"
+        ".ReferenceSourceRegistry.get_source",
+        return_value=_Source,
+    ):
+        return fetcher.fetch("PMID:20301575"), calls
+
+
+@pytest.mark.parametrize(
+    "content_type", ["full_text_html", "full_text_xml", "abstract_only"]
+)
+def test_a_trusted_entry_is_served_without_any_fetch(tmp_path, content_type):
+    """The point: no network call at all, so nothing can go wrong during one."""
+    _cache(tmp_path, content_type)
+    content, calls = _fetch(tmp_path, trust=True)
+
+    assert content is not None, f"{content_type} should be served from disk"
+    assert BODY in (content.content or "")
+    assert calls == [], f"a trusted entry must not be re-fetched, got {calls}"
+
+
+def test_the_file_is_left_exactly_as_it_was(tmp_path):
+    """Nothing is rewritten, so a validation run leaves the cache untouched."""
+    _cache(tmp_path)
+    before = (tmp_path / "PMID_20301575.md").read_text(encoding="utf-8")
+    _fetch(tmp_path, trust=True)
+    assert (tmp_path / "PMID_20301575.md").read_text(encoding="utf-8") == before
+
+
+def test_the_default_still_re_fetches(tmp_path):
+    """Off by default: an unstamped entry is still treated as stale."""
+    _cache(tmp_path)
+    _content, calls = _fetch(tmp_path, trust=False)
+    assert calls == ["20301575"], (
+        "by default an unstamped entry is re-fetched; the flag must be opt-in"
+    )