linkml-reference-validator 0.3.0rc1__tar.gz → 0.3.0rc2__tar.gz

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: linkml_reference_validator
-Version: 0.3.0rc1
+Version: 0.3.0rc2
 Summary: Validation of supporting text from references and publications
 Author-email: Chris Mungall <cjmungall@lbl.gov>
 License-Expression: Apache-2.0

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc2}/src/linkml_reference_validator/etl/fulltext/pmc.py

@@ -6,7 +6,7 @@ fetched from the PMC XML API (with an HTML fallback) and extracted via XMLExtrac
 
 import logging
 import time
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 from Bio import Entrez  # type: ignore
 from bs4 import BeautifulSoup  # type: ignore
@@ -38,6 +38,45 @@ class TransientFullTextError(RuntimeError):
     """
 
 
+#: Classes PMC has used for the article body. ``main-article-body`` is what it
+#: serves now, on a ``<section>``; the two ``div`` classes are the older markup
+#: and are kept so entries cached against them still re-extract.
+#:
+#: Deliberately not ``body``, which PMC pairs with ``main-article-body`` on the
+#: same element. Matching it alone would match every page's ``<body>``, and the
+#: structural test is the only thing standing between this fetch and caching a
+#: bot-check interstitial served on an HTTP 200.
+PMC_ARTICLE_BODY_CLASSES = ("main-article-body", "article-body", "tsec")
+
+
+def _find_pmc_article_body(soup: BeautifulSoup) -> Optional[Any]:
+    """Return PMC's article-body element, or None if the page has none.
+
+    Matches on class rather than element name: the container moved from ``div``
+    to ``section`` when PMC changed its markup, so pinning the tag is what broke
+    the previous version for every article.
+
+    Args:
+        soup: A parsed PMC article page
+
+    Returns:
+        The article-body element, or None
+
+    Examples:
+        >>> from bs4 import BeautifulSoup
+        >>> html = '<section class="body main-article-body"><p>Text.</p></section>'
+        >>> _find_pmc_article_body(BeautifulSoup(html, "html.parser")) is not None
+        True
+        >>> _find_pmc_article_body(BeautifulSoup("<body><p>x</p></body>", "html.parser")) is None
+        True
+    """
+    for class_name in PMC_ARTICLE_BODY_CLASSES:
+        found = soup.find(class_=class_name)
+        if found is not None:
+            return found
+    return None
+
+
 @FullTextProviderRegistry.register
 class PMCFullTextProvider(FullTextProvider):
     """Provide PMC full text for a reference identified by PMID/PMCID.
@@ -132,7 +171,8 @@ class PMCFullTextProvider(FullTextProvider):
 
         This is the one page fetch the OA providers' "a landing page is not full
         text" rule does not cover, and what makes it safe is the requirement
-        below: text is returned only from a ``div.article-body`` or ``div.tsec``.
+        below: text is returned only from an element carrying one of
+        :data:`PMC_ARTICLE_BODY_CLASSES`.
         A bot-check interstitial -- which PMC serves on an HTTP 200, so no status
         check sees it -- carries neither, so this yields ``None`` rather than
         caching the page. The ``oa_url`` fallback that was removed had no such
@@ -152,7 +192,7 @@ class PMCFullTextProvider(FullTextProvider):
             )
 
         soup = BeautifulSoup(response.content, "html.parser")
-        article_body = soup.find("div", class_="article-body") or soup.find("div", class_="tsec")
+        article_body = _find_pmc_article_body(soup)
         if article_body:
             # The region is selected here, but the text comes out of the shared
             # extractor rather than a private copy of the paragraph walk, so

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc2}/src/linkml_reference_validator/etl/reference_fetcher.py

@@ -424,7 +424,11 @@ class ReferenceFetcher:
         if force_refresh:
             return FetchOutcome(content=None)
 
-        stale = self._load_from_disk(normalized_reference_id, allow_stale=True)
+        stale = self._load_from_disk(
+            normalized_reference_id,
+            allow_stale=True,
+            allow_stale_html=self.config.serve_stale_html_full_text,
+        )
         if stale is None:
             return FetchOutcome(content=None)
 
@@ -520,7 +524,11 @@ class ReferenceFetcher:
         # May its text be served? Asked without the bypass, so stale HTML is
         # withheld here exactly as _stale_fallback withholds it, and validation
         # falls back to the freshly fetched abstract.
-        servable = self._load_from_disk(normalized_reference_id, allow_stale=True)
+        servable = self._load_from_disk(
+            normalized_reference_id,
+            allow_stale=True,
+            allow_stale_html=self.config.serve_stale_html_full_text,
+        )
         return FetchOutcome(content=servable or fresh, served_stale=True)
 
     def _report_shrinking_refresh(

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc2}/src/linkml_reference_validator/etl/sources/pmid.py

@@ -582,9 +582,12 @@ class PMIDSource(ReferenceSource):
             return None
 
         soup = BeautifulSoup(response.content, "html.parser")
-        article_body = soup.find("div", class_="article-body") or soup.find(
-            "div", class_="tsec"
-        )
+        # Shared with the PMC full-text provider: the container moved from a
+        # ``div`` to a ``section`` and changed class, so both call sites went
+        # blind at once (dismech#12672).
+        from linkml_reference_validator.etl.fulltext.pmc import _find_pmc_article_body
+
+        article_body = _find_pmc_article_body(soup)
 
         if article_body:
             # Region selected here, text extracted by the shared extractor, for

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc2}/src/linkml_reference_validator/models.py

@@ -510,6 +510,20 @@ class ReferenceValidationConfig(BaseModel):
             "when a metadata source does not already return full text."
         ),
     )
+    serve_stale_html_full_text: bool = Field(
+        default=False,
+        description=(
+            "If True, serve a stale full_text_html cache entry as evidence instead "
+            "of withholding it. Off by default: a pre-stamp HTML entry may be a "
+            "publisher landing page rather than the article, and the default "
+            "assumes a re-fetch can repair it. Set this only for a cache you "
+            "control and review -- for a PMC-only article the repair route is "
+            "unreachable (PMC answers a Python HTTP client with a bot-check "
+            "interstitial on an HTTP 200), so the guard withholds text that "
+            "cannot be recovered. Affects serving only, never overwriting, and "
+            "the entry is left un-stamped so a later run still retries."
+        ),
+    )
     full_text_providers: list[str] = Field(
         default_factory=lambda: ["pmc", "epmc_preprint", "unpaywall", "openalex"],
         description=(

{linkml_reference_validator-0.3.0rc1 → linkml_reference_validator-0.3.0rc2}/tests/test_no_cache_downgrade.py

@@ -76,7 +76,21 @@ def _write_cache(cache_dir, reference_id, content_type, body, stamped=False):
 
 @pytest.fixture
 def fetcher(tmp_path):
-    return ReferenceFetcher(ReferenceValidationConfig(cache_dir=tmp_path, email="me@example.org"))
+    """A fetcher with the full-text chain off.
+
+    These tests are about what a *refresh that loses full text* does to the
+    cached file, so the chain has to be absent rather than merely expected to
+    fail. With it on they reach the network: the PMC HTML fallback fires and the
+    outcome depends on what PMC serves for PMC21078 that minute. They passed for
+    a while only because the fallback's container selector matched nothing at
+    all -- fixing that (dismech#12672) turned two of them red, which is the test
+    depending on a bug rather than the bug being load-bearing.
+    """
+    return ReferenceFetcher(
+        ReferenceValidationConfig(
+            cache_dir=tmp_path, email="me@example.org", fetch_full_text=False
+        )
+    )
 
 
 def _fetch_returning(fetcher, content_type, body):

linkml_reference_validator-0.3.0rc2/tests/test_pmc_modern_container.py

@@ -0,0 +1,59 @@
+"""PMC's current article markup must be recognised by the HTML fallback.
+
+The fallback looked for ``div.article-body`` or ``div.tsec``. PMC serves neither
+today: the article sits in ``<section class="body main-article-body">`` -- a
+different element *and* a different class. So the path declined for every PMC
+article, and the "retry when the source serves full text again to repair it"
+advice printed when a stale ``full_text_html`` entry is withheld pointed at a
+route that could never succeed (reported downstream as monarch-initiative/dismech#12672).
+
+The structural test is what makes this fetch safe -- a bot-check interstitial
+arrives on an HTTP 200 and carries no article container -- so widening the
+selector must not widen it to something an interstitial or a navigation page
+would satisfy.
+"""
+
+from __future__ import annotations
+
+import pytest
+from bs4 import BeautifulSoup
+
+from linkml_reference_validator.etl.fulltext.pmc import _find_pmc_article_body
+
+BODY = "<p>" + ("Real article prose. " * 40) + "</p>"
+
+
+@pytest.mark.parametrize(
+    "markup,label",
+    [
+        (f'<section class="body main-article-body">{BODY}</section>', "current PMC"),
+        (f'<div class="article-body">{BODY}</div>', "legacy div.article-body"),
+        (f'<div class="tsec">{BODY}</div>', "legacy div.tsec"),
+        (f'<section class="main-article-body">{BODY}</section>', "class without 'body'"),
+    ],
+)
+def test_article_containers_are_found(markup, label):
+    soup = BeautifulSoup(markup, "html.parser")
+    assert _find_pmc_article_body(soup) is not None, f"{label} should be recognised"
+
+
+@pytest.mark.parametrize(
+    "markup,label",
+    [
+        ("<html><body><p>Checking your browser before accessing.</p></body></html>",
+         "bot-check interstitial"),
+        ("<html><body><nav><a href='/'>Home</a></nav></body></html>", "navigation page"),
+        ("<html><body><div class='usa-banner'>An official website</div></body></html>",
+         "banner chrome only"),
+    ],
+)
+def test_pages_without_an_article_container_are_declined(markup, label):
+    soup = BeautifulSoup(markup, "html.parser")
+    assert _find_pmc_article_body(soup) is None, f"{label} must not be accepted"
+
+
+def test_a_bare_body_element_is_not_an_article_container():
+    """``<body>`` is on every page, so matching class="body" alone would accept
+    an interstitial. Only the article-body classes count."""
+    soup = BeautifulSoup(f"<html><body>{BODY}</body></html>", "html.parser")
+    assert _find_pmc_article_body(soup) is None

linkml_reference_validator-0.3.0rc2/tests/test_serve_stale_html_optin.py

@@ -0,0 +1,75 @@
+"""A consumer with a reviewed, committed cache can opt into serving stale HTML.
+
+Refusing stale ``full_text_html`` is right by default: a pre-fix entry may be a
+publisher landing page, and #12009 found exactly that stored as full text. But
+the refusal assumes the entry can be repaired by re-fetching, and for a
+PMC-only article it cannot -- PMC answers a Python HTTP client with a bot-check
+interstitial on an HTTP 200 regardless of headers, so the advertised repair route
+is unreachable.
+
+For a consumer whose cache is committed and reviewed, that turns a guard into
+data loss: every snippet quoting the body of any pre-stamp HTML entry fails, on
+files the pull request never touched (monarch-initiative/dismech#12672 --
+1,654 such files, ~294 of them with a snippet that is not in the abstract).
+
+``serve_stale_html_full_text`` lets that consumer say so explicitly. It changes
+only what may be *served*; nothing about overwriting, and nothing by default.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from linkml_reference_validator.etl.reference_fetcher import ReferenceFetcher
+from linkml_reference_validator.models import (
+    ReferenceValidationConfig,
+)
+
+BODY_SENTENCE = "A sentence that exists only in the article body."
+
+
+def _cache(tmp_path):
+    (tmp_path / "PMID_30929739.md").write_text(
+        "---\nreference_id: PMID:30929739\ntitle: A paper\n"
+        f"content_type: full_text_html\n---\n\n## Content\n\n{BODY_SENTENCE}\n",
+        encoding="utf-8",
+    )
+
+
+def _fetch(tmp_path, **cfg):
+    _cache(tmp_path)
+    fetcher = ReferenceFetcher(
+        ReferenceValidationConfig(
+            cache_dir=tmp_path, email="me@example.org", fetch_full_text=False, **cfg
+        )
+    )
+
+    class _Source:
+        def fetch(self, identifier, config):
+            return None  # the source yields nothing: the stale-fallback path
+
+    with patch(
+        "linkml_reference_validator.etl.reference_fetcher"
+        ".ReferenceSourceRegistry.get_source",
+        return_value=_Source,
+    ):
+        return fetcher.fetch("PMID:30929739")
+
+
+def test_stale_html_is_withheld_by_default(tmp_path):
+    """The default must not change: an unrepaired entry is still not quotable."""
+    assert _fetch(tmp_path) is None
+
+
+def test_stale_html_is_served_when_the_consumer_opts_in(tmp_path):
+    content = _fetch(tmp_path, serve_stale_html_full_text=True)
+    assert content is not None, "opting in must make the cached body servable"
+    assert BODY_SENTENCE in (content.content or "")
+
+
+def test_opting_in_does_not_stamp_or_rewrite_the_entry(tmp_path):
+    """Serving is not migrating. The entry stays stale so a later run retries."""
+    _fetch(tmp_path, serve_stale_html_full_text=True)
+    text = (tmp_path / "PMID_30929739.md").read_text(encoding="utf-8")
+    assert "extractor_version" not in text
+    assert BODY_SENTENCE in text