fetchpdf 0.1.0__py3-none-any.whl

fetchpdf/retrieval/context.py

@@ -0,0 +1,69 @@
+"""What a source is handed when the engine calls it.
+
+Its own module so `sources.*` can depend on it without importing the engine,
+which imports the sources back through the ladder config.
+"""
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+from .tiers import Ladder, Tier
+
+
+@dataclass
+class RetrievalContext:
+    """Per-record retrieval state shared with every source."""
+
+    http: object                       # http.HttpClient
+    resolver: object                   # resolve.BatchResolver
+    ladder: Ladder
+    save_path: str                     # the .pdf path the caller asked for
+    target_task: str = "extraction"
+    verbose: bool = False
+    use_playwright: bool = False
+    email: Optional[str] = None
+    delay: float = 0.1
+    provenance: Optional[object] = None   # provenance.ProvenanceRecord or None
+
+    #: --download-data-artifacts. Off, the linked-artifact providers route only
+    #: the paper's own material (CLASS_OWNED) and record everything else in the
+    #: sidecar without fetching it. On, they also route `related` deposits --
+    #: which is where the actual research data lives: in one 44-sidecar corpus
+    #: every `owned` link was a BioStudies mirror of supplementary files already
+    #: fetched, while the Zenodo and Dataverse replication packages were all
+    #: classified `related`.
+    download_data_artifacts: bool = False
+
+    #: Per-record ceiling for those artifacts, kept separate from the
+    #: supplementary budget so one large dataset cannot starve the paper's own
+    #: SI (or the reverse). None means the default in supplementary.py.
+    max_data_artifact_bytes: Optional[int] = None
+
+    #: --download-related-unverified. Off, a `related` deposit is skipped when
+    #: its own DataCite record declares it belongs to a DIFFERENT article. On,
+    #: every index assertion is taken at face value -- which is how "Tying
+    #: Odysseus to the Mast" ended up filed under a savings-reminder megastudy.
+    download_related_unverified: bool = False
+
+
+
+    #: Set once a T1 artifact has been accepted for this record. NCBI efetch and
+    #: Europe PMC serve the same underlying PMC JATS, so storing both is not
+    #: corroboration -- it manufactures false agreement downstream.
+    pmc_xml_already_taken: bool = False
+
+    #: Sources may leave breadcrumbs for later sources in the same walk.
+    scratch: dict = field(default_factory=dict)
+
+    def log(self, message: str) -> None:
+        if self.verbose:
+            print(message)
+
+    def threshold(self, key: str, default: int) -> int:
+        return self.ladder.threshold(key, default)
+
+    def is_extraction(self) -> bool:
+        return self.target_task == "extraction"
+
+
+__all__ = ["RetrievalContext", "Tier"]

fetchpdf/retrieval/corresponding.py

@@ -0,0 +1,349 @@
+"""Who to ask when the file cannot be fetched.
+
+For a paywalled paper or a withheld supplement the corresponding author is
+often the only remaining route -- no API has the file and no browser gets past
+the publisher. Their address is not in any metadata API, but it is nearly
+always in the artifacts already on disk.
+
+Measured over 43 papers in a real corpus:
+
+    Crossref  author[].email     0 of 3 probed -- not populated in practice
+    JATS      <corresp><email>   structural, exact, but only 7 records have XML
+    HTML      several shapes     highest precision when present (see below)
+    PDF       first two pages    26 of 31 PDF-only records
+
+Combined, ~79% of records lacking material yield an address. Everything here
+reads files already downloaded, so discovery costs no network requests.
+
+Precision, not recall, is the thing to protect: a wrong address emails a
+stranger in the user's name. The ordering below is by how *explicitly* each
+source states the corresponding role --
+
+    JATS  @corresp="yes"                    the document says so in markup
+    HTML  "Corresponding Author" + address  the page says so in markup
+    PDF   prose near a cue, scored          we are inferring
+
+-- and the scored PDF heuristic scores 97% (35/36 checkable picks) with 86%
+coverage. The single error was a Nature *news* page reprinting a different
+article's author block, which is why callers must skip records already
+classified as news/editorial.
+
+One HTML shape deserves its own note. Wiley renders the address as the literal
+text "[email protected]" -- Cloudflare email protection -- so a plain regex
+finds nothing on a page that does in fact carry the address. The real value is
+in `data-cfemail`, hex, XOR-ed with its own first byte. Verified:
+
+    data-cfemail="244a49455e45566446510a414051"  ->  nmazar@bu.edu
+
+which matches the same paper's PDF exactly.
+"""
+
+import html as _html
+import os
+import re
+from dataclasses import dataclass
+from typing import List, Optional
+
+#: How the address was found, most trustworthy first. Recorded on the Contact
+#: so a draft can show its provenance and a human can judge a stale address.
+SOURCE_JATS = "jats"
+SOURCE_HTML = "html"
+SOURCE_PDF = "pdf"
+
+_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
+
+#: Phrases that mark the following (or nearby) address as the corresponding
+#: author's. Taken from the shapes actually observed across the corpus, not
+#: guessed: "Corresponding author:", "E-mail addresses:", "Electronic mail may
+#: be sent to", "Address correspondence to".
+_CUE_RE = re.compile(
+    r"(correspond\w*|e-?mail address(es)?|electronic mail may be sent"
+    r"|address correspondence|to whom correspondence|e-?mail:)",
+    re.I,
+)
+
+#: Addresses that are never a person we should write to.
+_JUNK_RE = re.compile(
+    r"(^|[.@])(editor|permissions?|reprints?|support|info|help|noreply|no-reply"
+    r"|subscription|customerservice|webmaster|admin)([.@]|$)"
+    r"|@(wiley|elsevier|springernature|sciencedirect|tandf|sagepub)\.com$",
+    re.I,
+)
+
+#: How far back to look for a cue. One sentence, roughly -- far enough to cross
+#: "Corresponding author. E-mail address:" and its markup, short enough not to
+#: reach the previous author's block.
+_CUE_WINDOW = 160
+
+#: Author blocks live at the top. Reading two pages bounds the cost and avoids
+#: the reference list, where other people's addresses appear.
+_PDF_PAGES = 2
+
+
+@dataclass
+class Contact:
+    """One corresponding-author address, and where it came from."""
+
+    email: str
+    source: str                       # SOURCE_JATS | SOURCE_HTML | SOURCE_PDF
+    name: Optional[str] = None
+    cue: str = ""                     # the phrase that justified the pick
+    paper_doi: Optional[str] = None
+    paper_year: Optional[int] = None
+
+    @property
+    def confidence(self) -> str:
+        """Coarse, and honest about what each source proves."""
+        if self.source == SOURCE_JATS:
+            return "high"
+        if self.source == SOURCE_HTML:
+            return "high" if self.cue else "medium"
+        return "medium" if self.cue else "low"
+
+    @property
+    def provenance(self) -> str:
+        where = {SOURCE_JATS: "JATS <corresp>", SOURCE_HTML: "article HTML",
+                 SOURCE_PDF: "PDF first page"}.get(self.source, self.source)
+        bits = [where]
+        if self.paper_doi:
+            bits.append(f"of {self.paper_doi}")
+        if self.paper_year:
+            bits.append(f"({self.paper_year})")
+        if self.cue:
+            bits.append(f'cue "{self.cue}"')
+        return " ".join(bits)
+
+
+def _clean(email: str) -> str:
+    return email.strip().rstrip(".,;:)>\"'").lower()
+
+
+def _acceptable(email: str) -> bool:
+    return bool(email) and not _JUNK_RE.search(email)
+
+
+# -- Cloudflare email protection ---------------------------------------------
+
+def decode_cfemail(hex_blob: str) -> Optional[str]:
+    """Undo Cloudflare's data-cfemail obfuscation.
+
+    First byte is the XOR key; the rest is the address. Without this, every
+    Cloudflare-protected publisher page looks like it has no address at all --
+    the DOM shows only the literal string "[email protected]".
+    """
+    try:
+        raw = bytes.fromhex(hex_blob.strip())
+    except ValueError:
+        return None
+    if len(raw) < 2:
+        return None
+    key = raw[0]
+    try:
+        decoded = "".join(chr(byte ^ key) for byte in raw[1:])
+    except ValueError:
+        return None
+    return decoded if _EMAIL_RE.fullmatch(decoded) else None
+
+
+# -- JATS ---------------------------------------------------------------------
+
+_JATS_CORRESP_RE = re.compile(
+    r"<corresp\b[^>]*>(.*?)</corresp>", re.S | re.I)
+_JATS_EMAIL_RE = re.compile(r"<email[^>]*>([^<]+)</email>", re.I)
+
+
+def from_jats(text: str) -> Optional[Contact]:
+    """The address inside a <corresp> block -- the document's own statement."""
+    if not text:
+        return None
+    for block in _JATS_CORRESP_RE.findall(text):
+        found = _JATS_EMAIL_RE.search(block)
+        if found:
+            email = _clean(found.group(1))
+            if _acceptable(email):
+                return Contact(email=email, source=SOURCE_JATS,
+                               name=_name_in_corresp(block), cue="corresp")
+    # Some publishers put the email outside <corresp> but mark the author
+    # @corresp="yes"; fall back to the first <email> in the front matter.
+    head = text[:_front_matter_end(text)]
+    found = _JATS_EMAIL_RE.search(head)
+    if found:
+        email = _clean(found.group(1))
+        if _acceptable(email):
+            return Contact(email=email, source=SOURCE_JATS)
+    return None
+
+
+#: "should be addressed to: Mara Mather, 3715 McClintock Ave., ..." -- the
+#: name is the first comma-delimited fragment after the phrase. Only accepted
+#: when it looks like a personal name (2-4 capitalised words), so an address
+#: fragment or a department never becomes a salutation.
+_CORRESP_NAME_RE = re.compile(
+    r"(?:addressed to|correspondence to|author)[:\s]+"
+    r"([A-Z][A-Za-z.'\u2019-]+(?:\s+[A-Z][A-Za-z.'\u2019-]+){1,3})(?=[,.]|\s+at\b)")
+
+
+def _name_in_corresp(block: str) -> Optional[str]:
+    """The addressee named inside a <corresp> block, if it states one.
+
+    Better than "Dear Dr. [name]" in a draft the user has to edit, and safe to
+    skip when absent -- a wrong name is worse than a placeholder.
+    """
+    plain = " ".join(_TAG_RE.sub(" ", block or "").split())
+    found = _CORRESP_NAME_RE.search(plain)
+    return found.group(1).strip() if found else None
+
+
+def _front_matter_end(text: str) -> int:
+    """Where the article body starts, so we never read the reference list."""
+    for marker in ("</front>", "<body", "<ref-list"):
+        index = text.find(marker)
+        if index > 0:
+            return index
+    return min(len(text), 20000)
+
+
+# -- HTML ---------------------------------------------------------------------
+
+_TAG_RE = re.compile(r"<[^>]+>")
+_SCRIPT_RE = re.compile(r"<script.*?</script>|<style.*?</style>", re.S | re.I)
+_CFEMAIL_RE = re.compile(r'data-cfemail="([a-fA-F0-9]+)"')
+_MAILTO_RE = re.compile(r'href="mailto:([^"?]+)', re.I)
+_META_EMAIL_RE = re.compile(
+    r'<meta[^>]+name="citation_author_email"[^>]+content="([^"]+)"', re.I)
+
+
+def from_html(text: str) -> Optional[Contact]:
+    """Address from an article page, in descending order of explicitness."""
+    if not text:
+        return None
+
+    # 1. An explicit mailto is unambiguous.
+    for match in _MAILTO_RE.finditer(text):
+        email = _clean(match.group(1))
+        if _acceptable(email):
+            near = _TAG_RE.sub(" ", text[max(0, match.start() - 300):match.start()])
+            cue = _CUE_RE.search(near)
+            return Contact(email=email, source=SOURCE_HTML,
+                           cue=cue.group(0) if cue else "")
+
+    # 2. Cloudflare-obfuscated. Prefer one whose surrounding text says
+    #    "Corresponding Author" -- Wiley marks the role in the DOM even while
+    #    hiding the address.
+    best = None
+    for match in _CFEMAIL_RE.finditer(text):
+        email = decode_cfemail(match.group(1))
+        if not email or not _acceptable(_clean(email)):
+            continue
+        near = _TAG_RE.sub(" ", text[max(0, match.start() - 400):match.start()])
+        cue = _CUE_RE.search(near)
+        candidate = Contact(email=_clean(email), source=SOURCE_HTML,
+                            cue=cue.group(0) if cue else "")
+        if cue:
+            return candidate
+        best = best or candidate
+    if best:
+        return best
+
+    # 3. Publisher metadata, when emitted.
+    found = _META_EMAIL_RE.search(text)
+    if found:
+        email = _clean(found.group(1))
+        if _acceptable(email):
+            return Contact(email=email, source=SOURCE_HTML, cue="citation_author_email")
+
+    # 4. A bare address in the visible text, scored like the PDF case.
+    visible = _html.unescape(_TAG_RE.sub(" ", _SCRIPT_RE.sub(" ", text)))
+    return _scored_pick(visible, SOURCE_HTML)
+
+
+# -- PDF ----------------------------------------------------------------------
+
+def from_pdf(path: str) -> Optional[Contact]:
+    """Address from the first pages of the PDF.
+
+    pypdf rather than a pdftotext subprocess: pure Python, so this works on
+    Windows and needs no external binary.
+    """
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        return None
+    try:
+        reader = PdfReader(path)
+        pages = reader.pages[:_PDF_PAGES]
+        text = "\n".join((page.extract_text() or "") for page in pages)
+    except Exception:
+        return None
+    return _scored_pick(text, SOURCE_PDF)
+
+
+def _scored_pick(text: str, source: str) -> Optional[Contact]:
+    """The most likely corresponding address in a block of prose.
+
+    14 of 43 corpus PDFs carry more than one address, so first-match is wrong.
+    Score: +2 for a corresponding-author cue just before the match, +1 for
+    sitting in the first half of the text (author blocks precede everything
+    else). Ties break toward the earlier match.
+    """
+    if not text:
+        return None
+    best_score = -1
+    best: Optional[Contact] = None
+    midpoint = max(1, len(text) // 2)
+    for match in _EMAIL_RE.finditer(text):
+        email = _clean(match.group(0))
+        if not _acceptable(email):
+            continue
+        before = text[max(0, match.start() - _CUE_WINDOW):match.start()]
+        cue = _CUE_RE.search(before)
+        score = (2 if cue else 0) + (1 if match.start() < midpoint else 0)
+        if score > best_score:
+            best_score = score
+            best = Contact(email=email, source=source,
+                           cue=cue.group(0) if cue else "")
+    return best
+
+
+# -- the record-level entry point ---------------------------------------------
+
+#: Extensions searched, in the precision order argued for in the docstring.
+_MARKUP = ((".xml", from_jats), (".nxml", from_jats),
+           (".fulltext.html", from_html), (".html", from_html),
+           (".htm", from_html))
+
+
+def find_corresponding_author(stem: str, doi: Optional[str] = None,
+                              year: Optional[int] = None) -> Optional[Contact]:
+    """Best available address for one record, from artifacts already on disk.
+
+    `stem` is the output path stem, matching how the rest of the supplementary
+    subsystem addresses a record. Returns None when nothing usable is found --
+    the caller lists those separately rather than guessing.
+    """
+    if not stem:
+        return None
+
+    for extension, extract in _MARKUP:
+        path = stem + extension
+        if not os.path.exists(path):
+            continue
+        try:
+            with open(path, encoding="utf-8", errors="replace") as handle:
+                text = handle.read()
+        except OSError:
+            continue
+        contact = extract(text)
+        if contact:
+            contact.paper_doi = contact.paper_doi or doi
+            contact.paper_year = contact.paper_year or year
+            return contact
+
+    pdf_path = stem + ".pdf"
+    if os.path.exists(pdf_path):
+        contact = from_pdf(pdf_path)
+        if contact:
+            contact.paper_doi = contact.paper_doi or doi
+            contact.paper_year = contact.paper_year or year
+            return contact
+    return None