fetchpdf 0.1.6__tar.gz → 0.1.7__tar.gz

{fetchpdf-0.1.6/fetchpdf.egg-info → fetchpdf-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fetchpdf
-Version: 0.1.6
+Version: 0.1.7
 Summary: A comprehensive tool to download PDFs from DOIs using multiple sources
 Author-email: Dan Elton <delton17@gmail.com>
 License-Expression: MIT

{fetchpdf-0.1.6 → fetchpdf-0.1.7}/fetchpdf/__init__.py

@@ -15,7 +15,7 @@ from .fetch_metadata_from_doi import fetch_metadata_from_doi
 # behavioural difference, it is the same object.
 fetch_pdf_from_doi = fetch_pdf
 
-__version__ = "0.1.6"
+__version__ = "0.1.7"
 __all__ = [
     "fetch_pdf",
     "fetch_pdf_from_doi",  # deprecated alias for fetch_pdf

{fetchpdf-0.1.6 → fetchpdf-0.1.7}/fetchpdf/retrieval/pdf_identity.py

@@ -160,6 +160,44 @@ PAGE_COUNT_MIN_SPAN = 2
 
 _PAGE_RANGE_RE = re.compile(r"^\s*(?:[A-Za-z]*)(\d+)\s*(?:[-–—]\s*(?:[A-Za-z]*)(\d+))?\s*$")
 
+#: S1 on its own is not enough when the front pages are legible and do not carry
+#: the title (issue #5). The requested DOI is printed by the paper itself, but
+#: also by a paper that CITES it and by the article's own supplementary file:
+#: fetch_pdf("10.1037/0022-3514.74.6.1464") wrote 10.5964/meth.7155, whose
+#: abstract reads "(IAT; Greenwald et al., 1998. DOI: 10.1037/0022-3514.74.6.1464)",
+#: and the Supplementary Information PDF for 10.1038/nature08833 prints
+#: "doi: 10.1038/nature08833" on page 1. Measured over 121 cached PDFs whose DOI
+#: was found in the text, scoring the Crossref title against the front two pages:
+#: the citing paper 0.52, the supplement 0.64, a line-numbered medRxiv preprint
+#: (correct) 0.76, the other 118 correct papers >= 0.97. 0.70 sits in the gap; it
+#: is narrow, so a page too short to judge (LEGIBLE_FRONT_CHARS) keeps the DOI
+#: verdict rather than risk refusing a correct scan.
+DOI_TITLE_PAGES = 2
+DOI_TITLE_MIN = 0.70
+
+#: The same DOI printed inside an author-year citation -- "Greenwald et al.,
+#: 1998. DOI: 10.1037/...", "(Smith & Jones, 2020, https://doi.org/...)" -- is a
+#: reference to the paper, not the paper's own masthead. Matched on the text
+#: just before the DOI, so it also works when no title is known to compare.
+_CITED_BEFORE_DOI_RE = re.compile(
+    r"(?:et\s+al\.?|&\s*[a-z][a-z'\-]+|\band\s+[a-z][a-z'\-]+)"   # et al. / & Name / and Name
+    r"[\s,]*\(?\s*(?:1[89]|20)\d{2}[a-z]?\s*\)?"                     # , 1998 / (1998) / 2020a
+    r"\s*[.,;:]?\s*"
+    r"(?:doi\s*[:.]?\s*|(?:https?\s*:\s*//\s*)?(?:dx\s*\.\s*)?doi\s*\.\s*org\s*/\s*)?$",
+    re.IGNORECASE,
+)
+#: How far before the DOI the author-year pattern may end, and how far back a
+#: "cite this article" line may start. A paper's own front matter often prints
+#: a suggested citation in exactly the cited shape ("Cite this article as: Smith
+#: et al., 2021. https://doi.org/..."), and that is the paper's own DOI.
+CITED_CONTEXT_CHARS = 80
+OWN_CITATION_LINE_CHARS = 400
+_OWN_CITATION_RE = re.compile(
+    r"cite\s+(?:this|the)\s+(?:article|paper|chapter|work|preprint)|please\s+cite"
+    r"|how\s+to\s+cite|citation\s*:|recommended\s+citation|cite\s+as",
+    re.IGNORECASE,
+)
+
 #: Embedded titles that carry no identity: a converter's leftovers, a filename,
 #: a template default. Present in a large minority of real PDFs, and treating
 #: them as "declares a different article" would reject correct files.
@@ -279,11 +317,24 @@ def verify_pdf_identity(source, doi: Optional[str] = None,
 
     # S1: the requested DOI, printed on the paper. True of essentially every
     # version of record, and squashing means a line break inside it is harmless.
+    # Not on its own, though, when the DOI is only cited, or when the front pages
+    # are legible and do not carry the title (DOI_TITLE_MIN): then the later
+    # signals decide, and a refusal says the DOI was seen and why it did not count.
+    doi_discounted = ""
     if doi:
         wanted = squash(normalise_doi(doi))
         if wanted and wanted in squashed_page:
-            return truncated or IdentityVerdict(
-                VERIFIED, "requested DOI found in the PDF text", ["doi-in-text"])
+            if _doi_only_cited(text, doi):
+                doi_discounted = "only inside a citation"
+            elif title:
+                two_pages = squash(reader.text(DOI_TITLE_PAGES) or text)
+                fraction = matched_fraction(squash(title), two_pages)
+                if len(two_pages) >= LEGIBLE_FRONT_CHARS and fraction < DOI_TITLE_MIN:
+                    doi_discounted = ("but its front pages carry only {:.0%} of the "
+                                      "title".format(fraction))
+            if not doi_discounted:
+                return truncated or IdentityVerdict(
+                    VERIFIED, "requested DOI found in the PDF text", ["doi-in-text"])
 
     # S1b: the arXiv id, stamped down the margin of every PDF arXiv serves. It
     # is S1 for a class of record that S1 cannot reach and neither can S2-S4:
@@ -410,10 +461,23 @@ def verify_pdf_identity(source, doi: Optional[str] = None,
                 "so it cannot be checked against {}".format(
                     len(squashed_page), DOI_PAGES, doi or "the requested record"),
             )
+        if doi_discounted:
+            return IdentityVerdict(
+                WRONG,
+                'PDF prints the requested DOI {}, so it is not taken as "{}" '
+                "(likely a paper citing it, or its supplement)".format(
+                    doi_discounted, _clip(title)),
+            )
         return IdentityVerdict(
             WRONG,
             'PDF does not contain the requested DOI or the title "{}"'.format(_clip(title)),
         )
+    if doi_discounted:
+        return IdentityVerdict(
+            NO_REFERENCE,
+            "PDF prints {} {}, and no title is known to check it against".format(
+                doi or "the requested DOI", doi_discounted),
+        )
     return IdentityVerdict(
         NO_REFERENCE,
         "no title known for {} and its DOI is not printed in the PDF, so there "
@@ -630,6 +694,32 @@ def _is_generic_title(embedded: str) -> bool:
     return bool(_GENERIC_TITLE_RE.match(stripped) or _FILENAME_TITLE_RE.match(stripped))
 
 
+def _doi_only_cited(text: str, doi: str) -> bool:
+    """Whether every place `doi` is printed in `text` is inside a citation.
+
+    True only when the DOI appears at least once and each appearance follows an
+    author-year citation ("et al., 1998. DOI:") with no "cite this article" line
+    before it. A DOI printed once in a masthead and once in a citation is the
+    paper's own, so one non-citation appearance is enough to say False.
+    """
+    wanted = normalise_doi(doi)
+    if not wanted:
+        return False
+    # Line wraps land anywhere inside a DOI, so allow whitespace between its characters.
+    pattern = re.compile(r"\s*".join(re.escape(c) for c in wanted), re.IGNORECASE)
+    hits = list(pattern.finditer(text))
+    if not hits:
+        return False
+    for hit in hits:
+        before = text[max(0, hit.start() - CITED_CONTEXT_CHARS):hit.start()]
+        if not _CITED_BEFORE_DOI_RE.search(before):
+            return False
+        line_window = text[max(0, hit.start() - OWN_CITATION_LINE_CHARS):hit.start()]
+        if _OWN_CITATION_RE.search(line_window):
+            return False
+    return True
+
+
 def _clip(text: str, limit: int = 70) -> str:
     text = " ".join((text or "").split())
     return text if len(text) <= limit else text[:limit - 1] + "…"

{fetchpdf-0.1.6 → fetchpdf-0.1.7/fetchpdf.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fetchpdf
-Version: 0.1.6
+Version: 0.1.7
 Summary: A comprehensive tool to download PDFs from DOIs using multiple sources
 Author-email: Dan Elton <delton17@gmail.com>
 License-Expression: MIT

{fetchpdf-0.1.6 → fetchpdf-0.1.7}/tests/test_pdf_identity.py

@@ -14,6 +14,7 @@ from conftest import text_pdf
 from fetchpdf.retrieval import pdf_identity
 from fetchpdf.retrieval.pdf_identity import (
     NO_ENGINE,
+    NO_REFERENCE,
     TRUNCATED,
     UNREADABLE,
     VERIFIED,
@@ -583,3 +584,106 @@ def test_a_wall_is_discarded_even_when_metadata_is_unreachable(tmp_path, monkeyp
     monkeypatch.setattr(fpd, "_title_for", lambda doi, verbose=False: None)   # Crossref down
     assert fpd._accept_downloaded_pdf(str(path), DOI) is False
     assert not os.path.exists(path)
+
+
+# --------------------------------------------------------------------------
+# The requested DOI is printed, but not as this paper's own (issue #5)
+# --------------------------------------------------------------------------
+
+IAT_DOI = "10.1037/0022-3514.74.6.1464"
+IAT_TITLE = ("Measuring individual differences in implicit cognition: "
+             "The implicit association test")
+CITING_TITLE = ("Filling the gap between implicit associations and behavior: "
+                "A linear mixed-effects Rasch analysis of the Implicit Association Test")
+CITING_PROSE = ["Participants completed the task in a single session and responses "
+                "were modelled with crossed random effects for items and persons."] * 8
+
+
+def test_paper_citing_the_doi_is_not_accepted_as_it(tmp_path):
+    """10.5964/meth.7155, fetched for the 1998 IAT paper (issue #5).
+
+    Its abstract prints the requested DOI inside a citation. Before the fix the
+    DOI alone accepted it.
+    """
+    path = _pdf(tmp_path, "citing.pdf", [
+        CITING_TITLE,
+        "The measure obtained from the Implicit Association Test (IAT; Greenwald",
+        f"et al., 1998. DOI: {IAT_DOI}) is often used to predict behaviour.",
+    ] + CITING_PROSE)
+    verdict = verify_pdf_identity(path, IAT_DOI, IAT_TITLE)
+    assert verdict.state == WRONG
+    assert "only inside a citation" in verdict.reason
+
+
+def test_paper_citing_the_doi_is_not_accepted_when_no_title_is_known(tmp_path):
+    """The citation shape alone is enough to refuse: there is no title to check."""
+    path = _pdf(tmp_path, "citing-untitled.pdf", [
+        CITING_TITLE,
+        f"(Greenwald, McGhee, & Schwartz, 1998, https://doi.org/{IAT_DOI}).",
+    ] + CITING_PROSE)
+    verdict = verify_pdf_identity(path, IAT_DOI, None)
+    assert verdict.state == NO_REFERENCE
+    assert "only inside a citation" in verdict.reason
+
+
+def test_supplement_printing_the_doi_without_the_title_is_not_accepted(tmp_path):
+    """The Supplementary Information PDF for 10.1038/nature08833 prints the
+    article DOI on page 1 but not its title (0.64 of it, measured)."""
+    path = _pdf(tmp_path, "si.pdf", [
+        "SUPPLEMENTARY INFORMATION",
+        "doi: 10.1038/nature08833",
+        "Supplementary Methods, Supplementary Figures S1-S12 and Supplementary Tables S1-S4",
+    ] + ["Supplementary Figure legends and additional methods for the cohort "
+         "analyses, including sequencing depth and quality filters."] * 8)
+    verdict = verify_pdf_identity(path, "10.1038/nature08833", TITLE)
+    assert verdict.state == WRONG
+    assert "carry only" in verdict.reason
+
+
+def test_own_suggested_citation_still_counts_as_the_papers_doi(tmp_path):
+    """The false-reject guard for the citation rule. Front matter often prints the
+    paper's own suggested citation in exactly the cited shape."""
+    path = _pdf(tmp_path, "own-cite.pdf", [
+        "Cite this article as: Smith et al., 2021.",
+        f"https://doi.org/{DOI}",
+    ] + PROSE)
+    verdict = verify_pdf_identity(path, DOI, None)
+    assert verdict.state == VERIFIED
+    assert verdict.signals == ["doi-in-text"]
+
+
+def test_doi_in_masthead_and_in_a_citation_is_the_papers_own(tmp_path):
+    """One appearance outside a citation is enough to keep S1."""
+    path = _pdf(tmp_path, "masthead.pdf", [
+        f"https://doi.org/{DOI}",
+        f"As reported earlier (Smith et al., 2021. doi: {DOI}), the index held.",
+    ] + PROSE)
+    verdict = verify_pdf_identity(path, DOI, None)
+    assert verdict.state == VERIFIED
+    assert verdict.signals == ["doi-in-text"]
+
+
+def test_line_numbered_preprint_with_a_partly_legible_title_keeps_the_doi(tmp_path):
+    """A line-numbered medRxiv preprint (correct paper) scored 0.76: above the bar."""
+    words = TITLE.split()
+    half = len(words) // 2
+    numbered = [f"1 {' '.join(words[:half])}", "2 3 4 5 6", f"7 {' '.join(words[half:])}"]
+    path = _pdf(tmp_path, "medrxiv.pdf", numbered + [f"doi: {DOI}"] + PROSE)
+    from fetchpdf.retrieval._util import matched_fraction, squash
+    front = squash("\n".join(numbered + [f"doi: {DOI}"] + PROSE))
+    assert matched_fraction(squash(TITLE), front) >= pdf_identity.DOI_TITLE_MIN
+    verdict = verify_pdf_identity(path, DOI, TITLE)
+    assert verdict.state == VERIFIED
+
+
+def test_garbled_front_page_keeps_the_doi_verdict(tmp_path):
+    """Too little text to judge the title: the DOI is still trusted."""
+    # Between MIN_FRONT_CHARS (read at all) and LEGIBLE_FRONT_CHARS (judged).
+    stamp = ["Downloaded from https://academic.example.org by guest on June 5, 2016"] * 4
+    path = _pdf(tmp_path, "scan.pdf", [f"doi:{DOI}"] + stamp)
+    from fetchpdf.retrieval._util import squash
+    chars = len(squash(chr(10).join([f"doi:{DOI}"] + stamp)))
+    assert pdf_identity.MIN_FRONT_CHARS <= chars < pdf_identity.LEGIBLE_FRONT_CHARS, chars
+    verdict = verify_pdf_identity(path, DOI, TITLE)
+    assert verdict.state == VERIFIED
+    assert verdict.signals == ["doi-in-text"]

{fetchpdf-0.1.6 → fetchpdf-0.1.7}/tests/test_serpapi_scholar.py

@@ -193,7 +193,9 @@ def test_wrong_pdf_is_rejected_and_next_copy_is_tried(monkeypatch, tmp_path):
                                       "doi:10.9999/wrong"] + ["Sedimentary rocks form in layers."] * 20))
         else:
             assert not path.exists(), "Rejected PDF must be removed"
-            path.write_bytes(text_pdf(doi=DOI))
+            # The correct copy prints its title, as a real article does: a
+            # DOI alone on legible pages is what a citing paper also prints.
+            path.write_bytes(text_pdf([TITLE, f"doi:{DOI}"] + ["Full text of the trial report."] * 250))
         return True
 
     monkeypatch.setattr(fpd, "try_download", download)