litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,130 @@
1
+ """Agent-ready markdown export: PDF in, clean markdown out.
2
+
3
+ The default deliverable stays the PDF (human-readable); this module adds an
4
+ optional AI reading layer on top of the downloaded file via pymupdf4llm.
5
+
6
+ Post-processing repairs two deterministic classes of pymupdf4llm artifacts
7
+ before returning the text:
8
+
9
+ - Decomposed accents rendered as literal marks, e.g. ``Daniel S<sup>ˇ</sup>uta``
10
+ for ``Daniel Šuta`` — re-folded into combining characters, then NFC-normalized.
11
+ - Unicode replacement characters (``�``, U+FFFD) from lost font glyphs, which
12
+ cannot be recovered; they are counted and reported as a quality warning.
13
+
14
+ ``markdown_warnings`` lets agents know the text is readable but names/symbols
15
+ may need spot-checking, without blocking the export.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ import unicodedata
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+ # Diacritic marks that pymupdf4llm sometimes wraps in <sup> when the PDF font
26
+ # decomposed them: <sup>ˇ</sup> -> U+030C (combining caron), etc. NFC then
27
+ # merges the combining char into the preceding letter (S + U+030C -> Š).
28
+ _SUP_DIACRITIC_RE = re.compile(r"<sup>\s*([ˇˆ´`¨¯˙˝˘˜])\s*</sup>")
29
+
30
+ _DIACRITIC_TO_COMBINING = {
31
+ "ˇ": "\u030c", # caron
32
+ "ˆ": "\u0302", # circumflex
33
+ "´": "\u0301", # acute
34
+ "`": "\u0300", # grave
35
+ "¨": "\u0308", # diaeresis
36
+ "¯": "\u0304", # macron
37
+ "˙": "\u0307", # dot above
38
+ "˝": "\u030b", # double acute
39
+ "˘": "\u0306", # breve
40
+ "˜": "\u0303", # tilde
41
+ }
42
+
43
+ # After folding, a stray space split the accent from its letter
44
+ # ("S<sup>ˇ</sup> uta" -> "S\u030c uta"); drop the single space so NFC merges.
45
+ _COMBINING_SPACE_RE = re.compile(r"([\u0300-\u036f]) +(?=[A-Za-z])")
46
+
47
+ _REPLACEMENT_CHAR_RE = re.compile(r"\ufffd")
48
+ _NONPRINTABLE_RE = re.compile(r"[^\x09\x0a\x0d\x20-\x7e\u00a0-\uffff]")
49
+
50
+
51
+ def clean_markdown_text(md_text: str) -> str:
52
+ """Repair deterministic pymupdf4llm artifacts (see module docstring)."""
53
+ text = _SUP_DIACRITIC_RE.sub(
54
+ lambda m: _DIACRITIC_TO_COMBINING.get(m.group(1), m.group(0)), md_text or ""
55
+ )
56
+ text = _COMBINING_SPACE_RE.sub(r"\1", text)
57
+ return unicodedata.normalize("NFC", text)
58
+
59
+
60
+ def markdown_quality_scan(md_text: str) -> list[str]:
61
+ """Return human-readable warnings about lost/odd characters in md_text."""
62
+ text = md_text or ""
63
+ warnings: list[str] = []
64
+ n_replacement = len(_REPLACEMENT_CHAR_RE.findall(text))
65
+ if n_replacement:
66
+ warnings.append(
67
+ f"{n_replacement} 个替换字符(�)——PDF 字库缺失的符号(如 ©)无法恢复,"
68
+ "名称/符号可能需人工核对"
69
+ )
70
+ if text:
71
+ nonprintable = sum(1 for c in text if _NONPRINTABLE_RE.match(c))
72
+ if nonprintable / max(len(text), 1) > 0.01:
73
+ warnings.append(
74
+ f"{nonprintable} 个不可打印字符(比例 {nonprintable / len(text):.1%})——"
75
+ "文本编码或字体映射异常"
76
+ )
77
+ return warnings
78
+
79
+
80
+ def pdf_to_markdown(pdf_path: str | Path, write: bool = True) -> str | Path:
81
+ """Convert a downloaded PDF to markdown alongside it.
82
+
83
+ Returns the written .md path (write=True) or the markdown text.
84
+ Raises a RuntimeError with an install hint when pymupdf4llm is missing.
85
+ """
86
+ md_text, _warnings = pdf_to_markdown_detailed(pdf_path, write=write)
87
+ return md_text
88
+
89
+
90
+ def pdf_to_markdown_detailed(pdf_path: str | Path, write: bool = True) -> tuple[str | Path, list[str]]:
91
+ """Convert a PDF to markdown; return (markdown, quality_warnings).
92
+
93
+ ``markdown_warnings`` are non-fatal — the text is readable but some
94
+ characters (lost font glyphs) may need spot-checking.
95
+ """
96
+ try:
97
+ import pymupdf4llm
98
+ except ImportError as exc: # pragma: no cover
99
+ raise RuntimeError(
100
+ "markdown 导出需要 pymupdf4llm:pip install pymupdf4llm"
101
+ ) from exc
102
+
103
+ src = Path(pdf_path)
104
+ warnings: list[str] = []
105
+ raw_md = _to_markdown(pymupdf4llm, src, warnings)
106
+ md_text = clean_markdown_text(raw_md)
107
+ warnings += markdown_quality_scan(md_text)
108
+ if not write:
109
+ return md_text, warnings
110
+ out = src.with_suffix(".md")
111
+ out.write_text(md_text, encoding="utf-8")
112
+ return out, warnings
113
+
114
+
115
+ def _to_markdown(pymupdf4llm: Any, src: Path, warnings: list[str]) -> str:
116
+ """pymupdf4llm's markdown, without the layout model when it is not installed.
117
+
118
+ pymupdf4llm 2.x routes ``to_markdown`` through pymupdf-layout, which needs
119
+ onnxruntime (and networkx): about 67 MB of the install. When that optional
120
+ part is missing — a trimmed install, or a download of it that failed — the
121
+ text-only path still reads the PDF, with tables and columns less tidy.
122
+ """
123
+ try:
124
+ return str(pymupdf4llm.to_markdown(str(src)))
125
+ except ModuleNotFoundError as exc:
126
+ if not (exc.name or "").startswith(("onnxruntime", "networkx")):
127
+ raise
128
+ pymupdf4llm.use_layout(False)
129
+ warnings.append("版面分析组件没有安装,按纯文本顺序转换(表格和分栏可能不规整)")
130
+ return str(pymupdf4llm.to_markdown(str(src)))
@@ -0,0 +1,454 @@
1
+ """PDF detection, validation, and download helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import time
7
+ import urllib.parse
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ import requests
12
+
13
+ from ..network import fetch
14
+
15
+ try:
16
+ from bs4 import BeautifulSoup
17
+ except Exception:
18
+ BeautifulSoup = None
19
+
20
+
21
+ def is_pdf_file(path: Path) -> bool:
22
+ try:
23
+ size = path.stat().st_size
24
+ if size < 1000:
25
+ return False
26
+ with path.open("rb") as fh:
27
+ header = fh.read(5)
28
+ if header != b"%PDF-":
29
+ return False
30
+ fh.seek(max(0, size - 1024))
31
+ tail = fh.read()
32
+ return b"%%EOF" in tail
33
+ except OSError:
34
+ return False
35
+
36
+
37
+ def is_plausible_pdf_url(url: str) -> bool:
38
+ if not url or not url.startswith(("http://", "https://")):
39
+ return False
40
+ parsed = urllib.parse.urlparse(url)
41
+ path = parsed.path.lower()
42
+ query = parsed.query.lower()
43
+ host = (parsed.hostname or "").lower()
44
+ combined = (path + "?" + query).lower()
45
+
46
+ reject_markers = ["/data-providers/", "/data-provider/", "/providers/", "/journals/", "/subjects/"]
47
+ if any(marker in combined for marker in reject_markers):
48
+ return False
49
+
50
+ if path.endswith(".pdf"):
51
+ return True
52
+ if "/pdf" in path or "download/pdf" in path:
53
+ return True
54
+ if "format=pdf" in query or "type=pdf" in query:
55
+ return True
56
+ # A download endpoint asking for a PDF variant: Taylor & Francis books,
57
+ # /content/books/mono/download?...&type=googlepdf
58
+ if "/download" in path and "pdf" in query:
59
+ return True
60
+ if ("hal.science" in host or "archives-ouvertes" in host) and path.endswith("/document"):
61
+ return True
62
+ return False
63
+
64
+
65
+ def _response_looks_pdf(resp: requests.Response, first_chunk: bytes) -> bool:
66
+ ctype = resp.headers.get("content-type", "").lower()
67
+ return first_chunk.startswith(b"%PDF-") or "application/pdf" in ctype
68
+
69
+
70
+ _SINGLE_PAGE_CACHE: dict[str, bool] = {}
71
+
72
+
73
+ def _article_is_single_page(doi: str) -> bool:
74
+ """True when Crossref says the article itself has one page (e.g. "1503")."""
75
+ if not doi or not doi.startswith("10."):
76
+ return False
77
+ if doi not in _SINGLE_PAGE_CACHE:
78
+ try:
79
+ s = requests.Session()
80
+ s.trust_env = False
81
+ r = s.get(f"https://api.crossref.org/works/{doi}", timeout=10)
82
+ page = str(r.json().get("message", {}).get("page", "") or "")
83
+ except Exception:
84
+ return False # unknown: don't cache, keep the strict heuristic
85
+ # Only an explicit range proves it ("1503-1503"): a lone number is often
86
+ # an article number (Elsevier "117511"), saying nothing about length.
87
+ start, _, end = page.partition("-")
88
+ _SINGLE_PAGE_CACHE[doi] = bool(end) and start.strip() == end.strip()
89
+ return _SINGLE_PAGE_CACHE[doi]
90
+
91
+
92
+ def is_suspicious_pdf(path: Path, doi: str = "") -> bool:
93
+ """Check if a PDF looks like a cover page or preview (not full text).
94
+
95
+ Heuristics:
96
+ - Very small file (< 50 KB): likely a 1-page cover
97
+ - At most 1 readable page: preview/cover page. File size alone is NOT
98
+ trusted — Elsevier serves >100KB single-page previews to API keys
99
+ without full-text entitlement. Unless Crossref says the article
100
+ itself is one page (comments, News & Views): then it is the full text.
101
+ """
102
+ try:
103
+ size = path.stat().st_size
104
+ # Very small files are suspicious regardless
105
+ if size < 50_000:
106
+ return True
107
+ try:
108
+ import fitz # type: ignore[import-not-found]
109
+ except Exception:
110
+ # No pymupdf: fall back to the cheap regex heuristic
111
+ with path.open("rb") as fh:
112
+ content = fh.read(512_000)
113
+ # Count PDF page objects: look for "/Type /Page" not followed by "s"
114
+ import re
115
+ pages = len(re.findall(rb"/Type\s*/Page\b", content))
116
+ return pages <= 1 and not _article_is_single_page(doi)
117
+ try:
118
+ with fitz.open(path) as doc:
119
+ single = int(doc.page_count) <= 1
120
+ except Exception:
121
+ return True # unreadable PDF — treat as suspicious
122
+ return single and not _article_is_single_page(doi)
123
+ except OSError:
124
+ return False
125
+
126
+
127
+ def _first_pages_text(path: Path, pages: int = 3) -> str:
128
+ try:
129
+ import pymupdf
130
+ with pymupdf.open(path) as pdf:
131
+ return "".join(pdf[i].get_text() for i in range(min(pages, pdf.page_count)))
132
+ except Exception:
133
+ return ""
134
+
135
+
136
+ def doi_printed(path: Path, doi: str, pages: int = 3) -> bool:
137
+ """True when the DOI appears in the first pages' text (publisher PDFs print it)."""
138
+ return bool(doi) and doi.lower() in "".join(_first_pages_text(path, pages).lower().split())
139
+
140
+
141
+ _TITLE_CACHE: dict[str, str] = {}
142
+ _TITLE_SHARE = 0.6 # share of the title's words that must be on the first pages (own value)
143
+
144
+
145
+ def _title_of(doi: str, config: dict[str, Any] | None) -> str:
146
+ """The paper's title from Crossref, asked once per DOI ("" when unknown)."""
147
+ if doi not in _TITLE_CACHE:
148
+ from ..citations.formats import fetch_metadata
149
+ meta = fetch_metadata(doi, config or {}) or {}
150
+ titles = meta.get("title") or []
151
+ _TITLE_CACHE[doi] = str(titles[0]) if titles else ""
152
+ return _TITLE_CACHE[doi]
153
+
154
+
155
+ # Preprint servers, in the order their marks are looked for. The DOI prefixes
156
+ # are the ones a preprint PDF prints in its own header (Research Square's
157
+ # 10.21203/rs.3.rs-... on 2026-10-26), not the DOI the article was asked for.
158
+ _PREPRINT_DOI_PREFIXES: tuple[tuple[str, str], ...] = (
159
+ ("10.21203", "Research Square"),
160
+ ("10.48550", "arXiv"),
161
+ ("10.1101", "bioRxiv"),
162
+ ("10.26434", "ChemRxiv"),
163
+ ("10.2139", "SSRN"),
164
+ ("10.36227", "TechRxiv"),
165
+ ("10.31219", "OSF Preprints"),
166
+ ("10.20944", "Preprints.org"),
167
+ ("10.22541", "Authorea"),
168
+ )
169
+ _PREPRINT_NAMES: tuple[tuple[str, str], ...] = (
170
+ ("research square", "Research Square"),
171
+ ("arxiv", "arXiv"),
172
+ ("biorxiv", "bioRxiv"),
173
+ ("medrxiv", "medRxiv"),
174
+ ("chemrxiv", "ChemRxiv"),
175
+ ("techrxiv", "TechRxiv"),
176
+ ("ssrn", "SSRN"),
177
+ ("preprints.org", "Preprints.org"),
178
+ ("authorea", "Authorea"),
179
+ ("osf preprints", "OSF Preprints"),
180
+ )
181
+ # Phrases only a preprint prints (Research Square: "Posted Date:", "Additional
182
+ # Declarations:"; others say they are not peer reviewed). "Preprint submitted
183
+ # to Elsevier" is an author manuscript, also not the version of record.
184
+ _PREPRINT_PHRASES: tuple[str, ...] = (
185
+ "posted date:",
186
+ "additional declarations:",
187
+ "this is a preprint",
188
+ "has not been peer reviewed",
189
+ "not been peer-reviewed",
190
+ "preprint submitted to",
191
+ )
192
+ _PREPRINT_CACHE: dict[tuple[str, float], str] = {}
193
+
194
+
195
+ def preprint_mark(path: Path, doi: str = "") -> str:
196
+ """The preprint server a PDF came from, or "" for a version of record.
197
+
198
+ Read from the first page's header: the preprint server's own DOI prefix,
199
+ its name, or a phrase only a preprint prints (2026-10-26: for
200
+ 10.1038/s41598-024-67653-y the open-access route served the Research
201
+ Square preprint, which title-matching cannot tell from the published
202
+ version). A PDF that prints ``doi`` itself is the version of record.
203
+ """
204
+ try:
205
+ stat = path.stat()
206
+ except OSError:
207
+ return ""
208
+ key = (str(path), stat.st_mtime)
209
+ if key in _PREPRINT_CACHE:
210
+ return _PREPRINT_CACHE[key]
211
+ text = _first_pages_text(path, 1)
212
+ mark = ""
213
+ if len(text.strip()) >= 100 and not (doi and doi.lower() in "".join(text.lower().split())):
214
+ head = text[:1500].lower()
215
+ squash = "".join(head.split())
216
+ for prefix, name in _PREPRINT_DOI_PREFIXES:
217
+ if prefix in squash:
218
+ mark = name
219
+ break
220
+ # The platform's name or a preprint phrase counts only near the top of
221
+ # the page: a published paper's references can name arXiv or SSRN too.
222
+ top = " ".join(text[:800].lower().split())
223
+ if not mark:
224
+ for needle, name in _PREPRINT_NAMES:
225
+ if needle in top:
226
+ mark = name
227
+ break
228
+ if not mark:
229
+ for phrase in _PREPRINT_PHRASES:
230
+ if phrase in top:
231
+ mark = "preprint"
232
+ break
233
+ _PREPRINT_CACHE[key] = mark
234
+ return mark
235
+
236
+
237
+ def paper_check(path: Path, doi: str, config: dict[str, Any] | None = None) -> str:
238
+ """Is this PDF the paper ``doi`` names?
239
+
240
+ "match": the DOI is printed in the first pages (57 of 61 downloaded papers,
241
+ 2026-10-26), or, when it is not, most of the title's words are there.
242
+ "wrong": neither, and the title was there to compare (Sci-Hub served a JSTOR
243
+ essay for a Ceramics International DOI, 2026-10-28). "unknown": not a DOI,
244
+ no text to read (a scan without OCR) or no title to compare; accepted, since
245
+ the check must not block a download it cannot judge.
246
+ """
247
+ if not doi or not doi.startswith("10."):
248
+ return "unknown"
249
+ text = _first_pages_text(path)
250
+ if len(text.strip()) < 200:
251
+ return "unknown"
252
+ if doi.lower() in "".join(text.lower().split()):
253
+ return "match"
254
+ from ..terms.text import STOPWORDS, plain
255
+ words = {w for w in re.findall(r"[a-z0-9]+", plain(_title_of(doi, config)))
256
+ if len(w) >= 4 and w not in STOPWORDS}
257
+ if not words:
258
+ return "unknown"
259
+ page = plain(text)
260
+ found = sum(1 for w in words if w in page)
261
+ return "match" if found / len(words) >= _TITLE_SHARE else "wrong"
262
+
263
+
264
+ def wrong_paper(identifier: str, file_path: Path, source_label: str) -> dict[str, Any]:
265
+ """Build a result dict for a PDF that is a different paper."""
266
+ return {
267
+ "success": False,
268
+ "identifier": identifier,
269
+ "doi": identifier,
270
+ "file": str(file_path),
271
+ "source": source_label,
272
+ "error_type": "wrong_paper",
273
+ "reason": "PDF is a different paper (neither its DOI nor its title is on the first pages)",
274
+ }
275
+
276
+
277
+ def suspicious_pdf(identifier: str, file_path: Path, source_label: str) -> dict[str, Any]:
278
+ """Build a result dict for a suspicious/preview PDF (not full text)."""
279
+ return {
280
+ "success": False,
281
+ "identifier": identifier,
282
+ "doi": identifier,
283
+ "file": str(file_path),
284
+ "source": source_label,
285
+ "error_type": "suspicious_pdf",
286
+ "reason": "PDF appears to be a cover page or preview (too small / too few pages)",
287
+ }
288
+
289
+
290
+ def success(identifier: str, file_path: Path, source: str) -> dict[str, Any]:
291
+ size_kb = round(file_path.stat().st_size / 1024, 1)
292
+ return {
293
+ "success": True,
294
+ "identifier": identifier,
295
+ "doi": identifier,
296
+ "file": str(file_path),
297
+ "size_kb": size_kb,
298
+ "source": source,
299
+ }
300
+
301
+
302
+ def fail(
303
+ identifier: str,
304
+ reason: str = "not found",
305
+ extra: dict[str, Any] | None = None,
306
+ *,
307
+ error_type: str = "",
308
+ action: str = "",
309
+ ) -> dict[str, Any]:
310
+ result = {"success": False, "identifier": identifier, "doi": identifier, "reason": reason}
311
+ if error_type:
312
+ result["error_type"] = error_type
313
+ if action:
314
+ result["action"] = action
315
+ if extra:
316
+ result.update(extra)
317
+ return result
318
+
319
+
320
+ def dedupe(items: Any) -> list[str]:
321
+ seen: set[str] = set()
322
+ out: list[str] = []
323
+ for item in items:
324
+ if not item:
325
+ continue
326
+ item = item.strip() if isinstance(item, str) else str(item)
327
+ if item and item not in seen:
328
+ seen.add(item)
329
+ out.append(item)
330
+ return out
331
+
332
+
333
+ def iter_urls(obj: Any) -> Any:
334
+ if isinstance(obj, dict):
335
+ for key, value in obj.items():
336
+ if isinstance(value, str) and ("url" in key.lower() or value.startswith("http")):
337
+ yield value
338
+ else:
339
+ yield from iter_urls(value)
340
+ elif isinstance(obj, list):
341
+ for item in obj:
342
+ yield from iter_urls(item)
343
+
344
+
345
+ def extract_pdf_url_from_html(html: str, base_url: str) -> str | None:
346
+ urls: list[str] = []
347
+ for match in re.finditer(
348
+ r"""<meta[^>]+name=["']citation_pdf_url["'][^>]+content=["']([^"']+)["']""", html, re.I
349
+ ):
350
+ urls.append(urllib.parse.urljoin(base_url, match.group(1)))
351
+ for match in re.finditer(
352
+ r"""<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_pdf_url["']""", html, re.I
353
+ ):
354
+ urls.append(urllib.parse.urljoin(base_url, match.group(1)))
355
+ if BeautifulSoup is not None:
356
+ soup = BeautifulSoup(html, "html.parser")
357
+ for tag in soup.find_all(["iframe", "embed", "a"]):
358
+ candidate = tag.get("src") or tag.get("href")
359
+ if candidate:
360
+ urls.append(urllib.parse.urljoin(base_url, candidate))
361
+ else:
362
+ for match in re.finditer(r"""(?:src|href)=["']([^"']+)["']""", html, re.I):
363
+ urls.append(urllib.parse.urljoin(base_url, match.group(1)))
364
+
365
+ for url in dedupe(urls):
366
+ if is_plausible_pdf_url(url):
367
+ return url
368
+ return None
369
+
370
+
371
+ def download_pdf(
372
+ url: str,
373
+ output_path: Path,
374
+ config: dict[str, Any],
375
+ source: str,
376
+ *,
377
+ require_pdf_like_url: bool = True,
378
+ use_tor: bool = False,
379
+ cookies: Any = None,
380
+ referer: str = "",
381
+ browser_ua: bool = False,
382
+ ) -> dict[str, Any] | None:
383
+ if require_pdf_like_url and not is_plausible_pdf_url(url):
384
+ return None
385
+
386
+ try:
387
+ if cookies is not None:
388
+ from ..network import (
389
+ CHROME_UA,
390
+ request_timeout,
391
+ proxy_dict,
392
+ select_proxy_for_url,
393
+ USER_AGENT,
394
+ )
395
+ session = requests.Session()
396
+ session.trust_env = False
397
+ session.headers.update({"User-Agent": USER_AGENT})
398
+ if referer:
399
+ # Some shadow-library CDNs (sci.bban.top, observed 2026-09)
400
+ # 403 any PDF request without a same-site Referer.
401
+ session.headers.update({"Referer": referer})
402
+ if browser_ua:
403
+ # …and they also 403 non-browser User-Agents outright — the
404
+ # engine UA ("litloop/x") gets HTML even with Referer.
405
+ session.headers.update({"User-Agent": CHROME_UA})
406
+ session.cookies.update(cookies)
407
+ resp = session.get(
408
+ url,
409
+ timeout=request_timeout(config),
410
+ proxies=proxy_dict(select_proxy_for_url(url, config)),
411
+ allow_redirects=True,
412
+ stream=True,
413
+ )
414
+ else:
415
+ resp = fetch(url, config, stream=True, use_tor=use_tor)
416
+ if resp.status_code >= 400:
417
+ return None
418
+
419
+ iterator = resp.iter_content(chunk_size=8192)
420
+ first_chunk = next(iterator, b"")
421
+ if not _response_looks_pdf(resp, first_chunk):
422
+ return None
423
+
424
+ output_path.parent.mkdir(parents=True, exist_ok=True)
425
+ tmp_path = output_path.with_suffix(output_path.suffix + ".part")
426
+ try:
427
+ # requests' timeout tuple only guards connect + each socket read;
428
+ # a peer that drips bytes forever would hold the race open, so
429
+ # bound the whole stream with a wall-clock deadline too.
430
+ deadline = time.monotonic() + float(config.get("download_deadline_seconds", 60))
431
+ with tmp_path.open("wb") as fh:
432
+ fh.write(first_chunk)
433
+ fh.flush()
434
+ for chunk in iterator:
435
+ if time.monotonic() > deadline:
436
+ raise TimeoutError("stream deadline exceeded")
437
+ if chunk:
438
+ fh.write(chunk)
439
+ fh.flush()
440
+ tmp_path.replace(output_path)
441
+ except Exception:
442
+ tmp_path.unlink(missing_ok=True)
443
+ raise
444
+
445
+ if is_pdf_file(output_path):
446
+ return success(output_path.stem, output_path, source)
447
+ else:
448
+ try:
449
+ output_path.unlink(missing_ok=True)
450
+ except OSError:
451
+ pass
452
+ except Exception:
453
+ return None
454
+ return None