litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Agent-ready markdown export: PDF in, clean markdown out.
|
|
2
|
+
|
|
3
|
+
The default deliverable stays the PDF (human-readable); this module adds an
|
|
4
|
+
optional AI reading layer on top of the downloaded file via pymupdf4llm.
|
|
5
|
+
|
|
6
|
+
Post-processing repairs two deterministic classes of pymupdf4llm artifacts
|
|
7
|
+
before returning the text:
|
|
8
|
+
|
|
9
|
+
- Decomposed accents rendered as literal marks, e.g. ``Daniel S<sup>ˇ</sup>uta``
|
|
10
|
+
for ``Daniel Šuta`` — re-folded into combining characters, then NFC-normalized.
|
|
11
|
+
- Unicode replacement characters (``�``, U+FFFD) from lost font glyphs, which
|
|
12
|
+
cannot be recovered; they are counted and reported as a quality warning.
|
|
13
|
+
|
|
14
|
+
``markdown_warnings`` lets agents know the text is readable but names/symbols
|
|
15
|
+
may need spot-checking, without blocking the export.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
import unicodedata
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
# Diacritic marks that pymupdf4llm sometimes wraps in <sup> when the PDF font
|
|
26
|
+
# decomposed them: <sup>ˇ</sup> -> U+030C (combining caron), etc. NFC then
|
|
27
|
+
# merges the combining char into the preceding letter (S + U+030C -> Š).
|
|
28
|
+
_SUP_DIACRITIC_RE = re.compile(r"<sup>\s*([ˇˆ´`¨¯˙˝˘˜])\s*</sup>")
|
|
29
|
+
|
|
30
|
+
_DIACRITIC_TO_COMBINING = {
|
|
31
|
+
"ˇ": "\u030c", # caron
|
|
32
|
+
"ˆ": "\u0302", # circumflex
|
|
33
|
+
"´": "\u0301", # acute
|
|
34
|
+
"`": "\u0300", # grave
|
|
35
|
+
"¨": "\u0308", # diaeresis
|
|
36
|
+
"¯": "\u0304", # macron
|
|
37
|
+
"˙": "\u0307", # dot above
|
|
38
|
+
"˝": "\u030b", # double acute
|
|
39
|
+
"˘": "\u0306", # breve
|
|
40
|
+
"˜": "\u0303", # tilde
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# After folding, a stray space split the accent from its letter
|
|
44
|
+
# ("S<sup>ˇ</sup> uta" -> "S\u030c uta"); drop the single space so NFC merges.
|
|
45
|
+
_COMBINING_SPACE_RE = re.compile(r"([\u0300-\u036f]) +(?=[A-Za-z])")
|
|
46
|
+
|
|
47
|
+
_REPLACEMENT_CHAR_RE = re.compile(r"\ufffd")
|
|
48
|
+
_NONPRINTABLE_RE = re.compile(r"[^\x09\x0a\x0d\x20-\x7e\u00a0-\uffff]")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def clean_markdown_text(md_text: str) -> str:
|
|
52
|
+
"""Repair deterministic pymupdf4llm artifacts (see module docstring)."""
|
|
53
|
+
text = _SUP_DIACRITIC_RE.sub(
|
|
54
|
+
lambda m: _DIACRITIC_TO_COMBINING.get(m.group(1), m.group(0)), md_text or ""
|
|
55
|
+
)
|
|
56
|
+
text = _COMBINING_SPACE_RE.sub(r"\1", text)
|
|
57
|
+
return unicodedata.normalize("NFC", text)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def markdown_quality_scan(md_text: str) -> list[str]:
|
|
61
|
+
"""Return human-readable warnings about lost/odd characters in md_text."""
|
|
62
|
+
text = md_text or ""
|
|
63
|
+
warnings: list[str] = []
|
|
64
|
+
n_replacement = len(_REPLACEMENT_CHAR_RE.findall(text))
|
|
65
|
+
if n_replacement:
|
|
66
|
+
warnings.append(
|
|
67
|
+
f"{n_replacement} 个替换字符(�)——PDF 字库缺失的符号(如 ©)无法恢复,"
|
|
68
|
+
"名称/符号可能需人工核对"
|
|
69
|
+
)
|
|
70
|
+
if text:
|
|
71
|
+
nonprintable = sum(1 for c in text if _NONPRINTABLE_RE.match(c))
|
|
72
|
+
if nonprintable / max(len(text), 1) > 0.01:
|
|
73
|
+
warnings.append(
|
|
74
|
+
f"{nonprintable} 个不可打印字符(比例 {nonprintable / len(text):.1%})——"
|
|
75
|
+
"文本编码或字体映射异常"
|
|
76
|
+
)
|
|
77
|
+
return warnings
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def pdf_to_markdown(pdf_path: str | Path, write: bool = True) -> str | Path:
|
|
81
|
+
"""Convert a downloaded PDF to markdown alongside it.
|
|
82
|
+
|
|
83
|
+
Returns the written .md path (write=True) or the markdown text.
|
|
84
|
+
Raises a RuntimeError with an install hint when pymupdf4llm is missing.
|
|
85
|
+
"""
|
|
86
|
+
md_text, _warnings = pdf_to_markdown_detailed(pdf_path, write=write)
|
|
87
|
+
return md_text
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def pdf_to_markdown_detailed(pdf_path: str | Path, write: bool = True) -> tuple[str | Path, list[str]]:
|
|
91
|
+
"""Convert a PDF to markdown; return (markdown, quality_warnings).
|
|
92
|
+
|
|
93
|
+
``markdown_warnings`` are non-fatal — the text is readable but some
|
|
94
|
+
characters (lost font glyphs) may need spot-checking.
|
|
95
|
+
"""
|
|
96
|
+
try:
|
|
97
|
+
import pymupdf4llm
|
|
98
|
+
except ImportError as exc: # pragma: no cover
|
|
99
|
+
raise RuntimeError(
|
|
100
|
+
"markdown 导出需要 pymupdf4llm:pip install pymupdf4llm"
|
|
101
|
+
) from exc
|
|
102
|
+
|
|
103
|
+
src = Path(pdf_path)
|
|
104
|
+
warnings: list[str] = []
|
|
105
|
+
raw_md = _to_markdown(pymupdf4llm, src, warnings)
|
|
106
|
+
md_text = clean_markdown_text(raw_md)
|
|
107
|
+
warnings += markdown_quality_scan(md_text)
|
|
108
|
+
if not write:
|
|
109
|
+
return md_text, warnings
|
|
110
|
+
out = src.with_suffix(".md")
|
|
111
|
+
out.write_text(md_text, encoding="utf-8")
|
|
112
|
+
return out, warnings
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _to_markdown(pymupdf4llm: Any, src: Path, warnings: list[str]) -> str:
|
|
116
|
+
"""pymupdf4llm's markdown, without the layout model when it is not installed.
|
|
117
|
+
|
|
118
|
+
pymupdf4llm 2.x routes ``to_markdown`` through pymupdf-layout, which needs
|
|
119
|
+
onnxruntime (and networkx): about 67 MB of the install. When that optional
|
|
120
|
+
part is missing — a trimmed install, or a download of it that failed — the
|
|
121
|
+
text-only path still reads the PDF, with tables and columns less tidy.
|
|
122
|
+
"""
|
|
123
|
+
try:
|
|
124
|
+
return str(pymupdf4llm.to_markdown(str(src)))
|
|
125
|
+
except ModuleNotFoundError as exc:
|
|
126
|
+
if not (exc.name or "").startswith(("onnxruntime", "networkx")):
|
|
127
|
+
raise
|
|
128
|
+
pymupdf4llm.use_layout(False)
|
|
129
|
+
warnings.append("版面分析组件没有安装,按纯文本顺序转换(表格和分栏可能不规整)")
|
|
130
|
+
return str(pymupdf4llm.to_markdown(str(src)))
|
litloop/download/pdf.py
ADDED
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
"""PDF detection, validation, and download helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
import urllib.parse
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
|
|
13
|
+
from ..network import fetch
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
from bs4 import BeautifulSoup
|
|
17
|
+
except Exception:
|
|
18
|
+
BeautifulSoup = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def is_pdf_file(path: Path) -> bool:
|
|
22
|
+
try:
|
|
23
|
+
size = path.stat().st_size
|
|
24
|
+
if size < 1000:
|
|
25
|
+
return False
|
|
26
|
+
with path.open("rb") as fh:
|
|
27
|
+
header = fh.read(5)
|
|
28
|
+
if header != b"%PDF-":
|
|
29
|
+
return False
|
|
30
|
+
fh.seek(max(0, size - 1024))
|
|
31
|
+
tail = fh.read()
|
|
32
|
+
return b"%%EOF" in tail
|
|
33
|
+
except OSError:
|
|
34
|
+
return False
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def is_plausible_pdf_url(url: str) -> bool:
|
|
38
|
+
if not url or not url.startswith(("http://", "https://")):
|
|
39
|
+
return False
|
|
40
|
+
parsed = urllib.parse.urlparse(url)
|
|
41
|
+
path = parsed.path.lower()
|
|
42
|
+
query = parsed.query.lower()
|
|
43
|
+
host = (parsed.hostname or "").lower()
|
|
44
|
+
combined = (path + "?" + query).lower()
|
|
45
|
+
|
|
46
|
+
reject_markers = ["/data-providers/", "/data-provider/", "/providers/", "/journals/", "/subjects/"]
|
|
47
|
+
if any(marker in combined for marker in reject_markers):
|
|
48
|
+
return False
|
|
49
|
+
|
|
50
|
+
if path.endswith(".pdf"):
|
|
51
|
+
return True
|
|
52
|
+
if "/pdf" in path or "download/pdf" in path:
|
|
53
|
+
return True
|
|
54
|
+
if "format=pdf" in query or "type=pdf" in query:
|
|
55
|
+
return True
|
|
56
|
+
# A download endpoint asking for a PDF variant: Taylor & Francis books,
|
|
57
|
+
# /content/books/mono/download?...&type=googlepdf
|
|
58
|
+
if "/download" in path and "pdf" in query:
|
|
59
|
+
return True
|
|
60
|
+
if ("hal.science" in host or "archives-ouvertes" in host) and path.endswith("/document"):
|
|
61
|
+
return True
|
|
62
|
+
return False
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _response_looks_pdf(resp: requests.Response, first_chunk: bytes) -> bool:
|
|
66
|
+
ctype = resp.headers.get("content-type", "").lower()
|
|
67
|
+
return first_chunk.startswith(b"%PDF-") or "application/pdf" in ctype
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
_SINGLE_PAGE_CACHE: dict[str, bool] = {}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _article_is_single_page(doi: str) -> bool:
|
|
74
|
+
"""True when Crossref says the article itself has one page (e.g. "1503")."""
|
|
75
|
+
if not doi or not doi.startswith("10."):
|
|
76
|
+
return False
|
|
77
|
+
if doi not in _SINGLE_PAGE_CACHE:
|
|
78
|
+
try:
|
|
79
|
+
s = requests.Session()
|
|
80
|
+
s.trust_env = False
|
|
81
|
+
r = s.get(f"https://api.crossref.org/works/{doi}", timeout=10)
|
|
82
|
+
page = str(r.json().get("message", {}).get("page", "") or "")
|
|
83
|
+
except Exception:
|
|
84
|
+
return False # unknown: don't cache, keep the strict heuristic
|
|
85
|
+
# Only an explicit range proves it ("1503-1503"): a lone number is often
|
|
86
|
+
# an article number (Elsevier "117511"), saying nothing about length.
|
|
87
|
+
start, _, end = page.partition("-")
|
|
88
|
+
_SINGLE_PAGE_CACHE[doi] = bool(end) and start.strip() == end.strip()
|
|
89
|
+
return _SINGLE_PAGE_CACHE[doi]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def is_suspicious_pdf(path: Path, doi: str = "") -> bool:
|
|
93
|
+
"""Check if a PDF looks like a cover page or preview (not full text).
|
|
94
|
+
|
|
95
|
+
Heuristics:
|
|
96
|
+
- Very small file (< 50 KB): likely a 1-page cover
|
|
97
|
+
- At most 1 readable page: preview/cover page. File size alone is NOT
|
|
98
|
+
trusted — Elsevier serves >100KB single-page previews to API keys
|
|
99
|
+
without full-text entitlement. Unless Crossref says the article
|
|
100
|
+
itself is one page (comments, News & Views): then it is the full text.
|
|
101
|
+
"""
|
|
102
|
+
try:
|
|
103
|
+
size = path.stat().st_size
|
|
104
|
+
# Very small files are suspicious regardless
|
|
105
|
+
if size < 50_000:
|
|
106
|
+
return True
|
|
107
|
+
try:
|
|
108
|
+
import fitz # type: ignore[import-not-found]
|
|
109
|
+
except Exception:
|
|
110
|
+
# No pymupdf: fall back to the cheap regex heuristic
|
|
111
|
+
with path.open("rb") as fh:
|
|
112
|
+
content = fh.read(512_000)
|
|
113
|
+
# Count PDF page objects: look for "/Type /Page" not followed by "s"
|
|
114
|
+
import re
|
|
115
|
+
pages = len(re.findall(rb"/Type\s*/Page\b", content))
|
|
116
|
+
return pages <= 1 and not _article_is_single_page(doi)
|
|
117
|
+
try:
|
|
118
|
+
with fitz.open(path) as doc:
|
|
119
|
+
single = int(doc.page_count) <= 1
|
|
120
|
+
except Exception:
|
|
121
|
+
return True # unreadable PDF — treat as suspicious
|
|
122
|
+
return single and not _article_is_single_page(doi)
|
|
123
|
+
except OSError:
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _first_pages_text(path: Path, pages: int = 3) -> str:
|
|
128
|
+
try:
|
|
129
|
+
import pymupdf
|
|
130
|
+
with pymupdf.open(path) as pdf:
|
|
131
|
+
return "".join(pdf[i].get_text() for i in range(min(pages, pdf.page_count)))
|
|
132
|
+
except Exception:
|
|
133
|
+
return ""
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def doi_printed(path: Path, doi: str, pages: int = 3) -> bool:
|
|
137
|
+
"""True when the DOI appears in the first pages' text (publisher PDFs print it)."""
|
|
138
|
+
return bool(doi) and doi.lower() in "".join(_first_pages_text(path, pages).lower().split())
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
_TITLE_CACHE: dict[str, str] = {}
|
|
142
|
+
_TITLE_SHARE = 0.6 # share of the title's words that must be on the first pages (own value)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _title_of(doi: str, config: dict[str, Any] | None) -> str:
|
|
146
|
+
"""The paper's title from Crossref, asked once per DOI ("" when unknown)."""
|
|
147
|
+
if doi not in _TITLE_CACHE:
|
|
148
|
+
from ..citations.formats import fetch_metadata
|
|
149
|
+
meta = fetch_metadata(doi, config or {}) or {}
|
|
150
|
+
titles = meta.get("title") or []
|
|
151
|
+
_TITLE_CACHE[doi] = str(titles[0]) if titles else ""
|
|
152
|
+
return _TITLE_CACHE[doi]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# Preprint servers, in the order their marks are looked for. The DOI prefixes
|
|
156
|
+
# are the ones a preprint PDF prints in its own header (Research Square's
|
|
157
|
+
# 10.21203/rs.3.rs-... on 2026-10-26), not the DOI the article was asked for.
|
|
158
|
+
_PREPRINT_DOI_PREFIXES: tuple[tuple[str, str], ...] = (
|
|
159
|
+
("10.21203", "Research Square"),
|
|
160
|
+
("10.48550", "arXiv"),
|
|
161
|
+
("10.1101", "bioRxiv"),
|
|
162
|
+
("10.26434", "ChemRxiv"),
|
|
163
|
+
("10.2139", "SSRN"),
|
|
164
|
+
("10.36227", "TechRxiv"),
|
|
165
|
+
("10.31219", "OSF Preprints"),
|
|
166
|
+
("10.20944", "Preprints.org"),
|
|
167
|
+
("10.22541", "Authorea"),
|
|
168
|
+
)
|
|
169
|
+
_PREPRINT_NAMES: tuple[tuple[str, str], ...] = (
|
|
170
|
+
("research square", "Research Square"),
|
|
171
|
+
("arxiv", "arXiv"),
|
|
172
|
+
("biorxiv", "bioRxiv"),
|
|
173
|
+
("medrxiv", "medRxiv"),
|
|
174
|
+
("chemrxiv", "ChemRxiv"),
|
|
175
|
+
("techrxiv", "TechRxiv"),
|
|
176
|
+
("ssrn", "SSRN"),
|
|
177
|
+
("preprints.org", "Preprints.org"),
|
|
178
|
+
("authorea", "Authorea"),
|
|
179
|
+
("osf preprints", "OSF Preprints"),
|
|
180
|
+
)
|
|
181
|
+
# Phrases only a preprint prints (Research Square: "Posted Date:", "Additional
|
|
182
|
+
# Declarations:"; others say they are not peer reviewed). "Preprint submitted
|
|
183
|
+
# to Elsevier" is an author manuscript, also not the version of record.
|
|
184
|
+
_PREPRINT_PHRASES: tuple[str, ...] = (
|
|
185
|
+
"posted date:",
|
|
186
|
+
"additional declarations:",
|
|
187
|
+
"this is a preprint",
|
|
188
|
+
"has not been peer reviewed",
|
|
189
|
+
"not been peer-reviewed",
|
|
190
|
+
"preprint submitted to",
|
|
191
|
+
)
|
|
192
|
+
_PREPRINT_CACHE: dict[tuple[str, float], str] = {}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def preprint_mark(path: Path, doi: str = "") -> str:
|
|
196
|
+
"""The preprint server a PDF came from, or "" for a version of record.
|
|
197
|
+
|
|
198
|
+
Read from the first page's header: the preprint server's own DOI prefix,
|
|
199
|
+
its name, or a phrase only a preprint prints (2026-10-26: for
|
|
200
|
+
10.1038/s41598-024-67653-y the open-access route served the Research
|
|
201
|
+
Square preprint, which title-matching cannot tell from the published
|
|
202
|
+
version). A PDF that prints ``doi`` itself is the version of record.
|
|
203
|
+
"""
|
|
204
|
+
try:
|
|
205
|
+
stat = path.stat()
|
|
206
|
+
except OSError:
|
|
207
|
+
return ""
|
|
208
|
+
key = (str(path), stat.st_mtime)
|
|
209
|
+
if key in _PREPRINT_CACHE:
|
|
210
|
+
return _PREPRINT_CACHE[key]
|
|
211
|
+
text = _first_pages_text(path, 1)
|
|
212
|
+
mark = ""
|
|
213
|
+
if len(text.strip()) >= 100 and not (doi and doi.lower() in "".join(text.lower().split())):
|
|
214
|
+
head = text[:1500].lower()
|
|
215
|
+
squash = "".join(head.split())
|
|
216
|
+
for prefix, name in _PREPRINT_DOI_PREFIXES:
|
|
217
|
+
if prefix in squash:
|
|
218
|
+
mark = name
|
|
219
|
+
break
|
|
220
|
+
# The platform's name or a preprint phrase counts only near the top of
|
|
221
|
+
# the page: a published paper's references can name arXiv or SSRN too.
|
|
222
|
+
top = " ".join(text[:800].lower().split())
|
|
223
|
+
if not mark:
|
|
224
|
+
for needle, name in _PREPRINT_NAMES:
|
|
225
|
+
if needle in top:
|
|
226
|
+
mark = name
|
|
227
|
+
break
|
|
228
|
+
if not mark:
|
|
229
|
+
for phrase in _PREPRINT_PHRASES:
|
|
230
|
+
if phrase in top:
|
|
231
|
+
mark = "preprint"
|
|
232
|
+
break
|
|
233
|
+
_PREPRINT_CACHE[key] = mark
|
|
234
|
+
return mark
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def paper_check(path: Path, doi: str, config: dict[str, Any] | None = None) -> str:
|
|
238
|
+
"""Is this PDF the paper ``doi`` names?
|
|
239
|
+
|
|
240
|
+
"match": the DOI is printed in the first pages (57 of 61 downloaded papers,
|
|
241
|
+
2026-10-26), or, when it is not, most of the title's words are there.
|
|
242
|
+
"wrong": neither, and the title was there to compare (Sci-Hub served a JSTOR
|
|
243
|
+
essay for a Ceramics International DOI, 2026-10-28). "unknown": not a DOI,
|
|
244
|
+
no text to read (a scan without OCR) or no title to compare; accepted, since
|
|
245
|
+
the check must not block a download it cannot judge.
|
|
246
|
+
"""
|
|
247
|
+
if not doi or not doi.startswith("10."):
|
|
248
|
+
return "unknown"
|
|
249
|
+
text = _first_pages_text(path)
|
|
250
|
+
if len(text.strip()) < 200:
|
|
251
|
+
return "unknown"
|
|
252
|
+
if doi.lower() in "".join(text.lower().split()):
|
|
253
|
+
return "match"
|
|
254
|
+
from ..terms.text import STOPWORDS, plain
|
|
255
|
+
words = {w for w in re.findall(r"[a-z0-9]+", plain(_title_of(doi, config)))
|
|
256
|
+
if len(w) >= 4 and w not in STOPWORDS}
|
|
257
|
+
if not words:
|
|
258
|
+
return "unknown"
|
|
259
|
+
page = plain(text)
|
|
260
|
+
found = sum(1 for w in words if w in page)
|
|
261
|
+
return "match" if found / len(words) >= _TITLE_SHARE else "wrong"
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def wrong_paper(identifier: str, file_path: Path, source_label: str) -> dict[str, Any]:
|
|
265
|
+
"""Build a result dict for a PDF that is a different paper."""
|
|
266
|
+
return {
|
|
267
|
+
"success": False,
|
|
268
|
+
"identifier": identifier,
|
|
269
|
+
"doi": identifier,
|
|
270
|
+
"file": str(file_path),
|
|
271
|
+
"source": source_label,
|
|
272
|
+
"error_type": "wrong_paper",
|
|
273
|
+
"reason": "PDF is a different paper (neither its DOI nor its title is on the first pages)",
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def suspicious_pdf(identifier: str, file_path: Path, source_label: str) -> dict[str, Any]:
|
|
278
|
+
"""Build a result dict for a suspicious/preview PDF (not full text)."""
|
|
279
|
+
return {
|
|
280
|
+
"success": False,
|
|
281
|
+
"identifier": identifier,
|
|
282
|
+
"doi": identifier,
|
|
283
|
+
"file": str(file_path),
|
|
284
|
+
"source": source_label,
|
|
285
|
+
"error_type": "suspicious_pdf",
|
|
286
|
+
"reason": "PDF appears to be a cover page or preview (too small / too few pages)",
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def success(identifier: str, file_path: Path, source: str) -> dict[str, Any]:
|
|
291
|
+
size_kb = round(file_path.stat().st_size / 1024, 1)
|
|
292
|
+
return {
|
|
293
|
+
"success": True,
|
|
294
|
+
"identifier": identifier,
|
|
295
|
+
"doi": identifier,
|
|
296
|
+
"file": str(file_path),
|
|
297
|
+
"size_kb": size_kb,
|
|
298
|
+
"source": source,
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def fail(
|
|
303
|
+
identifier: str,
|
|
304
|
+
reason: str = "not found",
|
|
305
|
+
extra: dict[str, Any] | None = None,
|
|
306
|
+
*,
|
|
307
|
+
error_type: str = "",
|
|
308
|
+
action: str = "",
|
|
309
|
+
) -> dict[str, Any]:
|
|
310
|
+
result = {"success": False, "identifier": identifier, "doi": identifier, "reason": reason}
|
|
311
|
+
if error_type:
|
|
312
|
+
result["error_type"] = error_type
|
|
313
|
+
if action:
|
|
314
|
+
result["action"] = action
|
|
315
|
+
if extra:
|
|
316
|
+
result.update(extra)
|
|
317
|
+
return result
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def dedupe(items: Any) -> list[str]:
|
|
321
|
+
seen: set[str] = set()
|
|
322
|
+
out: list[str] = []
|
|
323
|
+
for item in items:
|
|
324
|
+
if not item:
|
|
325
|
+
continue
|
|
326
|
+
item = item.strip() if isinstance(item, str) else str(item)
|
|
327
|
+
if item and item not in seen:
|
|
328
|
+
seen.add(item)
|
|
329
|
+
out.append(item)
|
|
330
|
+
return out
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def iter_urls(obj: Any) -> Any:
|
|
334
|
+
if isinstance(obj, dict):
|
|
335
|
+
for key, value in obj.items():
|
|
336
|
+
if isinstance(value, str) and ("url" in key.lower() or value.startswith("http")):
|
|
337
|
+
yield value
|
|
338
|
+
else:
|
|
339
|
+
yield from iter_urls(value)
|
|
340
|
+
elif isinstance(obj, list):
|
|
341
|
+
for item in obj:
|
|
342
|
+
yield from iter_urls(item)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def extract_pdf_url_from_html(html: str, base_url: str) -> str | None:
|
|
346
|
+
urls: list[str] = []
|
|
347
|
+
for match in re.finditer(
|
|
348
|
+
r"""<meta[^>]+name=["']citation_pdf_url["'][^>]+content=["']([^"']+)["']""", html, re.I
|
|
349
|
+
):
|
|
350
|
+
urls.append(urllib.parse.urljoin(base_url, match.group(1)))
|
|
351
|
+
for match in re.finditer(
|
|
352
|
+
r"""<meta[^>]+content=["']([^"']+)["'][^>]+name=["']citation_pdf_url["']""", html, re.I
|
|
353
|
+
):
|
|
354
|
+
urls.append(urllib.parse.urljoin(base_url, match.group(1)))
|
|
355
|
+
if BeautifulSoup is not None:
|
|
356
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
357
|
+
for tag in soup.find_all(["iframe", "embed", "a"]):
|
|
358
|
+
candidate = tag.get("src") or tag.get("href")
|
|
359
|
+
if candidate:
|
|
360
|
+
urls.append(urllib.parse.urljoin(base_url, candidate))
|
|
361
|
+
else:
|
|
362
|
+
for match in re.finditer(r"""(?:src|href)=["']([^"']+)["']""", html, re.I):
|
|
363
|
+
urls.append(urllib.parse.urljoin(base_url, match.group(1)))
|
|
364
|
+
|
|
365
|
+
for url in dedupe(urls):
|
|
366
|
+
if is_plausible_pdf_url(url):
|
|
367
|
+
return url
|
|
368
|
+
return None
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def download_pdf(
|
|
372
|
+
url: str,
|
|
373
|
+
output_path: Path,
|
|
374
|
+
config: dict[str, Any],
|
|
375
|
+
source: str,
|
|
376
|
+
*,
|
|
377
|
+
require_pdf_like_url: bool = True,
|
|
378
|
+
use_tor: bool = False,
|
|
379
|
+
cookies: Any = None,
|
|
380
|
+
referer: str = "",
|
|
381
|
+
browser_ua: bool = False,
|
|
382
|
+
) -> dict[str, Any] | None:
|
|
383
|
+
if require_pdf_like_url and not is_plausible_pdf_url(url):
|
|
384
|
+
return None
|
|
385
|
+
|
|
386
|
+
try:
|
|
387
|
+
if cookies is not None:
|
|
388
|
+
from ..network import (
|
|
389
|
+
CHROME_UA,
|
|
390
|
+
request_timeout,
|
|
391
|
+
proxy_dict,
|
|
392
|
+
select_proxy_for_url,
|
|
393
|
+
USER_AGENT,
|
|
394
|
+
)
|
|
395
|
+
session = requests.Session()
|
|
396
|
+
session.trust_env = False
|
|
397
|
+
session.headers.update({"User-Agent": USER_AGENT})
|
|
398
|
+
if referer:
|
|
399
|
+
# Some shadow-library CDNs (sci.bban.top, observed 2026-09)
|
|
400
|
+
# 403 any PDF request without a same-site Referer.
|
|
401
|
+
session.headers.update({"Referer": referer})
|
|
402
|
+
if browser_ua:
|
|
403
|
+
# …and they also 403 non-browser User-Agents outright — the
|
|
404
|
+
# engine UA ("litloop/x") gets HTML even with Referer.
|
|
405
|
+
session.headers.update({"User-Agent": CHROME_UA})
|
|
406
|
+
session.cookies.update(cookies)
|
|
407
|
+
resp = session.get(
|
|
408
|
+
url,
|
|
409
|
+
timeout=request_timeout(config),
|
|
410
|
+
proxies=proxy_dict(select_proxy_for_url(url, config)),
|
|
411
|
+
allow_redirects=True,
|
|
412
|
+
stream=True,
|
|
413
|
+
)
|
|
414
|
+
else:
|
|
415
|
+
resp = fetch(url, config, stream=True, use_tor=use_tor)
|
|
416
|
+
if resp.status_code >= 400:
|
|
417
|
+
return None
|
|
418
|
+
|
|
419
|
+
iterator = resp.iter_content(chunk_size=8192)
|
|
420
|
+
first_chunk = next(iterator, b"")
|
|
421
|
+
if not _response_looks_pdf(resp, first_chunk):
|
|
422
|
+
return None
|
|
423
|
+
|
|
424
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
425
|
+
tmp_path = output_path.with_suffix(output_path.suffix + ".part")
|
|
426
|
+
try:
|
|
427
|
+
# requests' timeout tuple only guards connect + each socket read;
|
|
428
|
+
# a peer that drips bytes forever would hold the race open, so
|
|
429
|
+
# bound the whole stream with a wall-clock deadline too.
|
|
430
|
+
deadline = time.monotonic() + float(config.get("download_deadline_seconds", 60))
|
|
431
|
+
with tmp_path.open("wb") as fh:
|
|
432
|
+
fh.write(first_chunk)
|
|
433
|
+
fh.flush()
|
|
434
|
+
for chunk in iterator:
|
|
435
|
+
if time.monotonic() > deadline:
|
|
436
|
+
raise TimeoutError("stream deadline exceeded")
|
|
437
|
+
if chunk:
|
|
438
|
+
fh.write(chunk)
|
|
439
|
+
fh.flush()
|
|
440
|
+
tmp_path.replace(output_path)
|
|
441
|
+
except Exception:
|
|
442
|
+
tmp_path.unlink(missing_ok=True)
|
|
443
|
+
raise
|
|
444
|
+
|
|
445
|
+
if is_pdf_file(output_path):
|
|
446
|
+
return success(output_path.stem, output_path, source)
|
|
447
|
+
else:
|
|
448
|
+
try:
|
|
449
|
+
output_path.unlink(missing_ok=True)
|
|
450
|
+
except OSError:
|
|
451
|
+
pass
|
|
452
|
+
except Exception:
|
|
453
|
+
return None
|
|
454
|
+
return None
|