litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,975 @@
|
|
|
1
|
+
"""Core paper fetching logic with multi-stage pipeline."""
|
|
2
|
+
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import random
|
|
8
|
+
import re
|
|
9
|
+
import time
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
|
|
15
|
+
from ..institutional.auth import EZProxyAuth, WebVPNAuth
|
|
16
|
+
from ..http_utils import request_with_retry
|
|
17
|
+
from ..extractors import html_extractor, pdf_extractor
|
|
18
|
+
from ..models import FetchResult, NextAction, Paper
|
|
19
|
+
from ..publishers.pdf_router import build_pdf_candidates, discover_pdf_candidates_from_html
|
|
20
|
+
from ..publishers.profiles import infer_publisher_profile, infer_publisher_profile_from_url
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
from ..browser.backend import is_available as _browser_backend_available
|
|
24
|
+
_HAS_CLOAKBROWSER = _browser_backend_available()
|
|
25
|
+
except ImportError:
|
|
26
|
+
_HAS_CLOAKBROWSER = False
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
DOI_PATTERN = re.compile(r"^10\.\d{4,9}/[^\s]+$")
|
|
31
|
+
|
|
32
|
+
MIN_FULLTEXT_LEN = 1000
|
|
33
|
+
|
|
34
|
+
# Challenge keywords that indicate a Cloudflare/bot-detection page
|
|
35
|
+
_CHALLENGE_KEYWORDS = ("请稍候", "just a moment", "attention required", "verify", "security check")
|
|
36
|
+
|
|
37
|
+
_ATTEMPT_LOG: ContextVar[list[dict[str, str]] | None] = ContextVar(
|
|
38
|
+
"litloop_fetch_attempt_log",
|
|
39
|
+
default=None,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _record_attempt(stage: str, status: str, reason: str = "", detail: str = "") -> None:
|
|
44
|
+
attempts = _ATTEMPT_LOG.get()
|
|
45
|
+
if attempts is None:
|
|
46
|
+
return
|
|
47
|
+
attempt = {"stage": stage, "status": status}
|
|
48
|
+
if reason:
|
|
49
|
+
attempt["reason"] = reason
|
|
50
|
+
if detail:
|
|
51
|
+
attempt["detail"] = detail
|
|
52
|
+
attempts.append(attempt)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _record_paper_attempt(stage: str, paper: Paper | None) -> None:
|
|
56
|
+
if paper is None:
|
|
57
|
+
_record_attempt(stage, "miss", reason="no_result")
|
|
58
|
+
return
|
|
59
|
+
if len(paper.full_text or "") >= MIN_FULLTEXT_LEN:
|
|
60
|
+
_record_attempt(stage, "success", reason="full_text")
|
|
61
|
+
return
|
|
62
|
+
_record_attempt(stage, "partial", reason=_paper_quality(paper))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _paper_quality(paper: Paper) -> str:
|
|
66
|
+
if paper.full_text:
|
|
67
|
+
return "short_text"
|
|
68
|
+
if paper.pdf_path:
|
|
69
|
+
return "pdf_only"
|
|
70
|
+
if paper.abstract:
|
|
71
|
+
return "abstract_only"
|
|
72
|
+
if paper.title or paper.authors or paper.journal or paper.year:
|
|
73
|
+
return "metadata_only"
|
|
74
|
+
return "none"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _apply_attempt_diagnostics(result: FetchResult, identifier: str) -> None:
|
|
78
|
+
for attempt in result.attempts:
|
|
79
|
+
if (
|
|
80
|
+
attempt.get("stage") == "doi_resolve"
|
|
81
|
+
and attempt.get("status") == "miss"
|
|
82
|
+
and attempt.get("reason") == "no_url"
|
|
83
|
+
):
|
|
84
|
+
query = (identifier or result.paper.title or result.paper.doi).replace('"', '\\"')
|
|
85
|
+
result.status = "blocked"
|
|
86
|
+
result.reason = "doi_resolution_failed"
|
|
87
|
+
result.next_action = NextAction(
|
|
88
|
+
kind="check_identifier",
|
|
89
|
+
command=f'litloop search "{query}"',
|
|
90
|
+
message=(
|
|
91
|
+
"DOI did not resolve to an article URL. Check the identifier or search "
|
|
92
|
+
"for the paper, then retry with a DOI or publisher URL."
|
|
93
|
+
),
|
|
94
|
+
)
|
|
95
|
+
return
|
|
96
|
+
|
|
97
|
+
if (
|
|
98
|
+
result.status == "partial"
|
|
99
|
+
and result.next_action
|
|
100
|
+
and result.next_action.kind == "login"
|
|
101
|
+
and any(
|
|
102
|
+
attempt.get("stage") == "institutional_access"
|
|
103
|
+
and attempt.get("status") == "partial"
|
|
104
|
+
and attempt.get("reason") in {"none", "metadata_only", "abstract_only", "short_text"}
|
|
105
|
+
for attempt in result.attempts
|
|
106
|
+
)
|
|
107
|
+
):
|
|
108
|
+
result.status = "auth_required"
|
|
109
|
+
result.reason = "institution_login_required"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _is_good_result(paper: Paper | None) -> bool:
|
|
113
|
+
"""Check if a paper result has enough full text to be useful."""
|
|
114
|
+
return bool(paper and len(paper.full_text or "") >= MIN_FULLTEXT_LEN)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _copy_paper_metadata(paper: Paper, url: str) -> Paper:
|
|
118
|
+
"""Create a new Paper copying metadata from an existing one, with a new URL."""
|
|
119
|
+
return Paper(
|
|
120
|
+
doi=paper.doi, title=paper.title, authors=paper.authors,
|
|
121
|
+
journal=paper.journal, year=paper.year, abstract=paper.abstract,
|
|
122
|
+
url=url,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _apply_pdf_bytes(paper: Paper, pdf_bytes: bytes, doi: str, source: str,
|
|
127
|
+
save_fn=None) -> None:
|
|
128
|
+
"""Extract text from PDF bytes and update paper fields in-place."""
|
|
129
|
+
paper.full_text = pdf_extractor.extract_from_bytes(pdf_bytes)
|
|
130
|
+
if save_fn:
|
|
131
|
+
pdf_path = save_fn(doi, pdf_bytes)
|
|
132
|
+
paper.pdf_path = str(pdf_path) if pdf_path else ""
|
|
133
|
+
paper.source = source
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _wait_for_challenge(page, max_tries: int = 6, *, current: str = "") -> None:
|
|
137
|
+
"""Wait for a visible browser challenge and expose it in the progress bar."""
|
|
138
|
+
reporter = None
|
|
139
|
+
attention_key = f"browser-challenge:{id(page)}:{current}"
|
|
140
|
+
attention_open = False
|
|
141
|
+
try:
|
|
142
|
+
from . import progress as reporter
|
|
143
|
+
except Exception:
|
|
144
|
+
reporter = None
|
|
145
|
+
try:
|
|
146
|
+
for i in range(max_tries):
|
|
147
|
+
title = page.title().lower()
|
|
148
|
+
if any(sig in title for sig in _CHALLENGE_KEYWORDS):
|
|
149
|
+
if not attention_open and reporter is not None:
|
|
150
|
+
reporter.set_attention(
|
|
151
|
+
attention_key,
|
|
152
|
+
"请在浏览器窗口完成安全验证",
|
|
153
|
+
current=current,
|
|
154
|
+
phase="人工验证",
|
|
155
|
+
)
|
|
156
|
+
attention_open = True
|
|
157
|
+
logger.info("Challenge page detected, waiting... (%d/%d)", i + 1, max_tries)
|
|
158
|
+
time.sleep(5)
|
|
159
|
+
else:
|
|
160
|
+
break
|
|
161
|
+
finally:
|
|
162
|
+
if attention_open and reporter is not None:
|
|
163
|
+
reporter.clear_attention(attention_key)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class PaperFetcher:
|
|
167
|
+
"""Main class for fetching academic papers via multi-stage pipeline."""
|
|
168
|
+
|
|
169
|
+
def __init__(self, config: dict | None = None):
|
|
170
|
+
from ..config import load_config
|
|
171
|
+
self.config = config or load_config()
|
|
172
|
+
self._auth: WebVPNAuth | EZProxyAuth | None = None
|
|
173
|
+
self._last_request_time = 0.0
|
|
174
|
+
|
|
175
|
+
def _challenge_max_tries(self) -> int:
|
|
176
|
+
"""Keep a challenged DOI open long enough for a human to act."""
|
|
177
|
+
try:
|
|
178
|
+
seconds = int(self.config.get("challenge_wait_seconds", 900) or 900)
|
|
179
|
+
except (AttributeError, TypeError, ValueError):
|
|
180
|
+
seconds = 900
|
|
181
|
+
return max(6, (max(0, seconds) + 4) // 5)
|
|
182
|
+
|
|
183
|
+
@property
|
|
184
|
+
def auth(self) -> WebVPNAuth | EZProxyAuth:
|
|
185
|
+
if self._auth is None:
|
|
186
|
+
from ..institutional.schools import get_school
|
|
187
|
+
school_name = (self.config.get("instsci_school", "")
|
|
188
|
+
or self.config.get("vpnsci_school", "")).strip()
|
|
189
|
+
if not school_name:
|
|
190
|
+
# Raised deliberately: fetch_with_result() converts ValueError
|
|
191
|
+
# into a structured config_needed outcome. get_school("") would
|
|
192
|
+
# otherwise fuzzy-match an arbitrary school and the cascade
|
|
193
|
+
# would navigate to a garbage/empty gateway URL.
|
|
194
|
+
raise ValueError(
|
|
195
|
+
"no institution configured (instsci_school/vpnsci_school are empty)"
|
|
196
|
+
)
|
|
197
|
+
entry = get_school(school_name)
|
|
198
|
+
if entry.school_type == "ezproxy":
|
|
199
|
+
self._auth = EZProxyAuth(self.config, proxy_base=entry.host)
|
|
200
|
+
else:
|
|
201
|
+
# Fall back to the school database gateway host when no
|
|
202
|
+
# explicit base URL is configured.
|
|
203
|
+
self._auth = WebVPNAuth(self.config, key=entry.key, iv=entry.iv,
|
|
204
|
+
base_url=getattr(entry, "host", "") or "")
|
|
205
|
+
return self._auth
|
|
206
|
+
|
|
207
|
+
def fetch(self, identifier: str, use_cache: bool = True) -> Paper:
|
|
208
|
+
"""Fetch a paper by DOI or URL."""
|
|
209
|
+
doi = self._parse_doi(identifier)
|
|
210
|
+
url = self._parse_url(identifier)
|
|
211
|
+
|
|
212
|
+
if use_cache and doi:
|
|
213
|
+
cached = self._load_cache(doi)
|
|
214
|
+
if _is_good_result(cached):
|
|
215
|
+
_record_paper_attempt("cache", cached)
|
|
216
|
+
logger.info("Loaded from cache (good full text): %s", doi)
|
|
217
|
+
return cached
|
|
218
|
+
elif cached:
|
|
219
|
+
_record_paper_attempt("cache", cached)
|
|
220
|
+
logger.info("Cache hit but full text too short (%d chars), re-fetching: %s",
|
|
221
|
+
len(cached.full_text or ""), doi)
|
|
222
|
+
else:
|
|
223
|
+
_record_attempt("cache", "miss", reason="not_found")
|
|
224
|
+
|
|
225
|
+
paper = Paper(doi=doi or "", url=url or "")
|
|
226
|
+
|
|
227
|
+
# Helper: record attempt, cache if good, return whether result is usable
|
|
228
|
+
def _check(stage: str, result: Paper | None) -> bool:
|
|
229
|
+
_record_paper_attempt(stage, result)
|
|
230
|
+
if _is_good_result(result):
|
|
231
|
+
self._save_cache(result)
|
|
232
|
+
return True
|
|
233
|
+
return False
|
|
234
|
+
|
|
235
|
+
# Step 1: Try Open Access sources first
|
|
236
|
+
if doi:
|
|
237
|
+
oa_paper = self._try_open_access(doi, identifier)
|
|
238
|
+
if _check("open_access", oa_paper):
|
|
239
|
+
return oa_paper
|
|
240
|
+
if oa_paper:
|
|
241
|
+
paper = oa_paper
|
|
242
|
+
|
|
243
|
+
# Step 2: Try Elsevier API before institutional flows
|
|
244
|
+
if doi and not paper.pdf_path:
|
|
245
|
+
api_paper = self._try_elsevier_api(doi, paper)
|
|
246
|
+
if _check("elsevier_api", api_paper):
|
|
247
|
+
return api_paper
|
|
248
|
+
|
|
249
|
+
# Step 3: Resolve DOI to URL if needed
|
|
250
|
+
if doi and not url:
|
|
251
|
+
url = self._resolve_doi(doi)
|
|
252
|
+
_record_attempt("doi_resolve", "success" if url else "miss",
|
|
253
|
+
detail=url or "", reason="" if url else "no_url")
|
|
254
|
+
paper.url = url or ""
|
|
255
|
+
|
|
256
|
+
if not url:
|
|
257
|
+
logger.error("Could not determine URL for: %s", identifier)
|
|
258
|
+
return paper
|
|
259
|
+
|
|
260
|
+
# Step 4: Try CARSI federated access
|
|
261
|
+
if self.config.get("carsi_enabled") and doi:
|
|
262
|
+
carsi_paper = self._try_carsi_pdf(doi, url, paper)
|
|
263
|
+
if _check("carsi_pdf", carsi_paper):
|
|
264
|
+
return carsi_paper
|
|
265
|
+
carsi_paper = self._try_carsi_html(url, paper)
|
|
266
|
+
if _check("carsi_html", carsi_paper):
|
|
267
|
+
return carsi_paper
|
|
268
|
+
|
|
269
|
+
# Step 5: Try direct publisher PDF URL construction
|
|
270
|
+
if doi and not paper.pdf_path:
|
|
271
|
+
pdf_paper = self._try_publisher_pdf(doi, url, paper)
|
|
272
|
+
if _check("publisher_pdf", pdf_paper):
|
|
273
|
+
return pdf_paper
|
|
274
|
+
|
|
275
|
+
# Step 6: Try browser-based PDF download
|
|
276
|
+
if doi and not paper.pdf_path:
|
|
277
|
+
browser_paper = self._try_browser_pdf_download(doi, url, paper)
|
|
278
|
+
if _check("browser_pdf", browser_paper):
|
|
279
|
+
return browser_paper
|
|
280
|
+
|
|
281
|
+
# Step 7: Fetch via institutional campus access
|
|
282
|
+
self._rate_limit()
|
|
283
|
+
try:
|
|
284
|
+
paper = self._fetch_via_webvpn(url, paper)
|
|
285
|
+
except ValueError:
|
|
286
|
+
_record_attempt("institutional_access", "error", reason="config_needed")
|
|
287
|
+
raise
|
|
288
|
+
except requests.RequestException:
|
|
289
|
+
_record_attempt("institutional_access", "error", reason="gateway_unreachable")
|
|
290
|
+
raise
|
|
291
|
+
_record_paper_attempt("institutional_access", paper)
|
|
292
|
+
|
|
293
|
+
if _is_good_result(paper):
|
|
294
|
+
self._save_cache(paper)
|
|
295
|
+
|
|
296
|
+
return paper
|
|
297
|
+
|
|
298
|
+
def fetch_with_result(self, identifier: str, use_cache: bool = True) -> FetchResult:
|
|
299
|
+
"""Fetch a paper and return a structured, agent-friendly outcome."""
|
|
300
|
+
attempts: list[dict[str, str]] = []
|
|
301
|
+
token = _ATTEMPT_LOG.set(attempts)
|
|
302
|
+
try:
|
|
303
|
+
paper = self.fetch(identifier, use_cache=use_cache)
|
|
304
|
+
except ValueError as exc:
|
|
305
|
+
doi = self._parse_doi(identifier) or ""
|
|
306
|
+
url = self._parse_url(identifier) or ""
|
|
307
|
+
return FetchResult(
|
|
308
|
+
status="config_needed",
|
|
309
|
+
quality="none",
|
|
310
|
+
reason="institution_not_configured",
|
|
311
|
+
paper=Paper(doi=doi, url=url),
|
|
312
|
+
next_action=NextAction(
|
|
313
|
+
kind="configure_institution",
|
|
314
|
+
command="litloop config set instsci_school YOUR_SCHOOL",
|
|
315
|
+
message=f"Configure your school or institution before retrying. Detail: {exc}",
|
|
316
|
+
),
|
|
317
|
+
attempts=attempts,
|
|
318
|
+
)
|
|
319
|
+
except requests.RequestException as exc:
|
|
320
|
+
doi = self._parse_doi(identifier) or ""
|
|
321
|
+
url = self._parse_url(identifier) or ""
|
|
322
|
+
gateway_error = any(
|
|
323
|
+
a.get("stage") == "institutional_access"
|
|
324
|
+
and a.get("status") == "error"
|
|
325
|
+
and a.get("reason") == "gateway_unreachable"
|
|
326
|
+
for a in attempts
|
|
327
|
+
)
|
|
328
|
+
reason = "gateway_unreachable" if gateway_error else "network_error"
|
|
329
|
+
kind = "diagnose_gateway" if gateway_error else "diagnose"
|
|
330
|
+
message = (
|
|
331
|
+
"Institutional gateway could not be reached. Check VPN/proxy/CARSI/WebVPN "
|
|
332
|
+
f"configuration, then retry. Detail: {exc}"
|
|
333
|
+
if gateway_error
|
|
334
|
+
else f"Check network and institutional access configuration. Detail: {exc}"
|
|
335
|
+
)
|
|
336
|
+
return FetchResult(
|
|
337
|
+
status="blocked",
|
|
338
|
+
quality="none",
|
|
339
|
+
reason=reason,
|
|
340
|
+
paper=Paper(doi=doi, url=url),
|
|
341
|
+
next_action=NextAction(
|
|
342
|
+
kind=kind,
|
|
343
|
+
command="litloop config show",
|
|
344
|
+
message=message,
|
|
345
|
+
),
|
|
346
|
+
attempts=attempts,
|
|
347
|
+
)
|
|
348
|
+
finally:
|
|
349
|
+
_ATTEMPT_LOG.reset(token)
|
|
350
|
+
|
|
351
|
+
result = FetchResult.from_paper(
|
|
352
|
+
paper,
|
|
353
|
+
min_fulltext_len=MIN_FULLTEXT_LEN,
|
|
354
|
+
institution_configured=self._institution_configured(),
|
|
355
|
+
identifier=identifier,
|
|
356
|
+
)
|
|
357
|
+
result.attempts = attempts
|
|
358
|
+
_apply_attempt_diagnostics(result, identifier)
|
|
359
|
+
return result
|
|
360
|
+
|
|
361
|
+
def _institution_configured(self) -> bool:
|
|
362
|
+
return bool(
|
|
363
|
+
self.config.get("instsci_school")
|
|
364
|
+
or self.config.get("instsci_base_url")
|
|
365
|
+
or self.config.get("vpnsci_base_url")
|
|
366
|
+
or self.config.get("ezproxy_login_url")
|
|
367
|
+
or self.config.get("network_proxy")
|
|
368
|
+
or (self.config.get("carsi_enabled") and self.config.get("carsi_idp_name"))
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
def _try_open_access(self, doi: str, identifier: str = "") -> Paper | None:
|
|
372
|
+
"""Try to fetch paper from Open Access sources."""
|
|
373
|
+
from ..sources import unpaywall, arxiv
|
|
374
|
+
from ..identifiers import normalize_arxiv_id
|
|
375
|
+
|
|
376
|
+
output_dir = Path(self.config.get("output_dir", "."))
|
|
377
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
378
|
+
|
|
379
|
+
# Check if it's an arXiv paper first
|
|
380
|
+
arxiv_id = normalize_arxiv_id(identifier or doi)
|
|
381
|
+
if arxiv_id:
|
|
382
|
+
logger.info("Fetching from arXiv: %s", arxiv_id)
|
|
383
|
+
safe_id = arxiv_id.replace("/", "_")
|
|
384
|
+
output_path = output_dir / f"arxiv_{safe_id}.pdf"
|
|
385
|
+
result = arxiv.try_arxiv(arxiv_id, output_path, self.config)
|
|
386
|
+
if result and result.get("path"):
|
|
387
|
+
paper = Paper(
|
|
388
|
+
doi=doi,
|
|
389
|
+
url=result.get("url", f"https://arxiv.org/abs/{arxiv_id}"),
|
|
390
|
+
source="arxiv",
|
|
391
|
+
pdf_path=str(result["path"]),
|
|
392
|
+
)
|
|
393
|
+
self._extract_pdf_text(paper, result["path"])
|
|
394
|
+
return paper
|
|
395
|
+
|
|
396
|
+
# Try Unpaywall
|
|
397
|
+
logger.info("Checking Unpaywall for OA version of %s...", doi)
|
|
398
|
+
safe_doi = re.sub(r"[^\w\-.]", "_", doi)
|
|
399
|
+
output_path = output_dir / f"unpaywall_{safe_doi}.pdf"
|
|
400
|
+
result = unpaywall.try_unpaywall(doi, output_path, self.config)
|
|
401
|
+
if result and result.get("path"):
|
|
402
|
+
paper = Paper(
|
|
403
|
+
doi=doi,
|
|
404
|
+
url=result.get("url", ""),
|
|
405
|
+
source="open_access",
|
|
406
|
+
pdf_path=str(result["path"]),
|
|
407
|
+
)
|
|
408
|
+
self._extract_pdf_text(paper, result["path"])
|
|
409
|
+
return paper
|
|
410
|
+
|
|
411
|
+
return Paper(doi=doi)
|
|
412
|
+
|
|
413
|
+
def _try_publisher_pdf(self, doi: str, resolved_url: str, paper: Paper) -> Paper | None:
|
|
414
|
+
pdf_url = self._build_publisher_pdf_url(doi, resolved_url)
|
|
415
|
+
if not pdf_url:
|
|
416
|
+
return None
|
|
417
|
+
|
|
418
|
+
logger.info("Trying constructed publisher PDF URL: %s", pdf_url)
|
|
419
|
+
|
|
420
|
+
if not self.auth.login():
|
|
421
|
+
logger.error("Institutional access authentication failed.")
|
|
422
|
+
return None
|
|
423
|
+
|
|
424
|
+
self._rate_limit()
|
|
425
|
+
try:
|
|
426
|
+
resp = self.auth.fetch(pdf_url)
|
|
427
|
+
resp.raise_for_status()
|
|
428
|
+
ct = resp.headers.get("content-type", "").lower()
|
|
429
|
+
|
|
430
|
+
if "pdf" in ct and len(resp.content) > 10000:
|
|
431
|
+
_apply_pdf_bytes(paper, resp.content, doi, "institutional", self._save_pdf)
|
|
432
|
+
logger.info("Publisher PDF downloaded (%d bytes, %d chars text)",
|
|
433
|
+
len(resp.content), len(paper.full_text or ""))
|
|
434
|
+
return paper
|
|
435
|
+
else:
|
|
436
|
+
logger.info("Publisher PDF URL returned non-PDF or too small (ct=%s, size=%d)",
|
|
437
|
+
ct, len(resp.content))
|
|
438
|
+
except requests.RequestException as e:
|
|
439
|
+
logger.warning("Failed to fetch publisher PDF: %s", e)
|
|
440
|
+
|
|
441
|
+
return None
|
|
442
|
+
|
|
443
|
+
def _try_browser_pdf_download(self, doi: str, resolved_url: str, paper: Paper) -> Paper | None:
|
|
444
|
+
if not _HAS_CLOAKBROWSER:
|
|
445
|
+
return None
|
|
446
|
+
|
|
447
|
+
pdf_url = self._build_publisher_pdf_url(doi, resolved_url)
|
|
448
|
+
if not pdf_url:
|
|
449
|
+
return None
|
|
450
|
+
|
|
451
|
+
print(" [Browser] Downloading PDF via browser...")
|
|
452
|
+
|
|
453
|
+
live_context = getattr(self.auth, 'browser_context', None)
|
|
454
|
+
if live_context:
|
|
455
|
+
result = self._browser_pdf_download(live_context, resolved_url, doi, paper)
|
|
456
|
+
if result:
|
|
457
|
+
return result
|
|
458
|
+
|
|
459
|
+
logger.info("No live browser session. Triggering browser login for PDF download...")
|
|
460
|
+
if self.auth.login(force=True):
|
|
461
|
+
live_context = getattr(self.auth, 'browser_context', None)
|
|
462
|
+
if live_context:
|
|
463
|
+
result = self._browser_pdf_download(live_context, resolved_url, doi, paper)
|
|
464
|
+
if result:
|
|
465
|
+
return result
|
|
466
|
+
|
|
467
|
+
return None
|
|
468
|
+
|
|
469
|
+
def _browser_pdf_download(self, context, article_url: str, doi: str, paper: Paper) -> Paper | None:
|
|
470
|
+
page = context.new_page()
|
|
471
|
+
try:
|
|
472
|
+
return self._browser_pdf_do(page, article_url, doi, paper)
|
|
473
|
+
finally:
|
|
474
|
+
try:
|
|
475
|
+
page.close()
|
|
476
|
+
except Exception:
|
|
477
|
+
pass
|
|
478
|
+
|
|
479
|
+
def _browser_pdf_do(self, page, article_url: str, doi: str, paper: Paper) -> Paper | None:
|
|
480
|
+
try:
|
|
481
|
+
proxied_article = self.auth.convert_url(article_url)
|
|
482
|
+
except Exception:
|
|
483
|
+
proxied_article = article_url
|
|
484
|
+
|
|
485
|
+
logger.info("Browser: navigating to article page %s", proxied_article[:80])
|
|
486
|
+
try:
|
|
487
|
+
page.goto(proxied_article, wait_until="domcontentloaded", timeout=60000)
|
|
488
|
+
except Exception as e:
|
|
489
|
+
logger.warning("Navigation failed: %s", e)
|
|
490
|
+
|
|
491
|
+
challenge_tries = self._challenge_max_tries()
|
|
492
|
+
_wait_for_challenge(page, max_tries=challenge_tries, current=doi)
|
|
493
|
+
|
|
494
|
+
current_url = page.url
|
|
495
|
+
logger.info("Browser article page: url=%s, title=%s", current_url[:60], page.title()[:40])
|
|
496
|
+
|
|
497
|
+
if "cas" in current_url.lower() or "login" in current_url.lower() or \
|
|
498
|
+
"登录" in page.title() or "身份" in page.title():
|
|
499
|
+
logger.info("Browser landed on CAS page")
|
|
500
|
+
return None
|
|
501
|
+
|
|
502
|
+
if "linkinghub" in current_url or "retrieve/pii" in current_url:
|
|
503
|
+
logger.info("On linkinghub redirect page, waiting for redirect...")
|
|
504
|
+
time.sleep(5)
|
|
505
|
+
current_url = page.url
|
|
506
|
+
if "linkinghub" in current_url:
|
|
507
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", current_url)
|
|
508
|
+
if pii_match:
|
|
509
|
+
direct_url = f"https://www.sciencedirect.com/science/article/pii/{pii_match.group(1)}"
|
|
510
|
+
try:
|
|
511
|
+
proxied_direct = self.auth.convert_url(direct_url)
|
|
512
|
+
except Exception:
|
|
513
|
+
proxied_direct = direct_url
|
|
514
|
+
try:
|
|
515
|
+
page.goto(proxied_direct, wait_until="domcontentloaded", timeout=30000)
|
|
516
|
+
time.sleep(3)
|
|
517
|
+
except Exception:
|
|
518
|
+
pass
|
|
519
|
+
_wait_for_challenge(page, max_tries=challenge_tries, current=doi)
|
|
520
|
+
current_url = page.url
|
|
521
|
+
|
|
522
|
+
try:
|
|
523
|
+
page.wait_for_load_state("networkidle", timeout=15000)
|
|
524
|
+
except Exception:
|
|
525
|
+
pass
|
|
526
|
+
|
|
527
|
+
pdf_link = page.evaluate("""
|
|
528
|
+
(() => {
|
|
529
|
+
for (const meta of document.querySelectorAll('meta[name="citation_pdf_url"]')) {
|
|
530
|
+
if (meta.content) return meta.content;
|
|
531
|
+
}
|
|
532
|
+
for (const a of document.querySelectorAll('a[aria-label*="PDF" i], a[title*="PDF" i]')) {
|
|
533
|
+
if (a.href) return a.href;
|
|
534
|
+
}
|
|
535
|
+
for (const a of document.querySelectorAll('a')) {
|
|
536
|
+
const href = (a.href || '').toLowerCase();
|
|
537
|
+
const text = (a.textContent || '').toLowerCase();
|
|
538
|
+
if (href.includes('/pdfft') || href.includes('/pdfdirect')) return a.href;
|
|
539
|
+
if (href.includes('pdf') && !href.includes('supplement') && !href.includes('cite')) return a.href;
|
|
540
|
+
if (text.includes('pdf') && (text.includes('download') || text.includes('view'))) return a.href;
|
|
541
|
+
}
|
|
542
|
+
return null;
|
|
543
|
+
})()
|
|
544
|
+
""")
|
|
545
|
+
|
|
546
|
+
if not pdf_link or not isinstance(pdf_link, str):
|
|
547
|
+
logger.info("No PDF link found on article page")
|
|
548
|
+
return None
|
|
549
|
+
|
|
550
|
+
logger.info("Found PDF link on page: %s", pdf_link[:80])
|
|
551
|
+
|
|
552
|
+
if pdf_link.startswith("/"):
|
|
553
|
+
parsed = urlparse(current_url)
|
|
554
|
+
pdf_link = f"{parsed.scheme}://{parsed.netloc}{pdf_link}"
|
|
555
|
+
|
|
556
|
+
captured_pdf = {"bytes": None}
|
|
557
|
+
|
|
558
|
+
def _on_response(response):
|
|
559
|
+
try:
|
|
560
|
+
ct = response.headers.get("content-type", "")
|
|
561
|
+
if "pdf" in ct or "octet" in ct:
|
|
562
|
+
body = response.body()
|
|
563
|
+
if body[:5] == b"%PDF-":
|
|
564
|
+
captured_pdf["bytes"] = body
|
|
565
|
+
except Exception:
|
|
566
|
+
pass
|
|
567
|
+
|
|
568
|
+
page.on("response", _on_response)
|
|
569
|
+
|
|
570
|
+
pdf_paths = self._build_browser_pdf_paths(doi, current_url)
|
|
571
|
+
all_urls = [pdf_link]
|
|
572
|
+
_parsed = urlparse(current_url)
|
|
573
|
+
_origin = f"{_parsed.scheme}://{_parsed.netloc}"
|
|
574
|
+
for p in pdf_paths:
|
|
575
|
+
full = _origin + p if p.startswith("/") else p
|
|
576
|
+
if full not in all_urls:
|
|
577
|
+
all_urls.append(full)
|
|
578
|
+
|
|
579
|
+
for pdf_url in all_urls:
|
|
580
|
+
if captured_pdf["bytes"]:
|
|
581
|
+
break
|
|
582
|
+
logger.info("Navigating to PDF URL: %s", pdf_url[:80])
|
|
583
|
+
try:
|
|
584
|
+
page.goto(pdf_url, wait_until="domcontentloaded", timeout=30000)
|
|
585
|
+
except Exception as e:
|
|
586
|
+
logger.debug("Navigation to %s failed: %s", pdf_url[:60], e)
|
|
587
|
+
continue
|
|
588
|
+
|
|
589
|
+
_wait_for_challenge(page, max_tries=challenge_tries, current=doi)
|
|
590
|
+
|
|
591
|
+
if not captured_pdf["bytes"]:
|
|
592
|
+
try:
|
|
593
|
+
page.wait_for_load_state("networkidle", timeout=10000)
|
|
594
|
+
except Exception:
|
|
595
|
+
pass
|
|
596
|
+
try:
|
|
597
|
+
pdf_from_viewer = page.evaluate("""
|
|
598
|
+
() => {
|
|
599
|
+
const obj = document.querySelector('object[type="application/pdf"]');
|
|
600
|
+
if (obj && obj.data) return obj.data;
|
|
601
|
+
const embed = document.querySelector('embed[type="application/pdf"]');
|
|
602
|
+
if (embed && embed.src) return embed.src;
|
|
603
|
+
const iframe = document.querySelector('iframe[src*="pdf"]');
|
|
604
|
+
if (iframe && iframe.src) return iframe.src;
|
|
605
|
+
return null;
|
|
606
|
+
}
|
|
607
|
+
""")
|
|
608
|
+
if pdf_from_viewer and isinstance(pdf_from_viewer, str):
|
|
609
|
+
if pdf_from_viewer.startswith("/"):
|
|
610
|
+
pdf_from_viewer = _origin + pdf_from_viewer
|
|
611
|
+
try:
|
|
612
|
+
page.goto(pdf_from_viewer, wait_until="domcontentloaded", timeout=30000)
|
|
613
|
+
time.sleep(2)
|
|
614
|
+
except Exception:
|
|
615
|
+
pass
|
|
616
|
+
except Exception:
|
|
617
|
+
pass
|
|
618
|
+
|
|
619
|
+
if not captured_pdf["bytes"]:
|
|
620
|
+
final_title = page.title().lower()
|
|
621
|
+
if "robot" in final_title or "captcha" in final_title:
|
|
622
|
+
logger.warning("Anti-bot detected on PDF page, trying next URL")
|
|
623
|
+
continue
|
|
624
|
+
|
|
625
|
+
page.remove_listener("response", _on_response)
|
|
626
|
+
|
|
627
|
+
pdf_bytes = captured_pdf["bytes"]
|
|
628
|
+
if pdf_bytes and pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
629
|
+
_apply_pdf_bytes(paper, pdf_bytes, doi, "browser", self._save_pdf)
|
|
630
|
+
logger.info("Browser PDF downloaded via response capture (%d bytes)", len(pdf_bytes))
|
|
631
|
+
return paper
|
|
632
|
+
|
|
633
|
+
logger.info("Could not capture PDF from browser navigation")
|
|
634
|
+
return None
|
|
635
|
+
|
|
636
|
+
@staticmethod
|
|
637
|
+
def _build_browser_pdf_paths(doi: str, current_url: str) -> list[str]:
|
|
638
|
+
parsed = urlparse(current_url)
|
|
639
|
+
doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
|
|
640
|
+
|
|
641
|
+
paths = []
|
|
642
|
+
if "sciencedirect" in parsed.netloc or "elsevier" in parsed.netloc:
|
|
643
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", current_url)
|
|
644
|
+
if pii_match:
|
|
645
|
+
pii = pii_match.group(1)
|
|
646
|
+
paths.append(f"/science/article/pii/{pii}/pdfft")
|
|
647
|
+
paths.append(f"/doi/pdfdirect/{doi}")
|
|
648
|
+
elif "springer" in parsed.netloc:
|
|
649
|
+
paths.append(f"/content/pdf/{doi}.pdf")
|
|
650
|
+
elif "wiley" in parsed.netloc:
|
|
651
|
+
paths.append(f"/doi/pdfdirect/{doi}")
|
|
652
|
+
paths.append(f"/doi/pdf/{doi}")
|
|
653
|
+
paths.append(f"/doi/pdf/{doi}")
|
|
654
|
+
return paths
|
|
655
|
+
|
|
656
|
+
def _try_carsi_pdf(self, doi: str, resolved_url: str, paper: Paper) -> Paper | None:
|
|
657
|
+
from ..institutional.carsi_login import CARSIClient
|
|
658
|
+
|
|
659
|
+
pdf_url = self._build_publisher_pdf_url(doi, resolved_url)
|
|
660
|
+
if not pdf_url:
|
|
661
|
+
return None
|
|
662
|
+
|
|
663
|
+
logger.info("Trying CARSI publisher PDF: %s", pdf_url)
|
|
664
|
+
self._rate_limit()
|
|
665
|
+
try:
|
|
666
|
+
carsi = CARSIClient(self.config)
|
|
667
|
+
resp = carsi.fetch(pdf_url)
|
|
668
|
+
resp.raise_for_status()
|
|
669
|
+
ct = resp.headers.get("content-type", "").lower()
|
|
670
|
+
if "pdf" in ct and len(resp.content) > 10000:
|
|
671
|
+
result = _copy_paper_metadata(paper, pdf_url)
|
|
672
|
+
_apply_pdf_bytes(result, resp.content, doi, "carsi", self._save_pdf)
|
|
673
|
+
logger.info("CARSI PDF downloaded (%d bytes)", len(resp.content))
|
|
674
|
+
return result
|
|
675
|
+
except Exception as e:
|
|
676
|
+
logger.warning("CARSI PDF failed: %s", e)
|
|
677
|
+
return None
|
|
678
|
+
|
|
679
|
+
def _try_carsi_html(self, url: str, paper: Paper) -> Paper | None:
|
|
680
|
+
from ..institutional.carsi_login import CARSIClient
|
|
681
|
+
|
|
682
|
+
logger.info("Trying CARSI HTML: %s", url)
|
|
683
|
+
self._rate_limit()
|
|
684
|
+
try:
|
|
685
|
+
carsi = CARSIClient(self.config)
|
|
686
|
+
resp = carsi.fetch(url)
|
|
687
|
+
resp.raise_for_status()
|
|
688
|
+
ct = resp.headers.get("content-type", "").lower()
|
|
689
|
+
if "pdf" in ct:
|
|
690
|
+
result = _copy_paper_metadata(paper, url)
|
|
691
|
+
_apply_pdf_bytes(result, resp.content, paper.doi or "unknown", "carsi",
|
|
692
|
+
self._save_pdf if paper.doi else None)
|
|
693
|
+
return result
|
|
694
|
+
|
|
695
|
+
result = _copy_paper_metadata(paper, url)
|
|
696
|
+
self._apply_extracted(result, html_extractor.extract(resp.text, url))
|
|
697
|
+
result.source = "carsi"
|
|
698
|
+
if _is_good_result(result):
|
|
699
|
+
return result
|
|
700
|
+
except Exception as e:
|
|
701
|
+
logger.warning("CARSI HTML failed: %s", e)
|
|
702
|
+
return None
|
|
703
|
+
|
|
704
|
+
def _fetch_via_webvpn(self, url: str, paper: Paper) -> Paper:
|
|
705
|
+
if not self.auth.login():
|
|
706
|
+
logger.error("Institutional access authentication failed.")
|
|
707
|
+
return paper
|
|
708
|
+
|
|
709
|
+
paper.source = "institutional"
|
|
710
|
+
|
|
711
|
+
try:
|
|
712
|
+
resp = self.auth.fetch(url)
|
|
713
|
+
resp.raise_for_status()
|
|
714
|
+
except requests.RequestException as e:
|
|
715
|
+
logger.error("Failed to fetch via institutional access: %s", e)
|
|
716
|
+
return paper
|
|
717
|
+
|
|
718
|
+
if "pdf" in resp.headers.get("content-type", "").lower():
|
|
719
|
+
_apply_pdf_bytes(paper, resp.content, paper.doi or "unknown", "institutional",
|
|
720
|
+
self._save_pdf)
|
|
721
|
+
return paper
|
|
722
|
+
|
|
723
|
+
self._apply_extracted(paper, html_extractor.extract(resp.text, resp.url))
|
|
724
|
+
|
|
725
|
+
pdf_url = self._find_pdf_link(resp.text, resp.url)
|
|
726
|
+
if pdf_url:
|
|
727
|
+
logger.info("Found PDF link in HTML, downloading: %s", pdf_url)
|
|
728
|
+
self._rate_limit()
|
|
729
|
+
try:
|
|
730
|
+
pdf_resp = self.auth.fetch(pdf_url)
|
|
731
|
+
pdf_resp.raise_for_status()
|
|
732
|
+
ct = pdf_resp.headers.get("content-type", "").lower()
|
|
733
|
+
if "pdf" in ct and len(pdf_resp.content) > 10000:
|
|
734
|
+
pdf_path = self._save_pdf(paper.doi or "unknown", pdf_resp.content)
|
|
735
|
+
paper.pdf_path = str(pdf_path) if pdf_path else ""
|
|
736
|
+
if not _is_good_result(paper):
|
|
737
|
+
paper.full_text = pdf_extractor.extract_from_bytes(pdf_resp.content)
|
|
738
|
+
except requests.RequestException as e:
|
|
739
|
+
logger.warning("Failed to download PDF: %s", e)
|
|
740
|
+
|
|
741
|
+
return paper
|
|
742
|
+
|
|
743
|
+
def _try_elsevier_api(self, doi: str, paper: Paper) -> Paper | None:
|
|
744
|
+
from ..sources import elsevier_api
|
|
745
|
+
|
|
746
|
+
api_key = elsevier_api.get_api_key(self.config.get("elsevier_api_key", ""))
|
|
747
|
+
if not api_key:
|
|
748
|
+
logger.debug("No Elsevier API key configured")
|
|
749
|
+
return None
|
|
750
|
+
|
|
751
|
+
if not doi.startswith("10.1016/"):
|
|
752
|
+
return None
|
|
753
|
+
|
|
754
|
+
logger.info("Trying Elsevier API for %s", doi)
|
|
755
|
+
inst_token = self.config.get("elsevier_insttoken", "")
|
|
756
|
+
|
|
757
|
+
data = elsevier_api.fetch_fulltext(doi, api_key=api_key, inst_token=inst_token, config=self.config)
|
|
758
|
+
if data and data.get("full_text"):
|
|
759
|
+
result = Paper(
|
|
760
|
+
doi=doi,
|
|
761
|
+
url=paper.url,
|
|
762
|
+
source="elsevier_api",
|
|
763
|
+
title=data.get("title", "") or paper.title,
|
|
764
|
+
authors=data.get("authors", []) or paper.authors,
|
|
765
|
+
abstract=data.get("abstract", "") or paper.abstract,
|
|
766
|
+
full_text=data["full_text"],
|
|
767
|
+
)
|
|
768
|
+
logger.info("Elsevier API XML: %d chars of full text", len(data["full_text"]))
|
|
769
|
+
return result
|
|
770
|
+
|
|
771
|
+
pdf_bytes = elsevier_api.fetch_pdf(doi, api_key=api_key, inst_token=inst_token, config=self.config)
|
|
772
|
+
if pdf_bytes:
|
|
773
|
+
_apply_pdf_bytes(paper, pdf_bytes, doi, "elsevier_api", self._save_pdf)
|
|
774
|
+
logger.info("Elsevier API PDF: %d bytes", len(pdf_bytes))
|
|
775
|
+
return paper
|
|
776
|
+
|
|
777
|
+
return None
|
|
778
|
+
|
|
779
|
+
@staticmethod
|
|
780
|
+
def _build_publisher_pdf_url(doi: str, resolved_url: str) -> str | None:
|
|
781
|
+
profile = infer_publisher_profile_from_url(resolved_url) or infer_publisher_profile(doi)
|
|
782
|
+
if profile is None:
|
|
783
|
+
return None
|
|
784
|
+
candidates = build_pdf_candidates(profile, doi, source_url=resolved_url)
|
|
785
|
+
return candidates[0] if candidates else None
|
|
786
|
+
|
|
787
|
+
@staticmethod
|
|
788
|
+
def _extract_pdf_text(paper: Paper, pdf_path: str) -> None:
|
|
789
|
+
"""Try to extract full text from a saved PDF file into the paper."""
|
|
790
|
+
try:
|
|
791
|
+
paper.full_text = pdf_extractor.extract_from_bytes(Path(pdf_path).read_bytes())
|
|
792
|
+
except Exception:
|
|
793
|
+
pass
|
|
794
|
+
|
|
795
|
+
def _apply_extracted(self, paper: Paper, extracted: dict):
|
|
796
|
+
paper.title = paper.title or extracted.get("title", "")
|
|
797
|
+
paper.authors = paper.authors or extracted.get("authors", [])
|
|
798
|
+
paper.abstract = paper.abstract or extracted.get("abstract", "")
|
|
799
|
+
paper.full_text = extracted.get("full_text", "")
|
|
800
|
+
paper.figures = extracted.get("figures", [])
|
|
801
|
+
paper.references = extracted.get("references", [])
|
|
802
|
+
|
|
803
|
+
def _find_pdf_link(self, html: str, base_url: str) -> str | None:
|
|
804
|
+
candidates = discover_pdf_candidates_from_html(html, base_url)
|
|
805
|
+
if candidates:
|
|
806
|
+
return candidates[0]
|
|
807
|
+
|
|
808
|
+
from bs4 import BeautifulSoup
|
|
809
|
+
|
|
810
|
+
soup = BeautifulSoup(html, "lxml")
|
|
811
|
+
parsed = urlparse(base_url)
|
|
812
|
+
base = f"{parsed.scheme}://{parsed.netloc}"
|
|
813
|
+
hostname = parsed.netloc.lower()
|
|
814
|
+
|
|
815
|
+
meta_pdf = soup.find("meta", attrs={"name": "citation_pdf_url"})
|
|
816
|
+
if meta_pdf and meta_pdf.get("content"):
|
|
817
|
+
return self._resolve_url(meta_pdf["content"], base)
|
|
818
|
+
|
|
819
|
+
for a in soup.find_all("a", href=True):
|
|
820
|
+
href = a["href"]
|
|
821
|
+
text = a.get_text(strip=True).lower()
|
|
822
|
+
classes = " ".join(a.get("class", []))
|
|
823
|
+
|
|
824
|
+
if any(kw in text for kw in ["pdf", "download pdf", "full text pdf",
|
|
825
|
+
"view pdf", "get pdf"]):
|
|
826
|
+
return self._resolve_url(href, base)
|
|
827
|
+
if any(kw in classes for kw in ["pdf", "download-pdf", "pdf-download",
|
|
828
|
+
"article-pdf", "article__pdf"]):
|
|
829
|
+
return self._resolve_url(href, base)
|
|
830
|
+
if href.endswith(".pdf"):
|
|
831
|
+
return self._resolve_url(href, base)
|
|
832
|
+
if "/doi/pdf/" in href:
|
|
833
|
+
return self._resolve_url(href, base)
|
|
834
|
+
if "/doi/pdfdirect/" in href or "/doi/epdf/" in href:
|
|
835
|
+
return self._resolve_url(href, base)
|
|
836
|
+
|
|
837
|
+
path = parsed.path
|
|
838
|
+
if "pubs.acs.org" in hostname and "/doi/" in path and "/pdf/" not in path:
|
|
839
|
+
doi_part = path.split("/doi/")[-1]
|
|
840
|
+
if doi_part:
|
|
841
|
+
return f"{base}/doi/pdf/{doi_part}"
|
|
842
|
+
|
|
843
|
+
if "onlinelibrary.wiley.com" in hostname and "/doi/" in path and "/pdfdirect/" not in path:
|
|
844
|
+
doi_part = path.split("/doi/")[-1]
|
|
845
|
+
if doi_part:
|
|
846
|
+
return f"{base}/doi/pdfdirect/{doi_part}"
|
|
847
|
+
|
|
848
|
+
if "pubs.rsc.org" in hostname and "/articlelanding/" in path:
|
|
849
|
+
return base_url.replace("/articlelanding/", "/articlepdf/")
|
|
850
|
+
|
|
851
|
+
if "tandfonline.com" in hostname and "/doi/" in path and "/pdf/" not in path:
|
|
852
|
+
doi_part = re.sub(r"/doi/(?:full|abs)/", "/doi/pdf/", path)
|
|
853
|
+
if doi_part != path:
|
|
854
|
+
return f"{base}{doi_part}"
|
|
855
|
+
|
|
856
|
+
if ("elsevier.com" in hostname or "sciencedirect.com" in hostname):
|
|
857
|
+
pii_match = re.search(r"pii/([A-Z0-9]+)", path)
|
|
858
|
+
if pii_match:
|
|
859
|
+
pii = pii_match.group(1)
|
|
860
|
+
return f"https://www.sciencedirect.com/science/article/pii/{pii}/pdfft"
|
|
861
|
+
|
|
862
|
+
return None
|
|
863
|
+
|
|
864
|
+
def _resolve_url(self, href: str, base: str) -> str:
|
|
865
|
+
if href.startswith("http"):
|
|
866
|
+
return href
|
|
867
|
+
if href.startswith("//"):
|
|
868
|
+
return "https:" + href
|
|
869
|
+
if href.startswith("/"):
|
|
870
|
+
return base + href
|
|
871
|
+
return base + "/" + href
|
|
872
|
+
|
|
873
|
+
def _parse_doi(self, identifier: str) -> str | None:
|
|
874
|
+
identifier = identifier.strip()
|
|
875
|
+
|
|
876
|
+
if DOI_PATTERN.match(identifier):
|
|
877
|
+
return identifier
|
|
878
|
+
|
|
879
|
+
for prefix in ["https://doi.org/", "http://doi.org/", "https://dx.doi.org/"]:
|
|
880
|
+
if identifier.lower().startswith(prefix):
|
|
881
|
+
return identifier[len(prefix):]
|
|
882
|
+
|
|
883
|
+
doi_match = re.search(r"(10\.\d{4,9}/[^\s&?#]+)", identifier)
|
|
884
|
+
if doi_match:
|
|
885
|
+
return doi_match.group(1)
|
|
886
|
+
|
|
887
|
+
return None
|
|
888
|
+
|
|
889
|
+
def _parse_url(self, identifier: str) -> str | None:
|
|
890
|
+
identifier = identifier.strip()
|
|
891
|
+
return identifier if identifier.startswith("http") else None
|
|
892
|
+
|
|
893
|
+
def _resolve_doi(self, doi: str) -> str | None:
|
|
894
|
+
from ..network import user_agent
|
|
895
|
+
|
|
896
|
+
try:
|
|
897
|
+
resp = request_with_retry(
|
|
898
|
+
"GET",
|
|
899
|
+
f"https://doi.org/{doi}",
|
|
900
|
+
allow_redirects=True,
|
|
901
|
+
timeout=15,
|
|
902
|
+
headers={"User-Agent": user_agent()},
|
|
903
|
+
stream=True,
|
|
904
|
+
)
|
|
905
|
+
resp.close()
|
|
906
|
+
if resp.url and resp.url != f"https://doi.org/{doi}":
|
|
907
|
+
logger.info("Resolved DOI %s → %s (status=%d)", doi, resp.url, resp.status_code)
|
|
908
|
+
return resp.url
|
|
909
|
+
except requests.RequestException as e:
|
|
910
|
+
logger.warning("Failed to resolve DOI %s: %s", doi, e)
|
|
911
|
+
return None
|
|
912
|
+
|
|
913
|
+
def _rate_limit(self):
|
|
914
|
+
elapsed = time.time() - self._last_request_time
|
|
915
|
+
delay_min = float(self.config.get("request_delay_min", 2.0))
|
|
916
|
+
delay_max = float(self.config.get("request_delay_max", 5.0))
|
|
917
|
+
delay = random.uniform(delay_min, delay_max)
|
|
918
|
+
if elapsed < delay:
|
|
919
|
+
sleep_time = delay - elapsed
|
|
920
|
+
logger.debug("Rate limiting: sleeping %.1fs", sleep_time)
|
|
921
|
+
time.sleep(sleep_time)
|
|
922
|
+
self._last_request_time = time.time()
|
|
923
|
+
|
|
924
|
+
def _save_pdf(self, doi: str, pdf_bytes: bytes) -> Path | None:
|
|
925
|
+
output_dir = Path(self.config.get("output_dir", "."))
|
|
926
|
+
safe_name = re.sub(r"[^\w\-.]", "_", doi)
|
|
927
|
+
pdf_path = output_dir / f"{safe_name}.pdf"
|
|
928
|
+
try:
|
|
929
|
+
pdf_path.parent.mkdir(parents=True, exist_ok=True)
|
|
930
|
+
pdf_path.write_bytes(pdf_bytes)
|
|
931
|
+
logger.info("Saved PDF to %s", pdf_path)
|
|
932
|
+
return pdf_path
|
|
933
|
+
except OSError as e:
|
|
934
|
+
logger.error("Failed to save PDF: %s", e)
|
|
935
|
+
return None
|
|
936
|
+
|
|
937
|
+
def _cache_key(self, doi: str) -> Path:
|
|
938
|
+
h = hashlib.md5(doi.encode()).hexdigest()
|
|
939
|
+
cache_dir = Path(self.config.get("cache_dir", ".cache"))
|
|
940
|
+
return cache_dir / f"{h}.json"
|
|
941
|
+
|
|
942
|
+
def _load_cache(self, doi: str) -> Paper | None:
|
|
943
|
+
path = self._cache_key(doi)
|
|
944
|
+
if not path.exists():
|
|
945
|
+
return None
|
|
946
|
+
try:
|
|
947
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
948
|
+
return Paper.from_json(data)
|
|
949
|
+
except (json.JSONDecodeError, OSError) as e:
|
|
950
|
+
logger.warning("Failed to load cache for %s: %s", doi, e)
|
|
951
|
+
return None
|
|
952
|
+
|
|
953
|
+
def _save_cache(self, paper: Paper):
|
|
954
|
+
if not paper.doi:
|
|
955
|
+
return
|
|
956
|
+
if len(paper.full_text or "") < MIN_FULLTEXT_LEN:
|
|
957
|
+
return
|
|
958
|
+
path = self._cache_key(paper.doi)
|
|
959
|
+
try:
|
|
960
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
961
|
+
path.write_text(paper.to_json(), encoding="utf-8")
|
|
962
|
+
logger.info("Cached result for %s (%d chars)", paper.doi, len(paper.full_text or ""))
|
|
963
|
+
except OSError as e:
|
|
964
|
+
logger.warning("Failed to save cache for %s: %s", paper.doi, e)
|
|
965
|
+
|
|
966
|
+
def clear_cache(self):
|
|
967
|
+
cache_dir = Path(self.config.get("cache_dir", ".cache"))
|
|
968
|
+
if cache_dir.exists():
|
|
969
|
+
for f in cache_dir.glob("*.json"):
|
|
970
|
+
f.unlink()
|
|
971
|
+
logger.info("Cache cleared.")
|
|
972
|
+
|
|
973
|
+
def close(self):
|
|
974
|
+
if self._auth:
|
|
975
|
+
self._auth.close()
|