litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,975 @@
1
+ """Core paper fetching logic with multi-stage pipeline."""
2
+
3
+ from contextvars import ContextVar
4
+ import hashlib
5
+ import json
6
+ import logging
7
+ import random
8
+ import re
9
+ import time
10
+ from pathlib import Path
11
+ from urllib.parse import urlparse
12
+
13
+ import requests
14
+
15
+ from ..institutional.auth import EZProxyAuth, WebVPNAuth
16
+ from ..http_utils import request_with_retry
17
+ from ..extractors import html_extractor, pdf_extractor
18
+ from ..models import FetchResult, NextAction, Paper
19
+ from ..publishers.pdf_router import build_pdf_candidates, discover_pdf_candidates_from_html
20
+ from ..publishers.profiles import infer_publisher_profile, infer_publisher_profile_from_url
21
+
22
+ try:
23
+ from ..browser.backend import is_available as _browser_backend_available
24
+ _HAS_CLOAKBROWSER = _browser_backend_available()
25
+ except ImportError:
26
+ _HAS_CLOAKBROWSER = False
27
+
28
+ logger = logging.getLogger(__name__)
29
+
30
+ DOI_PATTERN = re.compile(r"^10\.\d{4,9}/[^\s]+$")
31
+
32
+ MIN_FULLTEXT_LEN = 1000
33
+
34
+ # Challenge keywords that indicate a Cloudflare/bot-detection page
35
+ _CHALLENGE_KEYWORDS = ("请稍候", "just a moment", "attention required", "verify", "security check")
36
+
37
+ _ATTEMPT_LOG: ContextVar[list[dict[str, str]] | None] = ContextVar(
38
+ "litloop_fetch_attempt_log",
39
+ default=None,
40
+ )
41
+
42
+
43
+ def _record_attempt(stage: str, status: str, reason: str = "", detail: str = "") -> None:
44
+ attempts = _ATTEMPT_LOG.get()
45
+ if attempts is None:
46
+ return
47
+ attempt = {"stage": stage, "status": status}
48
+ if reason:
49
+ attempt["reason"] = reason
50
+ if detail:
51
+ attempt["detail"] = detail
52
+ attempts.append(attempt)
53
+
54
+
55
+ def _record_paper_attempt(stage: str, paper: Paper | None) -> None:
56
+ if paper is None:
57
+ _record_attempt(stage, "miss", reason="no_result")
58
+ return
59
+ if len(paper.full_text or "") >= MIN_FULLTEXT_LEN:
60
+ _record_attempt(stage, "success", reason="full_text")
61
+ return
62
+ _record_attempt(stage, "partial", reason=_paper_quality(paper))
63
+
64
+
65
+ def _paper_quality(paper: Paper) -> str:
66
+ if paper.full_text:
67
+ return "short_text"
68
+ if paper.pdf_path:
69
+ return "pdf_only"
70
+ if paper.abstract:
71
+ return "abstract_only"
72
+ if paper.title or paper.authors or paper.journal or paper.year:
73
+ return "metadata_only"
74
+ return "none"
75
+
76
+
77
+ def _apply_attempt_diagnostics(result: FetchResult, identifier: str) -> None:
78
+ for attempt in result.attempts:
79
+ if (
80
+ attempt.get("stage") == "doi_resolve"
81
+ and attempt.get("status") == "miss"
82
+ and attempt.get("reason") == "no_url"
83
+ ):
84
+ query = (identifier or result.paper.title or result.paper.doi).replace('"', '\\"')
85
+ result.status = "blocked"
86
+ result.reason = "doi_resolution_failed"
87
+ result.next_action = NextAction(
88
+ kind="check_identifier",
89
+ command=f'litloop search "{query}"',
90
+ message=(
91
+ "DOI did not resolve to an article URL. Check the identifier or search "
92
+ "for the paper, then retry with a DOI or publisher URL."
93
+ ),
94
+ )
95
+ return
96
+
97
+ if (
98
+ result.status == "partial"
99
+ and result.next_action
100
+ and result.next_action.kind == "login"
101
+ and any(
102
+ attempt.get("stage") == "institutional_access"
103
+ and attempt.get("status") == "partial"
104
+ and attempt.get("reason") in {"none", "metadata_only", "abstract_only", "short_text"}
105
+ for attempt in result.attempts
106
+ )
107
+ ):
108
+ result.status = "auth_required"
109
+ result.reason = "institution_login_required"
110
+
111
+
112
+ def _is_good_result(paper: Paper | None) -> bool:
113
+ """Check if a paper result has enough full text to be useful."""
114
+ return bool(paper and len(paper.full_text or "") >= MIN_FULLTEXT_LEN)
115
+
116
+
117
+ def _copy_paper_metadata(paper: Paper, url: str) -> Paper:
118
+ """Create a new Paper copying metadata from an existing one, with a new URL."""
119
+ return Paper(
120
+ doi=paper.doi, title=paper.title, authors=paper.authors,
121
+ journal=paper.journal, year=paper.year, abstract=paper.abstract,
122
+ url=url,
123
+ )
124
+
125
+
126
+ def _apply_pdf_bytes(paper: Paper, pdf_bytes: bytes, doi: str, source: str,
127
+ save_fn=None) -> None:
128
+ """Extract text from PDF bytes and update paper fields in-place."""
129
+ paper.full_text = pdf_extractor.extract_from_bytes(pdf_bytes)
130
+ if save_fn:
131
+ pdf_path = save_fn(doi, pdf_bytes)
132
+ paper.pdf_path = str(pdf_path) if pdf_path else ""
133
+ paper.source = source
134
+
135
+
136
+ def _wait_for_challenge(page, max_tries: int = 6, *, current: str = "") -> None:
137
+ """Wait for a visible browser challenge and expose it in the progress bar."""
138
+ reporter = None
139
+ attention_key = f"browser-challenge:{id(page)}:{current}"
140
+ attention_open = False
141
+ try:
142
+ from . import progress as reporter
143
+ except Exception:
144
+ reporter = None
145
+ try:
146
+ for i in range(max_tries):
147
+ title = page.title().lower()
148
+ if any(sig in title for sig in _CHALLENGE_KEYWORDS):
149
+ if not attention_open and reporter is not None:
150
+ reporter.set_attention(
151
+ attention_key,
152
+ "请在浏览器窗口完成安全验证",
153
+ current=current,
154
+ phase="人工验证",
155
+ )
156
+ attention_open = True
157
+ logger.info("Challenge page detected, waiting... (%d/%d)", i + 1, max_tries)
158
+ time.sleep(5)
159
+ else:
160
+ break
161
+ finally:
162
+ if attention_open and reporter is not None:
163
+ reporter.clear_attention(attention_key)
164
+
165
+
166
+ class PaperFetcher:
167
+ """Main class for fetching academic papers via multi-stage pipeline."""
168
+
169
+ def __init__(self, config: dict | None = None):
170
+ from ..config import load_config
171
+ self.config = config or load_config()
172
+ self._auth: WebVPNAuth | EZProxyAuth | None = None
173
+ self._last_request_time = 0.0
174
+
175
+ def _challenge_max_tries(self) -> int:
176
+ """Keep a challenged DOI open long enough for a human to act."""
177
+ try:
178
+ seconds = int(self.config.get("challenge_wait_seconds", 900) or 900)
179
+ except (AttributeError, TypeError, ValueError):
180
+ seconds = 900
181
+ return max(6, (max(0, seconds) + 4) // 5)
182
+
183
+ @property
184
+ def auth(self) -> WebVPNAuth | EZProxyAuth:
185
+ if self._auth is None:
186
+ from ..institutional.schools import get_school
187
+ school_name = (self.config.get("instsci_school", "")
188
+ or self.config.get("vpnsci_school", "")).strip()
189
+ if not school_name:
190
+ # Raised deliberately: fetch_with_result() converts ValueError
191
+ # into a structured config_needed outcome. get_school("") would
192
+ # otherwise fuzzy-match an arbitrary school and the cascade
193
+ # would navigate to a garbage/empty gateway URL.
194
+ raise ValueError(
195
+ "no institution configured (instsci_school/vpnsci_school are empty)"
196
+ )
197
+ entry = get_school(school_name)
198
+ if entry.school_type == "ezproxy":
199
+ self._auth = EZProxyAuth(self.config, proxy_base=entry.host)
200
+ else:
201
+ # Fall back to the school database gateway host when no
202
+ # explicit base URL is configured.
203
+ self._auth = WebVPNAuth(self.config, key=entry.key, iv=entry.iv,
204
+ base_url=getattr(entry, "host", "") or "")
205
+ return self._auth
206
+
207
+ def fetch(self, identifier: str, use_cache: bool = True) -> Paper:
208
+ """Fetch a paper by DOI or URL."""
209
+ doi = self._parse_doi(identifier)
210
+ url = self._parse_url(identifier)
211
+
212
+ if use_cache and doi:
213
+ cached = self._load_cache(doi)
214
+ if _is_good_result(cached):
215
+ _record_paper_attempt("cache", cached)
216
+ logger.info("Loaded from cache (good full text): %s", doi)
217
+ return cached
218
+ elif cached:
219
+ _record_paper_attempt("cache", cached)
220
+ logger.info("Cache hit but full text too short (%d chars), re-fetching: %s",
221
+ len(cached.full_text or ""), doi)
222
+ else:
223
+ _record_attempt("cache", "miss", reason="not_found")
224
+
225
+ paper = Paper(doi=doi or "", url=url or "")
226
+
227
+ # Helper: record attempt, cache if good, return whether result is usable
228
+ def _check(stage: str, result: Paper | None) -> bool:
229
+ _record_paper_attempt(stage, result)
230
+ if _is_good_result(result):
231
+ self._save_cache(result)
232
+ return True
233
+ return False
234
+
235
+ # Step 1: Try Open Access sources first
236
+ if doi:
237
+ oa_paper = self._try_open_access(doi, identifier)
238
+ if _check("open_access", oa_paper):
239
+ return oa_paper
240
+ if oa_paper:
241
+ paper = oa_paper
242
+
243
+ # Step 2: Try Elsevier API before institutional flows
244
+ if doi and not paper.pdf_path:
245
+ api_paper = self._try_elsevier_api(doi, paper)
246
+ if _check("elsevier_api", api_paper):
247
+ return api_paper
248
+
249
+ # Step 3: Resolve DOI to URL if needed
250
+ if doi and not url:
251
+ url = self._resolve_doi(doi)
252
+ _record_attempt("doi_resolve", "success" if url else "miss",
253
+ detail=url or "", reason="" if url else "no_url")
254
+ paper.url = url or ""
255
+
256
+ if not url:
257
+ logger.error("Could not determine URL for: %s", identifier)
258
+ return paper
259
+
260
+ # Step 4: Try CARSI federated access
261
+ if self.config.get("carsi_enabled") and doi:
262
+ carsi_paper = self._try_carsi_pdf(doi, url, paper)
263
+ if _check("carsi_pdf", carsi_paper):
264
+ return carsi_paper
265
+ carsi_paper = self._try_carsi_html(url, paper)
266
+ if _check("carsi_html", carsi_paper):
267
+ return carsi_paper
268
+
269
+ # Step 5: Try direct publisher PDF URL construction
270
+ if doi and not paper.pdf_path:
271
+ pdf_paper = self._try_publisher_pdf(doi, url, paper)
272
+ if _check("publisher_pdf", pdf_paper):
273
+ return pdf_paper
274
+
275
+ # Step 6: Try browser-based PDF download
276
+ if doi and not paper.pdf_path:
277
+ browser_paper = self._try_browser_pdf_download(doi, url, paper)
278
+ if _check("browser_pdf", browser_paper):
279
+ return browser_paper
280
+
281
+ # Step 7: Fetch via institutional campus access
282
+ self._rate_limit()
283
+ try:
284
+ paper = self._fetch_via_webvpn(url, paper)
285
+ except ValueError:
286
+ _record_attempt("institutional_access", "error", reason="config_needed")
287
+ raise
288
+ except requests.RequestException:
289
+ _record_attempt("institutional_access", "error", reason="gateway_unreachable")
290
+ raise
291
+ _record_paper_attempt("institutional_access", paper)
292
+
293
+ if _is_good_result(paper):
294
+ self._save_cache(paper)
295
+
296
+ return paper
297
+
298
+ def fetch_with_result(self, identifier: str, use_cache: bool = True) -> FetchResult:
299
+ """Fetch a paper and return a structured, agent-friendly outcome."""
300
+ attempts: list[dict[str, str]] = []
301
+ token = _ATTEMPT_LOG.set(attempts)
302
+ try:
303
+ paper = self.fetch(identifier, use_cache=use_cache)
304
+ except ValueError as exc:
305
+ doi = self._parse_doi(identifier) or ""
306
+ url = self._parse_url(identifier) or ""
307
+ return FetchResult(
308
+ status="config_needed",
309
+ quality="none",
310
+ reason="institution_not_configured",
311
+ paper=Paper(doi=doi, url=url),
312
+ next_action=NextAction(
313
+ kind="configure_institution",
314
+ command="litloop config set instsci_school YOUR_SCHOOL",
315
+ message=f"Configure your school or institution before retrying. Detail: {exc}",
316
+ ),
317
+ attempts=attempts,
318
+ )
319
+ except requests.RequestException as exc:
320
+ doi = self._parse_doi(identifier) or ""
321
+ url = self._parse_url(identifier) or ""
322
+ gateway_error = any(
323
+ a.get("stage") == "institutional_access"
324
+ and a.get("status") == "error"
325
+ and a.get("reason") == "gateway_unreachable"
326
+ for a in attempts
327
+ )
328
+ reason = "gateway_unreachable" if gateway_error else "network_error"
329
+ kind = "diagnose_gateway" if gateway_error else "diagnose"
330
+ message = (
331
+ "Institutional gateway could not be reached. Check VPN/proxy/CARSI/WebVPN "
332
+ f"configuration, then retry. Detail: {exc}"
333
+ if gateway_error
334
+ else f"Check network and institutional access configuration. Detail: {exc}"
335
+ )
336
+ return FetchResult(
337
+ status="blocked",
338
+ quality="none",
339
+ reason=reason,
340
+ paper=Paper(doi=doi, url=url),
341
+ next_action=NextAction(
342
+ kind=kind,
343
+ command="litloop config show",
344
+ message=message,
345
+ ),
346
+ attempts=attempts,
347
+ )
348
+ finally:
349
+ _ATTEMPT_LOG.reset(token)
350
+
351
+ result = FetchResult.from_paper(
352
+ paper,
353
+ min_fulltext_len=MIN_FULLTEXT_LEN,
354
+ institution_configured=self._institution_configured(),
355
+ identifier=identifier,
356
+ )
357
+ result.attempts = attempts
358
+ _apply_attempt_diagnostics(result, identifier)
359
+ return result
360
+
361
+ def _institution_configured(self) -> bool:
362
+ return bool(
363
+ self.config.get("instsci_school")
364
+ or self.config.get("instsci_base_url")
365
+ or self.config.get("vpnsci_base_url")
366
+ or self.config.get("ezproxy_login_url")
367
+ or self.config.get("network_proxy")
368
+ or (self.config.get("carsi_enabled") and self.config.get("carsi_idp_name"))
369
+ )
370
+
371
+ def _try_open_access(self, doi: str, identifier: str = "") -> Paper | None:
372
+ """Try to fetch paper from Open Access sources."""
373
+ from ..sources import unpaywall, arxiv
374
+ from ..identifiers import normalize_arxiv_id
375
+
376
+ output_dir = Path(self.config.get("output_dir", "."))
377
+ output_dir.mkdir(parents=True, exist_ok=True)
378
+
379
+ # Check if it's an arXiv paper first
380
+ arxiv_id = normalize_arxiv_id(identifier or doi)
381
+ if arxiv_id:
382
+ logger.info("Fetching from arXiv: %s", arxiv_id)
383
+ safe_id = arxiv_id.replace("/", "_")
384
+ output_path = output_dir / f"arxiv_{safe_id}.pdf"
385
+ result = arxiv.try_arxiv(arxiv_id, output_path, self.config)
386
+ if result and result.get("path"):
387
+ paper = Paper(
388
+ doi=doi,
389
+ url=result.get("url", f"https://arxiv.org/abs/{arxiv_id}"),
390
+ source="arxiv",
391
+ pdf_path=str(result["path"]),
392
+ )
393
+ self._extract_pdf_text(paper, result["path"])
394
+ return paper
395
+
396
+ # Try Unpaywall
397
+ logger.info("Checking Unpaywall for OA version of %s...", doi)
398
+ safe_doi = re.sub(r"[^\w\-.]", "_", doi)
399
+ output_path = output_dir / f"unpaywall_{safe_doi}.pdf"
400
+ result = unpaywall.try_unpaywall(doi, output_path, self.config)
401
+ if result and result.get("path"):
402
+ paper = Paper(
403
+ doi=doi,
404
+ url=result.get("url", ""),
405
+ source="open_access",
406
+ pdf_path=str(result["path"]),
407
+ )
408
+ self._extract_pdf_text(paper, result["path"])
409
+ return paper
410
+
411
+ return Paper(doi=doi)
412
+
413
+ def _try_publisher_pdf(self, doi: str, resolved_url: str, paper: Paper) -> Paper | None:
414
+ pdf_url = self._build_publisher_pdf_url(doi, resolved_url)
415
+ if not pdf_url:
416
+ return None
417
+
418
+ logger.info("Trying constructed publisher PDF URL: %s", pdf_url)
419
+
420
+ if not self.auth.login():
421
+ logger.error("Institutional access authentication failed.")
422
+ return None
423
+
424
+ self._rate_limit()
425
+ try:
426
+ resp = self.auth.fetch(pdf_url)
427
+ resp.raise_for_status()
428
+ ct = resp.headers.get("content-type", "").lower()
429
+
430
+ if "pdf" in ct and len(resp.content) > 10000:
431
+ _apply_pdf_bytes(paper, resp.content, doi, "institutional", self._save_pdf)
432
+ logger.info("Publisher PDF downloaded (%d bytes, %d chars text)",
433
+ len(resp.content), len(paper.full_text or ""))
434
+ return paper
435
+ else:
436
+ logger.info("Publisher PDF URL returned non-PDF or too small (ct=%s, size=%d)",
437
+ ct, len(resp.content))
438
+ except requests.RequestException as e:
439
+ logger.warning("Failed to fetch publisher PDF: %s", e)
440
+
441
+ return None
442
+
443
+ def _try_browser_pdf_download(self, doi: str, resolved_url: str, paper: Paper) -> Paper | None:
444
+ if not _HAS_CLOAKBROWSER:
445
+ return None
446
+
447
+ pdf_url = self._build_publisher_pdf_url(doi, resolved_url)
448
+ if not pdf_url:
449
+ return None
450
+
451
+ print(" [Browser] Downloading PDF via browser...")
452
+
453
+ live_context = getattr(self.auth, 'browser_context', None)
454
+ if live_context:
455
+ result = self._browser_pdf_download(live_context, resolved_url, doi, paper)
456
+ if result:
457
+ return result
458
+
459
+ logger.info("No live browser session. Triggering browser login for PDF download...")
460
+ if self.auth.login(force=True):
461
+ live_context = getattr(self.auth, 'browser_context', None)
462
+ if live_context:
463
+ result = self._browser_pdf_download(live_context, resolved_url, doi, paper)
464
+ if result:
465
+ return result
466
+
467
+ return None
468
+
469
+ def _browser_pdf_download(self, context, article_url: str, doi: str, paper: Paper) -> Paper | None:
470
+ page = context.new_page()
471
+ try:
472
+ return self._browser_pdf_do(page, article_url, doi, paper)
473
+ finally:
474
+ try:
475
+ page.close()
476
+ except Exception:
477
+ pass
478
+
479
+ def _browser_pdf_do(self, page, article_url: str, doi: str, paper: Paper) -> Paper | None:
480
+ try:
481
+ proxied_article = self.auth.convert_url(article_url)
482
+ except Exception:
483
+ proxied_article = article_url
484
+
485
+ logger.info("Browser: navigating to article page %s", proxied_article[:80])
486
+ try:
487
+ page.goto(proxied_article, wait_until="domcontentloaded", timeout=60000)
488
+ except Exception as e:
489
+ logger.warning("Navigation failed: %s", e)
490
+
491
+ challenge_tries = self._challenge_max_tries()
492
+ _wait_for_challenge(page, max_tries=challenge_tries, current=doi)
493
+
494
+ current_url = page.url
495
+ logger.info("Browser article page: url=%s, title=%s", current_url[:60], page.title()[:40])
496
+
497
+ if "cas" in current_url.lower() or "login" in current_url.lower() or \
498
+ "登录" in page.title() or "身份" in page.title():
499
+ logger.info("Browser landed on CAS page")
500
+ return None
501
+
502
+ if "linkinghub" in current_url or "retrieve/pii" in current_url:
503
+ logger.info("On linkinghub redirect page, waiting for redirect...")
504
+ time.sleep(5)
505
+ current_url = page.url
506
+ if "linkinghub" in current_url:
507
+ pii_match = re.search(r"pii/([A-Z0-9]+)", current_url)
508
+ if pii_match:
509
+ direct_url = f"https://www.sciencedirect.com/science/article/pii/{pii_match.group(1)}"
510
+ try:
511
+ proxied_direct = self.auth.convert_url(direct_url)
512
+ except Exception:
513
+ proxied_direct = direct_url
514
+ try:
515
+ page.goto(proxied_direct, wait_until="domcontentloaded", timeout=30000)
516
+ time.sleep(3)
517
+ except Exception:
518
+ pass
519
+ _wait_for_challenge(page, max_tries=challenge_tries, current=doi)
520
+ current_url = page.url
521
+
522
+ try:
523
+ page.wait_for_load_state("networkidle", timeout=15000)
524
+ except Exception:
525
+ pass
526
+
527
+ pdf_link = page.evaluate("""
528
+ (() => {
529
+ for (const meta of document.querySelectorAll('meta[name="citation_pdf_url"]')) {
530
+ if (meta.content) return meta.content;
531
+ }
532
+ for (const a of document.querySelectorAll('a[aria-label*="PDF" i], a[title*="PDF" i]')) {
533
+ if (a.href) return a.href;
534
+ }
535
+ for (const a of document.querySelectorAll('a')) {
536
+ const href = (a.href || '').toLowerCase();
537
+ const text = (a.textContent || '').toLowerCase();
538
+ if (href.includes('/pdfft') || href.includes('/pdfdirect')) return a.href;
539
+ if (href.includes('pdf') && !href.includes('supplement') && !href.includes('cite')) return a.href;
540
+ if (text.includes('pdf') && (text.includes('download') || text.includes('view'))) return a.href;
541
+ }
542
+ return null;
543
+ })()
544
+ """)
545
+
546
+ if not pdf_link or not isinstance(pdf_link, str):
547
+ logger.info("No PDF link found on article page")
548
+ return None
549
+
550
+ logger.info("Found PDF link on page: %s", pdf_link[:80])
551
+
552
+ if pdf_link.startswith("/"):
553
+ parsed = urlparse(current_url)
554
+ pdf_link = f"{parsed.scheme}://{parsed.netloc}{pdf_link}"
555
+
556
+ captured_pdf = {"bytes": None}
557
+
558
+ def _on_response(response):
559
+ try:
560
+ ct = response.headers.get("content-type", "")
561
+ if "pdf" in ct or "octet" in ct:
562
+ body = response.body()
563
+ if body[:5] == b"%PDF-":
564
+ captured_pdf["bytes"] = body
565
+ except Exception:
566
+ pass
567
+
568
+ page.on("response", _on_response)
569
+
570
+ pdf_paths = self._build_browser_pdf_paths(doi, current_url)
571
+ all_urls = [pdf_link]
572
+ _parsed = urlparse(current_url)
573
+ _origin = f"{_parsed.scheme}://{_parsed.netloc}"
574
+ for p in pdf_paths:
575
+ full = _origin + p if p.startswith("/") else p
576
+ if full not in all_urls:
577
+ all_urls.append(full)
578
+
579
+ for pdf_url in all_urls:
580
+ if captured_pdf["bytes"]:
581
+ break
582
+ logger.info("Navigating to PDF URL: %s", pdf_url[:80])
583
+ try:
584
+ page.goto(pdf_url, wait_until="domcontentloaded", timeout=30000)
585
+ except Exception as e:
586
+ logger.debug("Navigation to %s failed: %s", pdf_url[:60], e)
587
+ continue
588
+
589
+ _wait_for_challenge(page, max_tries=challenge_tries, current=doi)
590
+
591
+ if not captured_pdf["bytes"]:
592
+ try:
593
+ page.wait_for_load_state("networkidle", timeout=10000)
594
+ except Exception:
595
+ pass
596
+ try:
597
+ pdf_from_viewer = page.evaluate("""
598
+ () => {
599
+ const obj = document.querySelector('object[type="application/pdf"]');
600
+ if (obj && obj.data) return obj.data;
601
+ const embed = document.querySelector('embed[type="application/pdf"]');
602
+ if (embed && embed.src) return embed.src;
603
+ const iframe = document.querySelector('iframe[src*="pdf"]');
604
+ if (iframe && iframe.src) return iframe.src;
605
+ return null;
606
+ }
607
+ """)
608
+ if pdf_from_viewer and isinstance(pdf_from_viewer, str):
609
+ if pdf_from_viewer.startswith("/"):
610
+ pdf_from_viewer = _origin + pdf_from_viewer
611
+ try:
612
+ page.goto(pdf_from_viewer, wait_until="domcontentloaded", timeout=30000)
613
+ time.sleep(2)
614
+ except Exception:
615
+ pass
616
+ except Exception:
617
+ pass
618
+
619
+ if not captured_pdf["bytes"]:
620
+ final_title = page.title().lower()
621
+ if "robot" in final_title or "captcha" in final_title:
622
+ logger.warning("Anti-bot detected on PDF page, trying next URL")
623
+ continue
624
+
625
+ page.remove_listener("response", _on_response)
626
+
627
+ pdf_bytes = captured_pdf["bytes"]
628
+ if pdf_bytes and pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
629
+ _apply_pdf_bytes(paper, pdf_bytes, doi, "browser", self._save_pdf)
630
+ logger.info("Browser PDF downloaded via response capture (%d bytes)", len(pdf_bytes))
631
+ return paper
632
+
633
+ logger.info("Could not capture PDF from browser navigation")
634
+ return None
635
+
636
+ @staticmethod
637
+ def _build_browser_pdf_paths(doi: str, current_url: str) -> list[str]:
638
+ parsed = urlparse(current_url)
639
+ doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
640
+
641
+ paths = []
642
+ if "sciencedirect" in parsed.netloc or "elsevier" in parsed.netloc:
643
+ pii_match = re.search(r"pii/([A-Z0-9]+)", current_url)
644
+ if pii_match:
645
+ pii = pii_match.group(1)
646
+ paths.append(f"/science/article/pii/{pii}/pdfft")
647
+ paths.append(f"/doi/pdfdirect/{doi}")
648
+ elif "springer" in parsed.netloc:
649
+ paths.append(f"/content/pdf/{doi}.pdf")
650
+ elif "wiley" in parsed.netloc:
651
+ paths.append(f"/doi/pdfdirect/{doi}")
652
+ paths.append(f"/doi/pdf/{doi}")
653
+ paths.append(f"/doi/pdf/{doi}")
654
+ return paths
655
+
656
+ def _try_carsi_pdf(self, doi: str, resolved_url: str, paper: Paper) -> Paper | None:
657
+ from ..institutional.carsi_login import CARSIClient
658
+
659
+ pdf_url = self._build_publisher_pdf_url(doi, resolved_url)
660
+ if not pdf_url:
661
+ return None
662
+
663
+ logger.info("Trying CARSI publisher PDF: %s", pdf_url)
664
+ self._rate_limit()
665
+ try:
666
+ carsi = CARSIClient(self.config)
667
+ resp = carsi.fetch(pdf_url)
668
+ resp.raise_for_status()
669
+ ct = resp.headers.get("content-type", "").lower()
670
+ if "pdf" in ct and len(resp.content) > 10000:
671
+ result = _copy_paper_metadata(paper, pdf_url)
672
+ _apply_pdf_bytes(result, resp.content, doi, "carsi", self._save_pdf)
673
+ logger.info("CARSI PDF downloaded (%d bytes)", len(resp.content))
674
+ return result
675
+ except Exception as e:
676
+ logger.warning("CARSI PDF failed: %s", e)
677
+ return None
678
+
679
+ def _try_carsi_html(self, url: str, paper: Paper) -> Paper | None:
680
+ from ..institutional.carsi_login import CARSIClient
681
+
682
+ logger.info("Trying CARSI HTML: %s", url)
683
+ self._rate_limit()
684
+ try:
685
+ carsi = CARSIClient(self.config)
686
+ resp = carsi.fetch(url)
687
+ resp.raise_for_status()
688
+ ct = resp.headers.get("content-type", "").lower()
689
+ if "pdf" in ct:
690
+ result = _copy_paper_metadata(paper, url)
691
+ _apply_pdf_bytes(result, resp.content, paper.doi or "unknown", "carsi",
692
+ self._save_pdf if paper.doi else None)
693
+ return result
694
+
695
+ result = _copy_paper_metadata(paper, url)
696
+ self._apply_extracted(result, html_extractor.extract(resp.text, url))
697
+ result.source = "carsi"
698
+ if _is_good_result(result):
699
+ return result
700
+ except Exception as e:
701
+ logger.warning("CARSI HTML failed: %s", e)
702
+ return None
703
+
704
+ def _fetch_via_webvpn(self, url: str, paper: Paper) -> Paper:
705
+ if not self.auth.login():
706
+ logger.error("Institutional access authentication failed.")
707
+ return paper
708
+
709
+ paper.source = "institutional"
710
+
711
+ try:
712
+ resp = self.auth.fetch(url)
713
+ resp.raise_for_status()
714
+ except requests.RequestException as e:
715
+ logger.error("Failed to fetch via institutional access: %s", e)
716
+ return paper
717
+
718
+ if "pdf" in resp.headers.get("content-type", "").lower():
719
+ _apply_pdf_bytes(paper, resp.content, paper.doi or "unknown", "institutional",
720
+ self._save_pdf)
721
+ return paper
722
+
723
+ self._apply_extracted(paper, html_extractor.extract(resp.text, resp.url))
724
+
725
+ pdf_url = self._find_pdf_link(resp.text, resp.url)
726
+ if pdf_url:
727
+ logger.info("Found PDF link in HTML, downloading: %s", pdf_url)
728
+ self._rate_limit()
729
+ try:
730
+ pdf_resp = self.auth.fetch(pdf_url)
731
+ pdf_resp.raise_for_status()
732
+ ct = pdf_resp.headers.get("content-type", "").lower()
733
+ if "pdf" in ct and len(pdf_resp.content) > 10000:
734
+ pdf_path = self._save_pdf(paper.doi or "unknown", pdf_resp.content)
735
+ paper.pdf_path = str(pdf_path) if pdf_path else ""
736
+ if not _is_good_result(paper):
737
+ paper.full_text = pdf_extractor.extract_from_bytes(pdf_resp.content)
738
+ except requests.RequestException as e:
739
+ logger.warning("Failed to download PDF: %s", e)
740
+
741
+ return paper
742
+
743
+ def _try_elsevier_api(self, doi: str, paper: Paper) -> Paper | None:
744
+ from ..sources import elsevier_api
745
+
746
+ api_key = elsevier_api.get_api_key(self.config.get("elsevier_api_key", ""))
747
+ if not api_key:
748
+ logger.debug("No Elsevier API key configured")
749
+ return None
750
+
751
+ if not doi.startswith("10.1016/"):
752
+ return None
753
+
754
+ logger.info("Trying Elsevier API for %s", doi)
755
+ inst_token = self.config.get("elsevier_insttoken", "")
756
+
757
+ data = elsevier_api.fetch_fulltext(doi, api_key=api_key, inst_token=inst_token, config=self.config)
758
+ if data and data.get("full_text"):
759
+ result = Paper(
760
+ doi=doi,
761
+ url=paper.url,
762
+ source="elsevier_api",
763
+ title=data.get("title", "") or paper.title,
764
+ authors=data.get("authors", []) or paper.authors,
765
+ abstract=data.get("abstract", "") or paper.abstract,
766
+ full_text=data["full_text"],
767
+ )
768
+ logger.info("Elsevier API XML: %d chars of full text", len(data["full_text"]))
769
+ return result
770
+
771
+ pdf_bytes = elsevier_api.fetch_pdf(doi, api_key=api_key, inst_token=inst_token, config=self.config)
772
+ if pdf_bytes:
773
+ _apply_pdf_bytes(paper, pdf_bytes, doi, "elsevier_api", self._save_pdf)
774
+ logger.info("Elsevier API PDF: %d bytes", len(pdf_bytes))
775
+ return paper
776
+
777
+ return None
778
+
779
+ @staticmethod
780
+ def _build_publisher_pdf_url(doi: str, resolved_url: str) -> str | None:
781
+ profile = infer_publisher_profile_from_url(resolved_url) or infer_publisher_profile(doi)
782
+ if profile is None:
783
+ return None
784
+ candidates = build_pdf_candidates(profile, doi, source_url=resolved_url)
785
+ return candidates[0] if candidates else None
786
+
787
+ @staticmethod
788
+ def _extract_pdf_text(paper: Paper, pdf_path: str) -> None:
789
+ """Try to extract full text from a saved PDF file into the paper."""
790
+ try:
791
+ paper.full_text = pdf_extractor.extract_from_bytes(Path(pdf_path).read_bytes())
792
+ except Exception:
793
+ pass
794
+
795
+ def _apply_extracted(self, paper: Paper, extracted: dict):
796
+ paper.title = paper.title or extracted.get("title", "")
797
+ paper.authors = paper.authors or extracted.get("authors", [])
798
+ paper.abstract = paper.abstract or extracted.get("abstract", "")
799
+ paper.full_text = extracted.get("full_text", "")
800
+ paper.figures = extracted.get("figures", [])
801
+ paper.references = extracted.get("references", [])
802
+
803
+ def _find_pdf_link(self, html: str, base_url: str) -> str | None:
804
+ candidates = discover_pdf_candidates_from_html(html, base_url)
805
+ if candidates:
806
+ return candidates[0]
807
+
808
+ from bs4 import BeautifulSoup
809
+
810
+ soup = BeautifulSoup(html, "lxml")
811
+ parsed = urlparse(base_url)
812
+ base = f"{parsed.scheme}://{parsed.netloc}"
813
+ hostname = parsed.netloc.lower()
814
+
815
+ meta_pdf = soup.find("meta", attrs={"name": "citation_pdf_url"})
816
+ if meta_pdf and meta_pdf.get("content"):
817
+ return self._resolve_url(meta_pdf["content"], base)
818
+
819
+ for a in soup.find_all("a", href=True):
820
+ href = a["href"]
821
+ text = a.get_text(strip=True).lower()
822
+ classes = " ".join(a.get("class", []))
823
+
824
+ if any(kw in text for kw in ["pdf", "download pdf", "full text pdf",
825
+ "view pdf", "get pdf"]):
826
+ return self._resolve_url(href, base)
827
+ if any(kw in classes for kw in ["pdf", "download-pdf", "pdf-download",
828
+ "article-pdf", "article__pdf"]):
829
+ return self._resolve_url(href, base)
830
+ if href.endswith(".pdf"):
831
+ return self._resolve_url(href, base)
832
+ if "/doi/pdf/" in href:
833
+ return self._resolve_url(href, base)
834
+ if "/doi/pdfdirect/" in href or "/doi/epdf/" in href:
835
+ return self._resolve_url(href, base)
836
+
837
+ path = parsed.path
838
+ if "pubs.acs.org" in hostname and "/doi/" in path and "/pdf/" not in path:
839
+ doi_part = path.split("/doi/")[-1]
840
+ if doi_part:
841
+ return f"{base}/doi/pdf/{doi_part}"
842
+
843
+ if "onlinelibrary.wiley.com" in hostname and "/doi/" in path and "/pdfdirect/" not in path:
844
+ doi_part = path.split("/doi/")[-1]
845
+ if doi_part:
846
+ return f"{base}/doi/pdfdirect/{doi_part}"
847
+
848
+ if "pubs.rsc.org" in hostname and "/articlelanding/" in path:
849
+ return base_url.replace("/articlelanding/", "/articlepdf/")
850
+
851
+ if "tandfonline.com" in hostname and "/doi/" in path and "/pdf/" not in path:
852
+ doi_part = re.sub(r"/doi/(?:full|abs)/", "/doi/pdf/", path)
853
+ if doi_part != path:
854
+ return f"{base}{doi_part}"
855
+
856
+ if ("elsevier.com" in hostname or "sciencedirect.com" in hostname):
857
+ pii_match = re.search(r"pii/([A-Z0-9]+)", path)
858
+ if pii_match:
859
+ pii = pii_match.group(1)
860
+ return f"https://www.sciencedirect.com/science/article/pii/{pii}/pdfft"
861
+
862
+ return None
863
+
864
+ def _resolve_url(self, href: str, base: str) -> str:
865
+ if href.startswith("http"):
866
+ return href
867
+ if href.startswith("//"):
868
+ return "https:" + href
869
+ if href.startswith("/"):
870
+ return base + href
871
+ return base + "/" + href
872
+
873
+ def _parse_doi(self, identifier: str) -> str | None:
874
+ identifier = identifier.strip()
875
+
876
+ if DOI_PATTERN.match(identifier):
877
+ return identifier
878
+
879
+ for prefix in ["https://doi.org/", "http://doi.org/", "https://dx.doi.org/"]:
880
+ if identifier.lower().startswith(prefix):
881
+ return identifier[len(prefix):]
882
+
883
+ doi_match = re.search(r"(10\.\d{4,9}/[^\s&?#]+)", identifier)
884
+ if doi_match:
885
+ return doi_match.group(1)
886
+
887
+ return None
888
+
889
+ def _parse_url(self, identifier: str) -> str | None:
890
+ identifier = identifier.strip()
891
+ return identifier if identifier.startswith("http") else None
892
+
893
+ def _resolve_doi(self, doi: str) -> str | None:
894
+ from ..network import user_agent
895
+
896
+ try:
897
+ resp = request_with_retry(
898
+ "GET",
899
+ f"https://doi.org/{doi}",
900
+ allow_redirects=True,
901
+ timeout=15,
902
+ headers={"User-Agent": user_agent()},
903
+ stream=True,
904
+ )
905
+ resp.close()
906
+ if resp.url and resp.url != f"https://doi.org/{doi}":
907
+ logger.info("Resolved DOI %s → %s (status=%d)", doi, resp.url, resp.status_code)
908
+ return resp.url
909
+ except requests.RequestException as e:
910
+ logger.warning("Failed to resolve DOI %s: %s", doi, e)
911
+ return None
912
+
913
+ def _rate_limit(self):
914
+ elapsed = time.time() - self._last_request_time
915
+ delay_min = float(self.config.get("request_delay_min", 2.0))
916
+ delay_max = float(self.config.get("request_delay_max", 5.0))
917
+ delay = random.uniform(delay_min, delay_max)
918
+ if elapsed < delay:
919
+ sleep_time = delay - elapsed
920
+ logger.debug("Rate limiting: sleeping %.1fs", sleep_time)
921
+ time.sleep(sleep_time)
922
+ self._last_request_time = time.time()
923
+
924
+ def _save_pdf(self, doi: str, pdf_bytes: bytes) -> Path | None:
925
+ output_dir = Path(self.config.get("output_dir", "."))
926
+ safe_name = re.sub(r"[^\w\-.]", "_", doi)
927
+ pdf_path = output_dir / f"{safe_name}.pdf"
928
+ try:
929
+ pdf_path.parent.mkdir(parents=True, exist_ok=True)
930
+ pdf_path.write_bytes(pdf_bytes)
931
+ logger.info("Saved PDF to %s", pdf_path)
932
+ return pdf_path
933
+ except OSError as e:
934
+ logger.error("Failed to save PDF: %s", e)
935
+ return None
936
+
937
+ def _cache_key(self, doi: str) -> Path:
938
+ h = hashlib.md5(doi.encode()).hexdigest()
939
+ cache_dir = Path(self.config.get("cache_dir", ".cache"))
940
+ return cache_dir / f"{h}.json"
941
+
942
+ def _load_cache(self, doi: str) -> Paper | None:
943
+ path = self._cache_key(doi)
944
+ if not path.exists():
945
+ return None
946
+ try:
947
+ data = json.loads(path.read_text(encoding="utf-8"))
948
+ return Paper.from_json(data)
949
+ except (json.JSONDecodeError, OSError) as e:
950
+ logger.warning("Failed to load cache for %s: %s", doi, e)
951
+ return None
952
+
953
+ def _save_cache(self, paper: Paper):
954
+ if not paper.doi:
955
+ return
956
+ if len(paper.full_text or "") < MIN_FULLTEXT_LEN:
957
+ return
958
+ path = self._cache_key(paper.doi)
959
+ try:
960
+ path.parent.mkdir(parents=True, exist_ok=True)
961
+ path.write_text(paper.to_json(), encoding="utf-8")
962
+ logger.info("Cached result for %s (%d chars)", paper.doi, len(paper.full_text or ""))
963
+ except OSError as e:
964
+ logger.warning("Failed to save cache for %s: %s", paper.doi, e)
965
+
966
+ def clear_cache(self):
967
+ cache_dir = Path(self.config.get("cache_dir", ".cache"))
968
+ if cache_dir.exists():
969
+ for f in cache_dir.glob("*.json"):
970
+ f.unlink()
971
+ logger.info("Cache cleared.")
972
+
973
+ def close(self):
974
+ if self._auth:
975
+ self._auth.close()