litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,274 @@
1
+ """Elsevier API key 权限自检与逐篇探测(纯查询,零正文下载)。
2
+
3
+ 标准化流程(对应 docs/PLAYBOOK.md 的 key 权限自检):
4
+ 1. 固定自检样本:大刊(Lancet)/ OA 对照 / 无 key 基线 —— 诊断 key 本身
5
+ 2. 用户样本:--doi 或 --file 逐篇双路由探测 —— 诊断具体文献
6
+ 3. 产出规范表格(固定 7 列,判定为规范化枚举)+ key 画像汇总
7
+
8
+ 判定枚举(杜绝自由文本):
9
+ ENTITLED / NOT_ENTITLED / NOT_FOUND / QUOTA / NO_KEY / NETWORK
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import time
15
+ import xml.etree.ElementTree as ET
16
+ from typing import Any
17
+
18
+ import requests
19
+
20
+ from ..network import configured_proxy
21
+
22
+ PREMIUM_DOI = "10.1016/S0140-6736(20)30183-5" # Lancet 知名付费论文(2026-08 实测 200)
23
+ OA_CONTROL_DOI = "10.1016/j.jenvman.2023.118901" # 无 key 可得的公开对照(实测 200)
24
+
25
+ ENTITLED = "ENTITLED"
26
+ NOT_ENTITLED = "NOT_ENTITLED"
27
+ NOT_FOUND = "NOT_FOUND"
28
+ QUOTA = "QUOTA"
29
+ NO_KEY = "NO_KEY"
30
+ NETWORK = "NETWORK"
31
+
32
+ PROXY_ROUTE = "proxy"
33
+ DIRECT_ROUTE = "direct"
34
+
35
+ # 判定 → 建议通道(规范化映射,自动给结论)
36
+ ROUTE_ADVICE = {
37
+ ENTITLED: "走 Elsevier API 车道(可达路由)",
38
+ NOT_ENTITLED: "key 无此刊权益——机构订阅确认,或转 OA/灰色通道",
39
+ NOT_FOUND: "DOI 未注册或写错——人工核对",
40
+ QUOTA: "配额限流——冷却 15s/篇 节流重试",
41
+ NO_KEY: "未配置 API key——运行 elsevier-setup",
42
+ NETWORK: "网络不可达——检查代理连通性",
43
+ }
44
+
45
+
46
+ def normalize_status(code: int | None) -> str:
47
+ """HTTP 状态码 → 规范化判定枚举。"""
48
+ return {
49
+ 200: ENTITLED,
50
+ 401: NOT_ENTITLED,
51
+ 403: NOT_ENTITLED,
52
+ 404: NOT_FOUND,
53
+ 429: QUOTA,
54
+ 406: NO_KEY,
55
+ }.get(code, NETWORK)
56
+
57
+
58
+ def _head_probe(doi: str, key: str, proxies: dict | None, *, http_accept: bool = True) -> tuple[str, str]:
59
+ """HEAD 探针:200=有权获取 PDF(不下载),403=无权限,406=缺 key。
60
+
61
+ http_accept=False 时不带 httpAccept 参数——用于 OA 对照(公开文章
62
+ 无 key 纯 HEAD 即 200;带参数反而 406)。
63
+ """
64
+ try:
65
+ headers: dict[str, str] = {"Accept": "application/pdf", **UA_HEADERS}
66
+ if key:
67
+ headers["X-ELS-APIKey"] = key
68
+ params = {"httpAccept": "application/pdf"} if http_accept else None
69
+ r = requests.head(
70
+ f"https://api.elsevier.com/content/article/doi/{doi}",
71
+ params=params,
72
+ headers=headers, timeout=(8, 12), proxies=proxies, allow_redirects=True,
73
+ )
74
+ return r.status_code, normalize_status(r.status_code)
75
+ except Exception as e:
76
+ return 0, f"{NETWORK}({type(e).__name__})"
77
+
78
+
79
+ # _probe_key_valid 的三态判定(独立于文章权益)
80
+ KEY_VALID = "KEY_VALID"
81
+ KEY_INVALID = "KEY_INVALID"
82
+
83
+ # 403 "Requestor configuration settings insufficient":key 本身有效,但
84
+ # key + 出口 IP 组合未被配置为可取全文(HEAD 无 body,只能 GET 读错误 XML)。
85
+ # 两种成因现象完全相同,按是否已走配置出口区分(#56 的姊妹场景):
86
+ # 1. 未走校园出口 → 出口 IP 不在机构注册段(校外直连的典型表现)
87
+ # 2. 已走校园出口 → 机构未在 Elsevier 侧给该 key 开 API 全文配置
88
+ REQUESTOR_CONFIG_MSG = "requestor configuration settings insufficient"
89
+
90
+
91
+ def _requestor_config_advice(proxy_configured: bool) -> str:
92
+ """按出口路由区分 403 'Requestor configuration settings insufficient' 的两种成因。"""
93
+ if proxy_configured:
94
+ return ("403 'Requestor configuration settings insufficient':已走配置出口仍被拒——"
95
+ "多为机构未给该 key 开 API 全文权限(联系机构 Elsevier 管理员,insttoken 为替代路径),"
96
+ "或该代理出口不在机构注册 IP 段")
97
+ return ("403 'Requestor configuration settings insufficient':多为出口 IP 不在机构注册段(校外直连)——"
98
+ "配 network_proxy/LITLOOP_PROXY 走校园出口重试;校园出口仍 403 则是机构未配置 key 权限"
99
+ "(联系机构 Elsevier 管理员,insttoken 为替代路径)")
100
+
101
+
102
+ def _error_status_text_probe(doi: str, key: str, proxies: dict | None) -> str:
103
+ """GET 诊断:读 403 错误 XML 的 <statusText>(view=META,零正文下载)。"""
104
+ try:
105
+ headers: dict[str, str] = {"Accept": "application/xml", **UA_HEADERS}
106
+ if key:
107
+ headers["X-ELS-APIKey"] = key
108
+ r = requests.get(
109
+ f"https://api.elsevier.com/content/article/doi/{doi}",
110
+ params={"view": "META"},
111
+ headers=headers, timeout=(8, 12), proxies=proxies,
112
+ )
113
+ if r.status_code != 403:
114
+ return ""
115
+ return _status_text(r.text)
116
+ except Exception:
117
+ return ""
118
+
119
+
120
+ def _status_text(body: str) -> str:
121
+ """Elsevier 错误 XML 的 <statusText>(兼容命名空间)。"""
122
+ try:
123
+ root = ET.fromstring(body)
124
+ except ET.ParseError:
125
+ return ""
126
+ for el in root.iter():
127
+ if el.tag.rsplit("}", 1)[-1] == "statusText":
128
+ return (el.text or "").strip()
129
+ return ""
130
+
131
+
132
+ def _probe_key_valid(key: str, proxies: dict | None) -> tuple[int, str]:
133
+ """key 本身有效性探针:打一个只需认证、不需订阅权益的元数据端点。
134
+
135
+ serial/title 对任何有效 key 都返回 200(与机构订了哪些刊无关)——
136
+ 大刊样本 403 只说明"无该刊权益",过去被当成"无效 key"(#56)。
137
+ 401/403/406 才是 key 无效;429 说明认证已通过、只是限流。
138
+ """
139
+ try:
140
+ r = requests.get(
141
+ "https://api.elsevier.com/content/serial/title",
142
+ params={"count": 1},
143
+ headers={"Accept": "application/json", "X-ELS-APIKey": key, **UA_HEADERS},
144
+ timeout=(8, 12), proxies=proxies,
145
+ )
146
+ if r.status_code == 200 or r.status_code == 429:
147
+ return r.status_code, KEY_VALID
148
+ if r.status_code in (401, 403, 406):
149
+ return r.status_code, KEY_INVALID
150
+ return r.status_code, NETWORK
151
+ except Exception as e:
152
+ return 0, f"{NETWORK}({type(e).__name__})"
153
+
154
+
155
+ def probe_dual_route(doi: str, key: str, config: dict[str, Any]) -> dict[str, Any]:
156
+ """单篇双路由探测(代理 + 直连)。返回规范行 dict。"""
157
+ proxy = configured_proxy(config)
158
+ proxies = {"http": proxy, "https": proxy} if proxy else None
159
+ p_code, p_verdict = _head_probe(doi, key, proxies)
160
+ time.sleep(0.3)
161
+ d_code, d_verdict = _head_probe(doi, key, None)
162
+ verdict, advice = _combine(p_verdict, d_verdict)
163
+ note = ""
164
+ if verdict == NOT_ENTITLED:
165
+ # HEAD 无 body,403 的解释只能 GET 读 <statusText>;META 视图守住"零正文下载"
166
+ text = _error_status_text_probe(doi, key, proxies)
167
+ if REQUESTOR_CONFIG_MSG in text.lower():
168
+ advice = _requestor_config_advice(bool(proxy))
169
+ note = f"403 statusText: {text}"
170
+ return {"doi": doi, "sample_type": "用户样本", "proxy": f"{p_code} {p_verdict}",
171
+ "direct": f"{d_code} {d_verdict}", "verdict": verdict,
172
+ "route_advice": advice, "note": note}
173
+
174
+
175
+ def _combine(p: str, d: str) -> tuple[str, str]:
176
+ """双路由判定合并:任一路由 ENTITLED 即可得。"""
177
+ for v in (p, d):
178
+ if v.startswith(ENTITLED):
179
+ return ENTITLED, ROUTE_ADVICE[ENTITLED]
180
+ for v in (p, d):
181
+ if v.startswith(QUOTA):
182
+ return QUOTA, ROUTE_ADVICE[QUOTA]
183
+ if p.startswith(NOT_FOUND) and d.startswith(NOT_FOUND):
184
+ return NOT_FOUND, ROUTE_ADVICE[NOT_FOUND]
185
+ if NOT_ENTITLED in (p.split("(")[0], d.split("(")[0]):
186
+ return NOT_ENTITLED, ROUTE_ADVICE[NOT_ENTITLED]
187
+ return NETWORK, ROUTE_ADVICE[NETWORK]
188
+
189
+
190
+ def check_key_profile(config: dict[str, Any]) -> tuple[list[dict[str, Any]], str, str]:
191
+ """固定自检样本 → (规范行, key 画像判定, 建议)。
192
+
193
+ 画像四选一:广覆盖机构key / key有效但无该订阅 / 无效key / 网络不可达。
194
+ key 有效性由 _probe_key_valid 独立判定,不再用"大刊样本拿不到"反推——
195
+ 有效 key 对未订阅的样本刊同样 403,那是权益问题不是 key 问题(#56)。
196
+ """
197
+ key = (config.get("elsevier_api_key") or "").strip()
198
+ proxy = configured_proxy(config)
199
+ proxies = {"http": proxy, "https": proxy} if proxy else None
200
+
201
+ rows: list[dict[str, Any]] = []
202
+
203
+ # 0) key 有效性(只需认证、不需权益的元数据端点)
204
+ key_code, key_verdict = _probe_key_valid(key, proxies)
205
+ rows.append({"doi": "serial/title (key 认证)", "sample_type": "自检-key有效性",
206
+ "proxy": f"{key_code} {key_verdict}", "direct": "-", "verdict": key_verdict,
207
+ "route_advice": "—", "note": "预期 200/KEY_VALID"})
208
+
209
+ # 1) 无 key 基线(证明端点连通且权限来自 key)
210
+ base_code, base_verdict = _head_probe(PREMIUM_DOI, "", proxies)
211
+ rows.append({"doi": PREMIUM_DOI, "sample_type": "自检-无key基线",
212
+ "proxy": f"{base_code} {base_verdict}", "direct": "-", "verdict": base_verdict,
213
+ "route_advice": "—", "note": "预期 406/NO_KEY"})
214
+
215
+ # 2) OA 对照(无 key 应可得:证明端点工作且该篇公开)
216
+ code_oa, verdict_oa = _head_probe(OA_CONTROL_DOI, "", proxies, http_accept=False)
217
+ rows.append({"doi": OA_CONTROL_DOI, "sample_type": "自检-OA对照",
218
+ "proxy": f"{code_oa} {verdict_oa}", "direct": "-", "verdict": verdict_oa,
219
+ "route_advice": "—", "note": "预期 200/ENTITLED"})
220
+
221
+ # 3) 大刊样本(有 key,双路由):测订阅广度
222
+ key_rows = []
223
+ for route, px in ((PROXY_ROUTE, proxies), (DIRECT_ROUTE, None)):
224
+ code, verdict = _head_probe(PREMIUM_DOI, key, px)
225
+ key_rows.append((route, code, verdict))
226
+ rows.append({"doi": PREMIUM_DOI, "sample_type": f"自检-大刊({route})",
227
+ "proxy": f"{code} {verdict}" if route == PROXY_ROUTE else "-",
228
+ "direct": "-" if route == PROXY_ROUTE else f"{code} {verdict}",
229
+ "verdict": verdict, "route_advice": "—", "note": "预期 200/ENTITLED"})
230
+
231
+ ent = any(v == ENTITLED for _, _, v in key_rows)
232
+ base_ok = base_verdict == NO_KEY or base_code in (401, 403, 406)
233
+ oa_ok = verdict_oa == ENTITLED or code_oa == 200
234
+ key_ok = key_verdict == KEY_VALID
235
+ # 网络异常时 verdict 是 "NETWORK(ExcName)"——只按前缀判
236
+ key_reachable = not key_verdict.startswith(NETWORK) or key_code != 0
237
+
238
+ if not key_reachable and not base_ok and not oa_ok:
239
+ profile, advice = "网络/端点不可达", "检查代理连通性后重试"
240
+ elif key_ok:
241
+ if ent:
242
+ profile = "广覆盖机构 key"
243
+ advice = "Elsevier 车道可用:API 优先,批量走快车道"
244
+ else:
245
+ profile = "key 有效但无该刊订阅"
246
+ advice = "该 key 未绑定机构权益或机构未订阅该样本刊——用机构身份(校园网出口)重新注册 key,或依赖 OA/灰色通道"
247
+ # 大刊双路由 403 时点名具体成因(403 ≠ key 无效,#56;两类成因按出口区分)
248
+ if any(c == 403 for _, c, _ in key_rows):
249
+ text = _error_status_text_probe(PREMIUM_DOI, key, proxies)
250
+ if REQUESTOR_CONFIG_MSG in text.lower():
251
+ advice = _requestor_config_advice(bool(proxy))
252
+ else:
253
+ profile = "无效 key"
254
+ advice = "重新运行 elsevier-setup --api-key 配置有效 key"
255
+
256
+ return rows, profile, advice
257
+
258
+
259
+ def check_dois(dois: list[str], config: dict[str, Any], progress: Any = None) -> list[dict[str, Any]]:
260
+ """用户样本逐篇双路由探测(规范化行)。"""
261
+ key = (config.get("elsevier_api_key") or "").strip()
262
+ out = []
263
+ for i, doi in enumerate(dois, 1):
264
+ row = probe_dual_route(doi, key, config)
265
+ row["sample_type"] = "用户样本"
266
+ out.append(row)
267
+ if progress is not None:
268
+ progress.advance(row["verdict"] == ENTITLED, current=doi)
269
+ return out
270
+
271
+
272
+ # UA 常量放底部避免循环导入(network.py 不导入本模块,可安全顶部导入;
273
+ # 这里保守处理,保持与其它模块一致)
274
+ UA_HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/151.0 Safari/537.36"}
@@ -0,0 +1,81 @@
1
+ """Europe PMC source for life sciences papers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import urllib.parse
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from ..network import fetch_json, polite_delay
10
+ from ..download.pdf import download_pdf, is_plausible_pdf_url, dedupe
11
+
12
+
13
+ def extract_europepmc_pdf_candidates(payload: dict[str, Any]) -> list[str]:
14
+ candidates: list[str] = []
15
+ results = payload.get("resultList", {}).get("result", [])
16
+ if not isinstance(results, list):
17
+ results = []
18
+ for result in results:
19
+ fulltext = result.get("fullTextUrlList", {}) if isinstance(result, dict) else {}
20
+ urls = fulltext.get("fullTextUrl", []) if isinstance(fulltext, dict) else []
21
+ if not isinstance(urls, list):
22
+ urls = [urls]
23
+ for entry in urls:
24
+ if not isinstance(entry, dict):
25
+ continue
26
+ url = entry.get("url", "")
27
+ style = str(entry.get("documentStyle", "")).lower()
28
+ if style == "pdf" or is_plausible_pdf_url(url):
29
+ if is_plausible_pdf_url(url):
30
+ candidates.append(url)
31
+ return dedupe(candidates)
32
+
33
+
34
+ def try_europepmc(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
35
+ q = urllib.parse.quote(f"DOI:{doi}")
36
+ url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={q}&format=json&pageSize=5"
37
+ payload = fetch_json(url, config)
38
+ if not payload:
39
+ return None
40
+
41
+ candidates = extract_europepmc_pdf_candidates(payload)
42
+ for pdf_url in candidates[: int(config.get("max_europepmc_candidates", 3))]:
43
+ polite_delay(config)
44
+ result = download_pdf(pdf_url, output_path, config, "EuropePMC")
45
+ if result:
46
+ result["doi"] = doi
47
+ result["identifier"] = doi
48
+ return result
49
+ return None
50
+
51
+
52
+ def try_pmc(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
53
+ """Try PubMed Central for free full-text PDF."""
54
+ # PMC ID Converter API (https://pmc.ncbi.nlm.nih.gov/tools/id-converter-api/):
55
+ # the old /pmc/utils/idconv/v1.0/ address now redirects here; NCBI asks
56
+ # callers to identify themselves with tool and email.
57
+ params = {"ids": doi, "idtype": "doi", "format": "json", "tool": "litloop"}
58
+ email = str(config.get("email") or "").strip()
59
+ if email and not email.endswith(".invalid"):
60
+ params["email"] = email
61
+ search_url = "https://pmc.ncbi.nlm.nih.gov/tools/idconv/api/v1/articles/?" + urllib.parse.urlencode(params)
62
+ try:
63
+ resp_data = fetch_json(search_url, config)
64
+ if not resp_data:
65
+ return None
66
+ records = resp_data.get("records", [])
67
+ pmcid = None
68
+ for rec in records:
69
+ if rec.get("pmcid"):
70
+ pmcid = rec["pmcid"]
71
+ break
72
+ if not pmcid:
73
+ return None
74
+ except Exception:
75
+ return None
76
+ pdf_url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/pdf/"
77
+ result = download_pdf(pdf_url, output_path, config, f"PMC({pmcid})")
78
+ if result:
79
+ result["doi"] = doi
80
+ result["identifier"] = doi
81
+ return result
@@ -0,0 +1,317 @@
1
+ """EZProxy institutional proxy source.
2
+
3
+ Uses the university library's EZProxy service to access papers.
4
+ EZProxy rewrites URLs through the library proxy, providing
5
+ institutional access to subscribed journals.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Any
13
+ from urllib.parse import urlparse
14
+
15
+ import requests
16
+
17
+ from ..log import get_logger
18
+ from ..download.pdf import (
19
+ _response_looks_pdf,
20
+ extract_pdf_url_from_html,
21
+ is_pdf_file,
22
+ is_plausible_pdf_url,
23
+ success,
24
+ )
25
+
26
+ log = get_logger()
27
+
28
+
29
+ def _get_ezproxy_base(config: dict[str, Any]) -> str:
30
+ """Get EZProxy login URL template."""
31
+ return config.get("ezproxy_login_url", "")
32
+
33
+
34
+ def _is_proxy_domain(domain: str) -> bool:
35
+ """True if a cookie domain / hostname belongs to the EZproxy itself.
36
+
37
+ Both proxy styles carry a marker label in every host: prefix-style
38
+ (``ezproxy.lib.foo.edu``) and hostname-rewriting
39
+ (``onlinelibrary-wiley-com.lib.ezproxy.hkust.edu.hk``, Berkeley's
40
+ ``*-libproxy.berkeley.edu``). Publisher hosts never do (#62).
41
+ """
42
+ d = (domain or "").lower()
43
+ return "ezproxy" in d or "libproxy" in d
44
+
45
+
46
+ def _ezproxy_origin(config: dict[str, Any]) -> str:
47
+ """Derive the proxy origin (``scheme://host``) from the configured template.
48
+
49
+ Supported template shapes:
50
+ - prefix-style: ``https://ezproxy.lib.foo.edu/login?url={url}``
51
+ - hostname-rewrite: ``https://{host_dashed}.lib.ezproxy.hkust.edu.hk``
52
+ """
53
+ base = _get_ezproxy_base(config)
54
+ if not base:
55
+ return ""
56
+ if "{host_dashed}" in base:
57
+ # The proxy origin is the template with the rewriting label removed.
58
+ parsed = urlparse(base.replace("{host_dashed}", ""))
59
+ if not parsed.netloc:
60
+ return ""
61
+ return f"{parsed.scheme}://{parsed.netloc.lstrip('.')}"
62
+ parsed = urlparse(base)
63
+ if parsed.netloc:
64
+ return f"{parsed.scheme}://{parsed.netloc}"
65
+ return ""
66
+
67
+
68
+ def _make_ezproxy_url(target_url: str, config: dict[str, Any]) -> str:
69
+ """Convert a target URL to an EZProxy-proxied URL.
70
+
71
+ ``{url}`` templates proxy by prefix substitution; ``{host_dashed}``
72
+ templates proxy by hostname rewriting — the publisher host becomes a
73
+ dash-separated subdomain of the proxy host, which is the only shape
74
+ hostname-rewriting deployments (HKUST et al.) accept (#62).
75
+ """
76
+ base = _get_ezproxy_base(config)
77
+ if not base:
78
+ return ""
79
+ if "{url}" in base:
80
+ return base.replace("{url}", target_url)
81
+ if "{host_dashed}" in base:
82
+ parsed = urlparse(target_url)
83
+ host = parsed.hostname or ""
84
+ if not host:
85
+ return ""
86
+ # Replace on the template's own origin (label still in place) — the
87
+ # stripped origin from _ezproxy_origin no longer carries the label.
88
+ parsed_base = urlparse(base)
89
+ if not parsed_base.netloc:
90
+ return ""
91
+ origin = f"{parsed_base.scheme}://{parsed_base.netloc}"
92
+ rest = parsed.path or "/"
93
+ if parsed.query:
94
+ rest += f"?{parsed.query}"
95
+ return origin.replace("{host_dashed}", host.replace(".", "-")) + rest
96
+ return ""
97
+
98
+
99
+ def _looks_like_login_url(url: str) -> bool:
100
+ """True when the URL path itself is a login challenge.
101
+
102
+ Only the path/query count: hostname-rewritten proxy URLs keep
103
+ ``ezproxy``/``libproxy`` in the host for EVERY proxied page, so the old
104
+ host-token check read a fully proxied (authenticated) page as
105
+ "login required" (#62).
106
+ """
107
+ parsed = urlparse(url)
108
+ path_q = f"{parsed.path}?{parsed.query}".lower()
109
+ return "/login" in path_q or "wayf" in path_q or "shibboleth" in path_q
110
+
111
+
112
+ def _validate_ezproxy_session(config: dict[str, Any]) -> bool:
113
+ """Check if saved EZProxy cookies still work."""
114
+ cookie_file = _ezproxy_cookie_path(config)
115
+ if not cookie_file.exists():
116
+ return False
117
+ try:
118
+ import json
119
+ cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
120
+ except Exception:
121
+ return False
122
+ if not cookies:
123
+ return False
124
+
125
+ # The session marker lives on the proxy's own domain. A jar with no
126
+ # proxy-domain cookie was captured off a raw publisher redirect and
127
+ # cannot authenticate anything — don't probe with it (#62).
128
+ if not any(_is_proxy_domain(c.get("domain", "")) for c in cookies):
129
+ return False
130
+
131
+ sess = requests.Session()
132
+ sess.trust_env = False
133
+ for c in cookies:
134
+ sess.cookies.set(c["name"], c["value"], domain=c.get("domain", ""), path=c.get("path", "/"))
135
+
136
+ # Probe the proxy's own /login endpoint: with a live session EZproxy
137
+ # redirects away (menu / start page), while an expired session keeps the
138
+ # URL on /login or bounces to the SSO challenge. This works for both
139
+ # prefix-style and hostname-rewriting deployments — probing a {url}
140
+ # prefix template on a rewriting deployment only ever saw the raw
141
+ # publisher and misreported valid sessions as expired (#62).
142
+ origin = _ezproxy_origin(config)
143
+ if not origin:
144
+ return False
145
+ try:
146
+ resp = sess.get(f"{origin}/login", timeout=15, allow_redirects=True)
147
+ final = urlparse(resp.url)
148
+ final_host = (final.hostname or "").lower()
149
+ if "/login" in final.path.lower():
150
+ return False
151
+ if not _is_proxy_domain(final_host) and any(
152
+ x in final_host for x in ("idp.", "sso.", "shibboleth", "auth.")
153
+ ):
154
+ return False
155
+ text = resp.text[:4000].lower()
156
+ if "ezproxy login" in text or "shibboleth authentication request" in text:
157
+ return False
158
+ return resp.status_code == 200
159
+ except Exception:
160
+ return False
161
+
162
+
163
+ def _ezproxy_cookie_path(config: dict[str, Any]) -> Path:
164
+ """Get path to saved EZProxy cookies."""
165
+ from ..config import DATA_DIR
166
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
167
+ return cache_dir / "ezproxy_cookies.json"
168
+
169
+
170
+ def ezproxy_login(config: dict[str, Any]) -> bool:
171
+ """Open browser for EZProxy login. Tries stealth browser first, falls back to Selenium."""
172
+ # Try stealth browser (stealth browser) first
173
+ try:
174
+ from ..browser.login import ezproxy_login as _browser_ezproxy
175
+ if _browser_ezproxy(config):
176
+ return True
177
+ except Exception as exc:
178
+ log.info(f" [EZProxy] stealth browser login failed: {exc}")
179
+
180
+ log.error(" [EZProxy] CloakBrowser login failed, no fallback available")
181
+ return False
182
+
183
+
184
+ def try_ezproxy(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
185
+ """Try downloading paper through EZProxy institutional proxy.
186
+
187
+ Uses Selenium browser to access the paper through the library proxy,
188
+ which handles authentication and cookie management automatically.
189
+ """
190
+ if not config.get("ezproxy_enabled", False):
191
+ return None
192
+
193
+ base = _get_ezproxy_base(config)
194
+ if not base:
195
+ return None
196
+
197
+ # Resolve DOI to get publisher URL
198
+ try:
199
+ resp = requests.head(f"https://doi.org/{doi}", allow_redirects=True, timeout=10)
200
+ resolved_url = resp.url
201
+ except Exception:
202
+ resolved_url = f"https://doi.org/{doi}"
203
+
204
+ # Construct EZProxy URL
205
+ ezproxy_url = _make_ezproxy_url(resolved_url, config)
206
+ if not ezproxy_url:
207
+ return None
208
+
209
+ log.info(f" [EZProxy] Trying {doi} via library proxy...")
210
+
211
+ try:
212
+ from ..browser.backend import launch
213
+ from ..browser.backend import is_available as _browser_backend_available
214
+ from ..browser.engine import _build_browser_args
215
+ if not _browser_backend_available():
216
+ log.info(" [EZProxy] no browser backend available")
217
+ return None
218
+ except ImportError:
219
+ log.info(" [EZProxy] no browser backend available")
220
+ return None
221
+
222
+ download_dir = str(output_path.parent)
223
+ args = _build_browser_args(config)
224
+ captured_pdf: list[bytes] = []
225
+
226
+ def _on_response(response):
227
+ try:
228
+ ct = response.headers.get("content-type", "")
229
+ if "pdf" in ct:
230
+ try:
231
+ body = response.body()
232
+ if body and len(body) > 5000:
233
+ captured_pdf.append(body)
234
+ except Exception:
235
+ pass
236
+ except Exception:
237
+ pass
238
+
239
+ browser = launch(headless=False, humanize=True, args=args)
240
+ try:
241
+ context = browser.new_context()
242
+ page = context.new_page()
243
+ page.on("response", _on_response)
244
+
245
+ # Load saved cookies if available
246
+ cookie_file = _ezproxy_cookie_path(config)
247
+ if cookie_file.exists():
248
+ import json
249
+ cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
250
+ if cookies:
251
+ context.add_cookies(cookies)
252
+
253
+ # Navigate to EZProxy URL
254
+ page.goto(ezproxy_url, wait_until="domcontentloaded", timeout=30000)
255
+ time.sleep(8)
256
+
257
+ # Check if redirected to a login challenge (path-based: proxied hosts
258
+ # keep ezproxy/libproxy in every URL — see _looks_like_login_url)
259
+ if _looks_like_login_url(page.url):
260
+ log.info(" [EZProxy] Login required. Please log in...")
261
+ max_wait = 180
262
+ elapsed = 0
263
+ while elapsed < max_wait:
264
+ time.sleep(3)
265
+ elapsed += 3
266
+ try:
267
+ url = page.url
268
+ except Exception:
269
+ return None
270
+ if not _looks_like_login_url(url):
271
+ break
272
+ else:
273
+ log.info(" [EZProxy] Login timed out.")
274
+ return None
275
+
276
+ # Check for captured PDF
277
+ if captured_pdf:
278
+ pdf_bytes = captured_pdf[-1]
279
+ if pdf_bytes[:5] == b"%PDF-":
280
+ output_path.parent.mkdir(parents=True, exist_ok=True)
281
+ output_path.write_bytes(pdf_bytes)
282
+ return success(doi, output_path, "EZProxy")
283
+
284
+ # Look for PDF link in page
285
+ pdf_link = page.evaluate("""() => {
286
+ const links = document.querySelectorAll('a');
287
+ for (const link of links) {
288
+ const text = (link.innerText || '').toLowerCase();
289
+ const href = link.href || '';
290
+ if (text.includes('pdf') && !text.includes('purchase') && href) {
291
+ return href;
292
+ }
293
+ }
294
+ return '';
295
+ }""")
296
+ if pdf_link:
297
+ log.info(f" [EZProxy] Found PDF link: {pdf_link[:80]}")
298
+ captured_pdf.clear()
299
+ page.goto(pdf_link, wait_until="commit", timeout=30000)
300
+ time.sleep(5)
301
+ if captured_pdf:
302
+ pdf_bytes = captured_pdf[-1]
303
+ if pdf_bytes[:5] == b"%PDF-":
304
+ output_path.parent.mkdir(parents=True, exist_ok=True)
305
+ output_path.write_bytes(pdf_bytes)
306
+ return success(doi, output_path, "EZProxy")
307
+
308
+ except Exception as e:
309
+ log.info(f" [EZProxy] Error: {e}")
310
+ finally:
311
+ try:
312
+ browser.close()
313
+ except Exception:
314
+ pass
315
+
316
+ return None
317
+