litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,347 @@
1
+ """Extract cookies via CloakBrowser for publisher access.
2
+
3
+ Opens a visible CloakBrowser window, lets user log in to their institution,
4
+ then captures and saves publisher cookies for automated downloads.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import time
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ from ..log import get_logger
15
+
16
+ log = get_logger()
17
+
18
+ # Publisher domains that benefit from institutional cookies
19
+ PUBLISHER_DOMAINS = [
20
+ "sciencedirect.com",
21
+ "elsevier.com",
22
+ "springer.com",
23
+ "springerlink.com",
24
+ "link.springer.com",
25
+ "onlinelibrary.wiley.com",
26
+ "wiley.com",
27
+ "ieeexplore.ieee.org",
28
+ "ieee.org",
29
+ "nature.com",
30
+ "science.org",
31
+ "sciencemag.org",
32
+ "tandfonline.com",
33
+ "pnas.org",
34
+ "jstor.org",
35
+ "acs.org",
36
+ "rsc.org",
37
+ "cambridge.org",
38
+ "oup.com",
39
+ "academic.oup.com",
40
+ "sagepub.com",
41
+ "sage.cnpereading.com",
42
+ "mdpi.com",
43
+ "frontiersin.org",
44
+ ]
45
+
46
+ # Publisher login URLs for quick access
47
+ PUBLISHER_LOGIN_URLS = {
48
+ "elsevier": "https://www.sciencedirect.com/",
49
+ "springer": "https://link.springer.com/",
50
+ "wiley": "https://onlinelibrary.wiley.com/",
51
+ "nature": "https://www.nature.com/",
52
+ "science": "https://www.science.org/",
53
+ "ieee": "https://ieeexplore.ieee.org/",
54
+ "tandfonline": "https://www.tandfonline.com/",
55
+ "pnas": "https://www.pnas.org/",
56
+ "acs": "https://pubs.acs.org/",
57
+ "rsc": "https://pubs.rsc.org/",
58
+ "aip": "https://pubs.aip.org/",
59
+ "aps": "https://journals.aps.org/",
60
+ "iop": "https://iopscience.iop.org/",
61
+ "oxford": "https://academic.oup.com/",
62
+ "acm": "https://dl.acm.org/",
63
+ "sage-cn": "https://sage.cnpereading.com/login",
64
+ }
65
+
66
+
67
+ def _save_cookies_json(cookies: list[dict[str, Any]], output_path: Path) -> None:
68
+ from ..institutional.session_store import keep_private
69
+ output_path.parent.mkdir(parents=True, exist_ok=True)
70
+ output_path.write_text(json.dumps(cookies, indent=2, ensure_ascii=False), encoding="utf-8")
71
+ keep_private(output_path)
72
+
73
+
74
+ def cookies_to_netscape(cookies: list[dict[str, Any]]) -> str:
75
+ """Convert a list of cookie dicts to Netscape HTTP Cookie File format."""
76
+ lines = ["# Netscape HTTP Cookie File\n"]
77
+ for c in cookies:
78
+ domain = c.get("domain", "")
79
+ flag = "TRUE" if domain.startswith(".") else "FALSE"
80
+ path = c.get("path", "/")
81
+ secure = "TRUE" if c.get("secure") else "FALSE"
82
+ expires = str(int(c.get("expires", 0)) or 0)
83
+ name = c.get("name", "")
84
+ value = c.get("value", "")
85
+ lines.append(f"{domain}\t{flag}\t{path}\t{secure}\t{expires}\t{name}\t{value}\n")
86
+ return "".join(lines)
87
+
88
+
89
+ def _save_cookies_netscape(cookies: list[dict[str, Any]], output_path: Path) -> None:
90
+ from ..institutional.session_store import keep_private
91
+ output_path.write_text(cookies_to_netscape(cookies), encoding="utf-8")
92
+ keep_private(output_path)
93
+
94
+
95
+ def _is_publisher_cookie(cookie: dict[str, Any]) -> bool:
96
+ """Check if a cookie belongs to a known publisher domain."""
97
+ domain = cookie.get("domain", "").lstrip(".")
98
+ return any(domain.endswith(d) for d in PUBLISHER_DOMAINS)
99
+
100
+
101
+ def extract_via_browser(
102
+ config: dict[str, Any],
103
+ *,
104
+ url: str = "https://www.sciencedirect.com/",
105
+ max_wait: int = 300,
106
+ ) -> dict[str, Any]:
107
+ """Open visible browser for institutional login, then capture publisher cookies.
108
+
109
+ Args:
110
+ config: litloop config dict.
111
+ url: URL to open (default: ScienceDirect).
112
+ max_wait: Max seconds to wait for login.
113
+
114
+ Returns:
115
+ Result dict with success, cookies_count, domains, etc.
116
+ """
117
+ try:
118
+ from .backend import launch
119
+ from .backend import is_available as _browser_backend_available
120
+ if not _browser_backend_available():
121
+ return {
122
+ "success": False,
123
+ "error": "no browser backend available",
124
+ "fix": "pip install patchright (or pip install cloakbrowser)",
125
+ }
126
+ except ImportError:
127
+ return {
128
+ "success": False,
129
+ "error": "no browser backend available",
130
+ "fix": "pip install patchright (or pip install cloakbrowser)",
131
+ }
132
+
133
+ from ..config import DATA_DIR
134
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
135
+ cookie_file = cache_dir / "publisher_cookies.json"
136
+ netscape_file = cache_dir / "publisher_cookies.txt"
137
+
138
+ log.info(f" [cookies] Opening browser: {url}")
139
+ print(f"\n 请在浏览器中登录你的机构账号(如学校图书馆)")
140
+ print(f" 打开页面: {url}")
141
+ print(f" 登录完成后关闭浏览器窗口即可\n")
142
+
143
+ try:
144
+ browser = launch(headless=False, humanize=True)
145
+ context = browser.new_context(viewport={"width": 1440, "height": 900})
146
+ page = context.new_page()
147
+
148
+ try:
149
+ page.goto(url, wait_until="domcontentloaded", timeout=60000)
150
+ except Exception as exc:
151
+ log.info(f" [cookies] Page load warning: {exc}")
152
+
153
+ # Auto-dismiss cookie consent popups (common on Elsevier/Cell/Wiley/etc.)
154
+ time.sleep(2)
155
+ for js in [
156
+ # Elsevier/Cell cookie banner
157
+ "document.querySelector('#onetrust-accept-btn-handler')?.click()",
158
+ # OneTrust general
159
+ "document.querySelector('.onetrust-close-btn-handler')?.click()",
160
+ # Generic consent banners
161
+ "document.querySelector('[class*=\"cookie-accept\"], [class*=\"consent-accept\"]')?.click()",
162
+ "document.querySelector('button[id*=\"accept\"], button[class*=\"accept\"]')?.click()",
163
+ ]:
164
+ try:
165
+ page.evaluate(js)
166
+ except Exception:
167
+ pass
168
+
169
+ # Wait for user to close browser
170
+ try:
171
+ page.wait_for_event("close", timeout=max_wait * 1000)
172
+ except Exception:
173
+ pass
174
+
175
+ # Capture all cookies
176
+ all_cookies = context.cookies()
177
+
178
+ if not all_cookies:
179
+ browser.close()
180
+ return {
181
+ "success": False,
182
+ "message": "未捕获到 cookies。请确保已登录机构账号。",
183
+ }
184
+
185
+ # Filter publisher cookies + keep all for completeness
186
+ publisher_cookies = [c for c in all_cookies if _is_publisher_cookie(c)]
187
+ save_cookies = publisher_cookies if publisher_cookies else all_cookies
188
+
189
+ # Save
190
+ _save_cookies_json(save_cookies, cookie_file)
191
+ _save_cookies_netscape(save_cookies, netscape_file)
192
+
193
+ # Import into CloakBrowser server if running
194
+ browser_imported = 0
195
+ try:
196
+ from .engine import import_cookies, is_available
197
+ if is_available(config):
198
+ browser_imported = import_cookies(netscape_file, config)
199
+ except Exception:
200
+ pass
201
+
202
+ browser.close()
203
+
204
+ domains_found = list({c.get("domain", "").lstrip(".") for c in save_cookies})[:10]
205
+ return {
206
+ "success": True,
207
+ "cookies_count": len(save_cookies),
208
+ "total_captured": len(all_cookies),
209
+ "cookie_file": str(cookie_file),
210
+ "netscape_file": str(netscape_file),
211
+ "domains": domains_found,
212
+ "browser_imported": browser_imported,
213
+ "message": f"捕获 {len(all_cookies)} 个 cookies,其中 {len(publisher_cookies)} 个属于出版社。"
214
+ f"已保存,后续下载自动使用。",
215
+ }
216
+
217
+ except Exception as exc:
218
+ log.info(f" [cookies] Error: {exc}")
219
+ return {"success": False, "error": str(exc)}
220
+
221
+
222
+ def load_saved_cookies(config: dict[str, Any]) -> list[dict[str, Any]]:
223
+ """Load previously saved publisher cookies, filtering out expired ones."""
224
+ from ..config import DATA_DIR
225
+ cookie_file = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "publisher_cookies.json"
226
+ if not cookie_file.exists():
227
+ return []
228
+ try:
229
+ cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
230
+ except Exception:
231
+ return []
232
+ now = time.time()
233
+ return [c for c in cookies if _is_cookie_valid(c, now)]
234
+
235
+
236
+ def _is_cookie_valid(cookie: dict[str, Any], now: float | None = None) -> bool:
237
+ """Check if a cookie is not expired. expires=0 means session cookie (always valid)."""
238
+ expires = cookie.get("expires", 0)
239
+ if not expires or expires == 0:
240
+ return True
241
+ if now is None:
242
+ now = time.time()
243
+ return expires > now
244
+
245
+
246
+ def inject_cookies(session: Any, config: dict[str, Any]) -> None:
247
+ """Inject saved publisher cookies into a requests.Session."""
248
+ cookies = load_saved_cookies(config)
249
+ for c in cookies:
250
+ session.cookies.set(c["name"], c["value"], domain=c.get("domain", ""), path=c.get("path", "/"))
251
+
252
+
253
+ def merge_cookies(new_cookies: list[dict[str, Any]], config: dict[str, Any]) -> list[dict[str, Any]]:
254
+ """Merge new cookies into saved publisher cookies, dedup by (name, domain, path).
255
+
256
+ Filters to publisher domains only. Updates existing cookies in-place
257
+ (new value overwrites old) and appends genuinely new ones. Saves both
258
+ JSON and Netscape formats.
259
+
260
+ Returns the merged cookie list.
261
+ """
262
+ from ..config import DATA_DIR
263
+
264
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
265
+ cookie_file = cache_dir / "publisher_cookies.json"
266
+ netscape_file = cache_dir / "publisher_cookies.txt"
267
+
268
+ existing = load_saved_cookies(config)
269
+
270
+ # Build index of existing cookies by (name, domain, path)
271
+ key = lambda c: (c.get("name", ""), c.get("domain", ""), c.get("path", "/"))
272
+ merged: dict[tuple, dict[str, Any]] = {key(c): c for c in existing}
273
+
274
+ for c in new_cookies:
275
+ if not _is_publisher_cookie(c):
276
+ continue
277
+ merged[key(c)] = c
278
+
279
+ result = list(merged.values())
280
+
281
+ if result:
282
+ _save_cookies_json(result, cookie_file)
283
+ _save_cookies_netscape(result, netscape_file)
284
+
285
+ return result
286
+
287
+
288
+ def _is_doi(text: str) -> bool:
289
+ return text.startswith("10.") and "/" in text
290
+
291
+
292
+ def publisher_login(
293
+ identifier: str,
294
+ config: dict[str, Any],
295
+ *,
296
+ max_wait: int = 300,
297
+ ) -> dict[str, Any]:
298
+ """Open browser for institutional login on a publisher article page.
299
+
300
+ Takes a DOI or publisher name, resolves to the article page, and opens
301
+ a visible stealth browser. User clicks "Access through your institution",
302
+ completes SSO login, then closes the browser. Cookies are captured and
303
+ imported into CloakBrowser for automated downloads.
304
+
305
+ Args:
306
+ identifier: DOI (e.g. 10.1126/science.aec6396) or publisher name
307
+ (e.g. "elsevier", "wiley", "nature", "springer", "ieee",
308
+ "science", "tandfonline", "pnas", "acs", "rsc", "aip",
309
+ "aps", "iop", "oxford", "acm")
310
+ config: litloop config dict
311
+ max_wait: Max seconds to wait for login (default 300)
312
+
313
+ Returns:
314
+ Result dict with success, cookies_count, domains, message.
315
+ """
316
+ import re
317
+
318
+ # Resolve identifier to a URL
319
+ if _is_doi(identifier):
320
+ # DOI → resolve to article URL
321
+ url = f"https://doi.org/{identifier}"
322
+ log.info(f" [login] DOI detected, opening article: {url}")
323
+ elif identifier.lower() in PUBLISHER_LOGIN_URLS:
324
+ url = PUBLISHER_LOGIN_URLS[identifier.lower()]
325
+ log.info(f" [login] Publisher '{identifier}', opening: {url}")
326
+ elif re.match(r"https?://", identifier):
327
+ url = identifier
328
+ log.info(f" [login] URL detected: {url}")
329
+ else:
330
+ # Try fuzzy publisher name match
331
+ lower = identifier.lower()
332
+ match = None
333
+ for key in PUBLISHER_LOGIN_URLS:
334
+ if key in lower or lower in key:
335
+ match = key
336
+ break
337
+ if match:
338
+ url = PUBLISHER_LOGIN_URLS[match]
339
+ log.info(f" [login] Matched publisher '{match}', opening: {url}")
340
+ else:
341
+ available = ", ".join(sorted(PUBLISHER_LOGIN_URLS.keys()))
342
+ return {
343
+ "success": False,
344
+ "error": f"Unknown publisher: '{identifier}'. Available: {available}",
345
+ }
346
+
347
+ return extract_via_browser(config, url=url, max_wait=max_wait)