litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/cli/web.py ADDED
@@ -0,0 +1,190 @@
1
+ """FastAPI web interface for LitLoop."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from fastapi import FastAPI, Request
10
+ from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
11
+ from fastapi.templating import Jinja2Templates
12
+ from pydantic import BaseModel
13
+
14
+ from ..config import load_config
15
+ from ..identifiers import is_arxiv_identifier, normalize_doi
16
+ from ..log import get_logger
17
+ from ..search.quick import search_papers
18
+ from ..sources import download
19
+
20
+ log = get_logger()
21
+
22
+ _TEMPLATE_DIR = Path(__file__).parent.parent / "templates"
23
+ templates = Jinja2Templates(directory=str(_TEMPLATE_DIR))
24
+ templates.env.cache_size = 0
25
+
26
+ app = FastAPI(title="LitLoop", description="Academic paper downloader web UI")
27
+
28
+
29
+ # --- Request/Response models ---
30
+
31
+ class DownloadRequest(BaseModel):
32
+ identifier: str
33
+
34
+
35
+ class SearchRequest(BaseModel):
36
+ query: str
37
+ limit: int = 10
38
+
39
+
40
+ # --- Helper ---
41
+
42
+ _DOI_PATTERN = re.compile(r"^10\.\d{4,}/")
43
+ _DOI_URL_PATTERN = re.compile(r"https?://doi\.org/")
44
+
45
+
46
+ def _is_doi_or_arxiv(text: str) -> bool:
47
+ """Check if input looks like a DOI or arXiv ID (not a title)."""
48
+ text = text.strip()
49
+ if is_arxiv_identifier(text):
50
+ return True
51
+ if _DOI_URL_PATTERN.match(text):
52
+ return True
53
+ if _DOI_PATTERN.match(text):
54
+ return True
55
+ return False
56
+
57
+
58
+ def _check_sources(config: dict[str, Any]) -> dict[str, Any]:
59
+ """Check availability of key download sources."""
60
+ sources: dict[str, bool | str] = {}
61
+
62
+ # CloakBrowser
63
+ try:
64
+ from ..browser.engine import is_available
65
+ cb_available = is_available(config)
66
+ sources["cloakbrowser"] = cb_available
67
+ except Exception:
68
+ sources["cloakbrowser"] = False
69
+
70
+ # Tor
71
+ try:
72
+ from ..tor.proxy import check_tor_circuit
73
+ tor_ok = check_tor_circuit(config)
74
+ sources["tor"] = tor_ok
75
+ except Exception:
76
+ sources["tor"] = False
77
+
78
+ # WebVPN
79
+ sources["webvpn"] = bool(config.get("webvpn_cookies"))
80
+
81
+ # CARSI
82
+ sources["carsi"] = bool(config.get("carsi_cookies"))
83
+
84
+ # Sci-Hub
85
+ sources["scihub"] = config.get("scihub_enabled", False)
86
+
87
+ return sources
88
+
89
+
90
+ # --- Routes ---
91
+
92
+ @app.get("/", response_class=HTMLResponse)
93
+ async def index(request: Request):
94
+ return templates.TemplateResponse("index.html", {"request": request})
95
+
96
+
97
+ @app.post("/api/download")
98
+ async def api_download(req: DownloadRequest):
99
+ """Download a paper by DOI or arXiv ID. Returns PDF file or error JSON."""
100
+ import asyncio
101
+
102
+ identifier = req.identifier.strip()
103
+ if not identifier:
104
+ return JSONResponse({"success": False, "error": "Empty identifier"}, status_code=400)
105
+
106
+ # Normalize DOI URL to bare DOI
107
+ if _DOI_URL_PATTERN.match(identifier):
108
+ identifier = _DOI_URL_PATTERN.sub("", identifier)
109
+
110
+ # If input looks like a title (not DOI/arXiv), try to resolve first
111
+ if not _is_doi_or_arxiv(identifier):
112
+ from ..search.title_to_doi import resolve_title_to_doi
113
+ config = load_config()
114
+ doi = resolve_title_to_doi(identifier, config)
115
+ if doi:
116
+ identifier = doi
117
+ else:
118
+ return JSONResponse(
119
+ {"success": False, "error": f"Could not resolve title to DOI: {identifier}"},
120
+ status_code=404,
121
+ )
122
+
123
+ # Run download in a worker thread to avoid blocking the event loop.
124
+ # asyncio.to_thread is the modern replacement for run_in_executor(None, fn)
125
+ # and avoids deprecation warnings around get_event_loop() in async context.
126
+ result = await asyncio.to_thread(download, identifier)
127
+
128
+ if result.get("success"):
129
+ file_path = result.get("file", "")
130
+ source = result.get("source", "unknown")
131
+ if file_path and Path(file_path).exists():
132
+ filename = Path(file_path).name
133
+ return FileResponse(
134
+ file_path,
135
+ media_type="application/pdf",
136
+ filename=filename,
137
+ headers={"X-LitLoop-Source": source},
138
+ )
139
+ return JSONResponse(
140
+ {"success": False, "error": "PDF file not found on disk after download"},
141
+ status_code=500,
142
+ )
143
+
144
+ # Enhance error response with actionable guidance
145
+ error_response = dict(result)
146
+ config = load_config()
147
+ sources = _check_sources(config)
148
+ error_response["sources"] = sources
149
+
150
+ # Add specific guidance based on what's available
151
+ guidance = error_response.get("guidance", [])
152
+ if not sources.get("cloakbrowser"):
153
+ guidance.insert(0, "CloakBrowser is not running. Start it to enable browser-based downloads for paywalled papers.")
154
+ if not sources.get("tor"):
155
+ guidance.append("Tor is not running. Start Tor for anonymous Sci-Hub access.")
156
+
157
+ error_response["guidance"] = guidance
158
+ return JSONResponse(error_response, status_code=404)
159
+
160
+
161
+ @app.post("/api/search")
162
+ async def api_search(req: SearchRequest):
163
+ """Search papers by keyword. Returns list of results."""
164
+ query = req.query.strip()
165
+ if not query:
166
+ return JSONResponse([], status_code=400)
167
+
168
+ # Normalize DOI URL
169
+ if _DOI_URL_PATTERN.match(query):
170
+ query = _DOI_URL_PATTERN.sub("", query)
171
+
172
+ # If input is a DOI/arXiv, skip search and return a single-item result
173
+ if _is_doi_or_arxiv(query):
174
+ return JSONResponse([{"doi": normalize_doi(query) if not is_arxiv_identifier(query) else query, "title": "", "is_direct": True}])
175
+
176
+ results = search_papers(query, limit=req.limit)
177
+ return JSONResponse(results)
178
+
179
+
180
+ @app.get("/api/status")
181
+ async def api_status():
182
+ """Health check with source availability."""
183
+ config = load_config()
184
+ sources = _check_sources(config)
185
+
186
+ return JSONResponse({
187
+ "status": "ok",
188
+ "output_dir": config.get("output_dir", ""),
189
+ "sources": sources,
190
+ })
litloop/config.py ADDED
@@ -0,0 +1,413 @@
1
+ """Configuration management for LitLoop."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ DATA_DIR = Path(os.environ.get("LITLOOP_DATA_DIR", str(Path.home() / ".litloop")))
12
+ CONFIG_FILE = DATA_DIR / "config.json"
13
+
14
+ DEFAULT_SCIHUB_DOMAINS = [
15
+ # Direct PDF mirrors (no CAPTCHA, PDF via sci.bban.top iframe) — verified 2026-07
16
+ # vg/al 现为 Turnstile 门/首页壳(2026-08-31 实测),保留用于人工点一次模式
17
+ "https://sci-hub.vg",
18
+ "https://sci-hub.al",
19
+ "https://sci-hub.mk",
20
+ # 2026-08-31 深夜直连实测新增:bz/mksa 稳定出文章页(iframe PDF,无验证墙)
21
+ "https://sci-hub.bz",
22
+ "https://sci-hub.mksa.top",
23
+ # ALTCHA-protected (stable; used as manual-download hint)
24
+ "https://sci-hub.ru",
25
+ "https://sci-hub.ee",
26
+ # Cloudflare-protected (requires CloakBrowser JS challenge bypass)
27
+ "https://sci-hub.st",
28
+ # Reported intermittent (2026-06); kept but deprioritized
29
+ "https://sci-hub.mksa.top",
30
+ # Legacy (currently down, kept for future recovery)
31
+ "https://sci-hub.se",
32
+ "https://sci-hub.is",
33
+ "https://sci-hub.41610.org",
34
+ ]
35
+
36
+ DEFAULT_CONFIG: dict[str, Any] = {
37
+ "email": "litloop@example.invalid",
38
+ "output_dir": str(DATA_DIR / "papers"),
39
+ "cache_dir": str(DATA_DIR / "cache"),
40
+ "network_proxy": "",
41
+ "proxy_pool": "", # 逗号分隔的代理列表;按代理轮换出口 IP 的批量下载(publisher_batch)已于 2026-10-20 删除,现在不起作用
42
+ "download_strategy": "fastest", # fastest / grey_only(all 3 grey) / scihub_only(Sci-Hub only) / scihub_first / oa_first / legal_only
43
+ "race_mode": "hedge", # hedge: score-ordered staggered cascade (fewer requests, less anti-bot heat) / full: flat parallel race
44
+ "hedge_delay_seconds": 1.5, # hedged cascade: wait this long before widening to the next lane
45
+ "batch_default_lanes": True, # batch downloads default to pretriage + channel-lane scheduling (fast HTTP -> grey -> institutional)
46
+ "lane_s2_batch_min": 10, # minimum batch size to enrich OA URLs via the S2 batch endpoint (500 ids/request)
47
+ "lane_mdpi_cdn": True, # fast lane: construct mdpi-res.com CDN URLs for 10.3390 DOIs (bypasses the bot-walled main site)
48
+ # Grey sources (Sci-Hub, LibGen, SciBban) only when the user turns them on (2026-10-16).
49
+ "scihub_enabled": False,
50
+ "scihub_domains": DEFAULT_SCIHUB_DOMAINS,
51
+ "vpnsci_enabled": False,
52
+ "vpnsci_school": "",
53
+ "vpnsci_base_url": "",
54
+ "vpnsci_cookie_file": "",
55
+ "carsi_enabled": False,
56
+ "carsi_idp_name": "",
57
+ "ezproxy_enabled": False,
58
+ "ezproxy_login_url": "",
59
+ # 机构会话自愈:下载走机构渠道前自动校验 WebVPN/CARSI 会话,
60
+ # 已过期时打开浏览器重新登录。仅当存在历史 cookie 且校验明确判定
61
+ # 过期才触发;新用户或网络不可达绝不弹浏览器。
62
+ "auto_relogin": True,
63
+ "core_api_key": "",
64
+ "openalex_api_key": "",
65
+ "elsevier_api_key": "",
66
+ "springer_api_key": "", # Springer Nature Full Text API (TDM) — entitlement follows the institution (ORCID-affiliated key, needs a Springer subscription with TDM rights)
67
+ "elsevier_insttoken": "",
68
+ "connect_timeout": 15,
69
+ "read_timeout": 30,
70
+ "request_delay_min": 2.0,
71
+ "request_delay_max": 5.0,
72
+ "fixed_request_delay_enabled": False,
73
+ "json_probe_cache_seconds": 3600,
74
+ "cache_ttl_hours": 168,
75
+ "search_cache_ttl_hours": 168,
76
+ "parallel_sources": True,
77
+ "parallel_probes": True,
78
+ "batch_workers": 10,
79
+ "batch_stagger_seconds": 0.3,
80
+ "min_pdf_size_bytes": 10000,
81
+ "browser_headless": False,
82
+ # 出版商官网的专用窗口先最小化,需要登录或人机验证时再弹到前台
83
+ "publisher_window_minimized": True,
84
+ # 开放获取一轮:直链被挡时用浏览器取一次,最多等这么多秒(自定值;没下到的逐篇下载时再试)
85
+ "oa_browser_fallback_timeout": 25,
86
+ "browser_humanize": True,
87
+ # 浏览器后端:patchright(默认,Apache-2.0 开源 playwright fork,内核随本机 Chrome 自动更新)
88
+ # 或 cloakbrowser(免费版内核卡 Chromium 146,作为可选回退)
89
+ "browser_backend": "patchright",
90
+ # 批量下载时每 N 篇回收一次浏览器上下文(cookie 内存交接,登录态不丢)。
91
+ # 0 = 关闭。长批次(上千篇)建议 100-200,避免 Chrome 长会话内存漂移。
92
+ "browser_restart_every": 0,
93
+ # 批量下载成功后自动抓取附件/补充材料(SI)。默认关——大多数任务只要主 PDF。
94
+ # 开启后成功论文会尝试抓取出版商补充材料,存为 {DOI}_SI{n}.{ext},并写 si_manifest.json。
95
+ "fast_retry_wait_sec": 60,
96
+ # 冷却重试期间的逐篇间隔(秒)。实测 15s/篇可 100% 绕开 Elsevier API 限流。
97
+ "fast_retry_delay_sec": 15,
98
+ # Turnstile 交互门(如 sci-hub.vg):无头会话下无法点击,默认跳过并冷却。
99
+ # scihub_browser_headless=false 时可开启人工点一次模式(点一次整批复用)。
100
+ "scihub_turnstile_click": True,
101
+ "turnstile_wait_sec": 180,
102
+ # 机构级联并行 fetcher 数(各持独立登录会话)。默认 1 = 串行;2 可省约一半
103
+ # Phase 2 时间,代价是同 IP 双浏览器会话。
104
+ "institutional_workers": 1,
105
+ # 灰色源竞速浏览器的独立无头开关(只影响 sci-hub 竞速,机构登录仍有可见窗口)。
106
+ # true = 竞速全程零窗口、零任务栏闪烁;指纹安全性由 UA 清洗保证。
107
+ "scihub_browser_headless": False,
108
+ "scihub_browser_first": True, # false = pure-HTTP Sci-Hub lane (use when the PDF CDN challenges headless browsers but serves plain requests)
109
+ # 浏览器内核选择(CloakBrowser 免费版内置 Chromium 146 已过老,遇 Cloudflare Turnstile 会反复验证):
110
+ # browser_executable: 显式指定浏览器二进制路径(本机 Chrome/Edge);留空=自动探测
111
+ # browser_auto_upgrade: True 时自动探测本机 Chrome/Edge(版本 > 146 优先于内置 stealth Chromium)
112
+ "browser_executable": "",
113
+ "browser_auto_upgrade": True,
114
+ # browser_executable 指向本机 Chrome 的副本时,启动前若系统 Chrome 已升级就同步更新副本
115
+ "browser_copy_follow_system": True,
116
+ "is_campus_network": False,
117
+ "tor_proxy": os.environ.get("TOR_PROXY", ""),
118
+ # Optional Springer Nature Meta API key (not the TDM key above): abstracts of Springer Nature papers,
119
+ # free plan 500 requests/day. See docs/API_LIMITS.md.
120
+ "springer_meta_api_key": "",
121
+ "springer_meta_daily_limit": 500,
122
+ "tor_use_bridges": False,
123
+ "use_tor_for_scihub": True,
124
+ "google_scholar_limit": 5,
125
+ "max_browser_workers": 1,
126
+ "scihub_browser_workers": 3, # Number of Sci-Hub domains to race in parallel via browser
127
+ "host_concurrency": {},
128
+ "auto_rename": True, # older switch; false means rename_format "off"
129
+ "rename_format": "author_year_title_doi", # see rename.FORMATS
130
+ # Fetch abstracts missing from search results (Semantic Scholar, then
131
+ # Scopus with elsevier_api_key, then Crossref) so screening reads them.
132
+ "search_fill_abstracts": True,
133
+ # Chinese abstracts in the app (translate.py): which ratings are translated
134
+ # without asking ("高", "高中", "all", "off"), and by whom: "deepseek" /
135
+ # "deepl" (the app, when the paper is opened; needs the key) or "agent"
136
+ # (the agent, when it rates: costs conversation tokens).
137
+ "abstract_translation": "高",
138
+ # Look for new versions of LitLoop, easyseek and zotero-mcp every N days
139
+ # (updates.py; 0 = never). Only a notice in results; nothing is installed.
140
+ "update_check_days": 7,
141
+ # Batches up to this many papers: Sci-Hub for all first (in parallel: plain
142
+ # HTTP), then a full single download (school routes included) for each miss,
143
+ # batch_small_workers at a time (pipeline.run_scihub_then_single). Those are
144
+ # the papers that need the browser, of which there is one
145
+ # (max_browser_workers): one at a time, so no paper's race clock runs out
146
+ # while it queues for the browser and each login window is for one paper.
147
+ # Longer lists use the lanes.
148
+ # The school's routes after phase 1, one after another (sources
149
+ # _INSTITUTIONAL_STEPS): no-login ones first, so one window at a time asks
150
+ # the user. Steps switched off in settings are skipped.
151
+ "institutional_order": ["webvpn_http", "webvpn_browser", "carsi", "ezproxy"],
152
+ # Late results after the 20 s race: a browser lane still at work gets up to
153
+ # race_grace_browser_seconds, otherwise race_grace_seconds; a lane waiting
154
+ # for the user is waited for as long as it says so (up to 10 min).
155
+ # Download full text through the Elsevier API (abstracts and Scopus always
156
+ # use the key). Off when the key has no full-text entitlement; LitLoop also
157
+ # learns that from the API's 403 and stops asking for 30 days.
158
+ "elsevier_api_fulltext": True,
159
+ # Sci-Hub and LibGen through host_proxy_url (the user's proxy) instead of
160
+ # the school's exit; direct when that proxy's port is closed.
161
+ "scihub_via_proxy": False,
162
+ # Papers published in or after this year skip Sci-Hub (and SciBban): it
163
+ # has almost none of them. The year comes from the app's list or the DOI
164
+ # (j.ceramint.2026.06.274); a paper with no known year is still asked. 0: off.
165
+ "scihub_skip_from_year": 2024,
166
+ # Route for Sci-Hub and LibGen: "direct", "proxy" (host_proxy_url only) or
167
+ # "auto" (a batch first measures mirrors on both routes, the fastest wins;
168
+ # sources/scihub_speed.py). Empty: "proxy" if scihub_via_proxy else "direct".
169
+ "scihub_route": "direct",
170
+ # The speed test: a paper Sci-Hub has (about 1 MB), how many mirrors, how long.
171
+ "scihub_speedtest_doi": "10.1016/j.colsurfb.2017.09.051",
172
+ "scihub_speedtest_mirrors": 5,
173
+ "scihub_speedtest_seconds": 40,
174
+ # A short list's first round: papers asked of Sci-Hub at once (HTTP only),
175
+ # and the workers their mirror requests share. More papers at once risks
176
+ # Sci-Hub's per-IP rate limit.
177
+ "scihub_round_workers": 10,
178
+ "scihub_http_workers": 12,
179
+ "race_grace_seconds": 15,
180
+ "race_grace_browser_seconds": 180,
181
+ # A school route (institutional_order) with no result after this long has its
182
+ # browsers closed and the next route runs; a login the user is doing extends it.
183
+ "institutional_step_seconds": 240,
184
+ # After a login to the publisher site with the school's identity still gives
185
+ # no full text, the school routes (WebVPN, CARSI, EZProxy) are skipped: they
186
+ # reach the same publisher as the same school. True tries them anyway.
187
+ "school_routes_after_publisher_login": False,
188
+ # DOI prefixes whose publisher site phase 1 does not open (2026-10-18 user
189
+ # decision): Taylor & Francis books (CRC 10.1201, Routledge 10.4324). The
190
+ # T&F site answers in about 12 s per request from here and has no chapter
191
+ # PDFs; WebVPN downloads the whole book directly.
192
+ "publisher_site_skip_prefixes": ["10.1201", "10.4324"],
193
+ "batch_small_max": 30,
194
+ "batch_small_workers": 1,
195
+ # Desktop app: the first-run guide (app_settings.STEPS) was finished or skipped.
196
+ "onboarding_done": False,
197
+ # Desktop app: searches started from the app are sent to this agent as a
198
+ # message (agent_runner.py); the prompt goes on stdin. Claude Code by default:
199
+ # stream-json lets the app show which tool it is using; only the litloop
200
+ # tools and skills are allowed, since nobody is there to approve anything.
201
+ # Which backend runs a search started in the app: "command" (backend A,
202
+ # the agent above) or "builtin" (backend B, the model API configured below).
203
+ "agent_backend": "command",
204
+ "agent_command": "claude",
205
+ # An agent run that prints nothing for this long is stopped (it hung or
206
+ # waits on a prompt nobody sees). A deep-search batch takes ~1-3 minutes.
207
+ "agent_idle_minutes": 15,
208
+ "agent_args": "-p --output-format stream-json --verbose --allowedTools mcp__litloop Skill Read",
209
+ # Backend B: the app calls a model API itself instead of starting an agent
210
+ # command. "openai" also covers DeepSeek, 通义, Kimi, GLM, OpenRouter, Ollama
211
+ # and vLLM; the address is what points at the one you use.
212
+ "builtin_agent_provider": "openai",
213
+ "builtin_agent_base_url": "",
214
+ "builtin_agent_api_key": "",
215
+ "builtin_agent_model": "",
216
+ "translation_provider": "deepseek",
217
+ "deepseek_api_key": "",
218
+ "deepl_api_key": "",
219
+ # zotero-mcp (github.com/54yyyu/zotero-mcp), the only route to Zotero: the
220
+ # Python that has it installed ("" = find it automatically), and an optional
221
+ # script that sets its ZOTERO_* env vars (its serve call must be under __main__).
222
+ "zotero_mcp_python": "",
223
+ "zotero_mcp_env_script": "",
224
+ "zotero_push_timeout": 900,
225
+ # Where on-demand components (the layout model) come from. Empty = PyPI,
226
+ # which is slow from some networks; a mirror URL avoids a 65 MB download
227
+ # crawling. Set in 设置 → 可选组件.
228
+ "package_index_url": "",
229
+ "watch_check_days": 7,
230
+ "deep_exact_queries": True, # and-style short phrases, plus a bonus when the phrase is in the title
231
+ # Desktop app: push a download job to Zotero when it finishes (off: the
232
+ # user pushes from the job page), into <root>/<the job's title>.
233
+ "zotero_auto_push": False,
234
+ "zotero_collection_root": "LitLoop",
235
+ # Per-host proxy exception: these domains (and subdomains) go through
236
+ # host_proxy_url, everything else stays direct. For publishers that grant
237
+ # access by login cookie rather than campus IP, where the direct route is
238
+ # slow (Springer Nature's CDN). Skipped while the proxy port is closed.
239
+ # false: Sci-Hub/LibGen never open a browser (ALTCHA is solved over HTTP;
240
+ # mirrors behind Cloudflare Turnstile are skipped).
241
+ "scihub_use_browser": True,
242
+ "host_proxy_url": "",
243
+ "host_proxy_hosts": [],
244
+ "cookie_path": "",
245
+ "chrome_profile_dir": "",
246
+ "carsi_cookie_dir": "",
247
+ }
248
+
249
+
250
+ ENV_KEYS = {"openalex_api_key": "OPENALEX_API_KEY", "elsevier_api_key": "ELSEVIER_API_KEY",
251
+ "springer_meta_api_key": "SPRINGER_META_API_KEY"}
252
+
253
+
254
+ def load_config() -> dict[str, Any]:
255
+ config = DEFAULT_CONFIG.copy()
256
+ if CONFIG_FILE.exists():
257
+ try:
258
+ with CONFIG_FILE.open("r", encoding="utf-8") as fh:
259
+ existing = json.load(fh)
260
+ if isinstance(existing, dict):
261
+ config.update(existing)
262
+ except Exception:
263
+ pass
264
+ for key, value in DEFAULT_CONFIG.items():
265
+ config.setdefault(key, value)
266
+ for key, env in ENV_KEYS.items(): # the config file wins; the environment fills the gap
267
+ if not config.get(key) and os.environ.get(env):
268
+ config[key] = os.environ[env]
269
+ return config
270
+
271
+
272
+ def save_config(config: dict[str, Any]) -> None:
273
+ """Write only what differs from DEFAULT_CONFIG, so a later change of a default
274
+ reaches everyone who never set that key (load_config fills defaults back in)."""
275
+ # Keys that came from the environment stay there: never write them to disk.
276
+ config = {k: ("" if k in ENV_KEYS and v and v == os.environ.get(ENV_KEYS[k]) else v) for k, v in config.items()}
277
+ config = {k: v for k, v in config.items() if k not in DEFAULT_CONFIG or v != DEFAULT_CONFIG[k]}
278
+ DATA_DIR.mkdir(parents=True, exist_ok=True)
279
+ with CONFIG_FILE.open("w", encoding="utf-8") as fh:
280
+ json.dump(config, fh, indent=2, ensure_ascii=False)
281
+
282
+
283
+ _VALIDATION_RULES: dict[str, tuple[type, Any, Any]] = {
284
+ # key: (type, min_value, max_value)
285
+ "connect_timeout": (int, 1, 60),
286
+ "read_timeout": (int, 1, 120),
287
+ "request_delay_min": (float, 0.0, 30.0),
288
+ "request_delay_max": (float, 0.0, 60.0),
289
+ "json_probe_cache_seconds": (int, 0, 86400),
290
+ "cache_ttl_hours": (int, 1, 8760),
291
+ "search_cache_ttl_hours": (int, 0, 8760),
292
+ "batch_workers": (int, 1, 50),
293
+ "batch_stagger_seconds": (float, 0.0, 10.0),
294
+ "min_pdf_size_bytes": (int, 100, 1000000),
295
+ "google_scholar_limit": (int, 1, 50),
296
+ }
297
+
298
+ _VALID_STRATEGIES = frozenset({"fastest", "scihub_first", "scihub_only", "grey_only", "oa_first", "legal_only"})
299
+
300
+
301
+ def update_config(key: str, value: str) -> dict[str, Any]:
302
+ import warnings as _warnings
303
+
304
+ config = load_config()
305
+
306
+ # Validate: warn on unknown keys
307
+ if key not in DEFAULT_CONFIG:
308
+ _warnings.warn(
309
+ f"Unknown config key '{key}' — it will be stored but may have no effect. "
310
+ f"Valid keys include: {', '.join(sorted(DEFAULT_CONFIG.keys()))}",
311
+ stacklevel=2,
312
+ )
313
+
314
+ # Special handling for download_strategy
315
+ if key == "download_strategy":
316
+ value_lower = value.lower().strip()
317
+ if value_lower not in _VALID_STRATEGIES:
318
+ raise ValueError(
319
+ f"Invalid download_strategy '{value}'. Valid options: {', '.join(sorted(_VALID_STRATEGIES))}"
320
+ )
321
+ config[key] = value_lower
322
+ save_config(config)
323
+ return config
324
+
325
+ if key in config:
326
+ old_type = type(config[key])
327
+ if old_type == bool:
328
+ config[key] = value.lower() in ("true", "1", "yes")
329
+ elif old_type == int:
330
+ try:
331
+ config[key] = int(value)
332
+ except ValueError:
333
+ raise ValueError(f"Invalid integer value for '{key}': '{value}'")
334
+ elif old_type == float:
335
+ try:
336
+ config[key] = float(value)
337
+ except ValueError:
338
+ raise ValueError(f"Invalid float value for '{key}': '{value}'")
339
+ elif old_type == list:
340
+ # Try JSON parse for list values (e.g. scihub_domains)
341
+ try:
342
+ parsed = json.loads(value)
343
+ if isinstance(parsed, list):
344
+ config[key] = parsed
345
+ else:
346
+ raise ValueError(f"Expected a JSON list for '{key}', got {type(parsed).__name__}")
347
+ except json.JSONDecodeError:
348
+ raise ValueError(
349
+ f"Invalid list value for '{key}'. Use JSON format, e.g.: '[\"https://sci-hub.vg\",\"https://sci-hub.al\"]'"
350
+ )
351
+ else:
352
+ config[key] = value
353
+ else:
354
+ config[key] = value
355
+
356
+ if key in _VALIDATION_RULES:
357
+ _, min_val, max_val = _VALIDATION_RULES[key]
358
+ if config[key] < min_val or config[key] > max_val:
359
+ config[key] = DEFAULT_CONFIG[key]
360
+
361
+ save_config(config)
362
+ return config
363
+
364
+
365
+ SENSITIVE_KEYS = [
366
+ "core_api_key",
367
+ "vpnsci_cookie_file",
368
+ "zotero_api_key", # no longer used (Zotero goes through zotero-mcp); old configs still masked
369
+ "zotero_library_id",
370
+ "elsevier_api_key",
371
+ "elsevier_insttoken",
372
+ ]
373
+
374
+ _PROXY_URL_CREDS_RE = re.compile(r"(//[^/@\s]+:)[^@\s]+@")
375
+
376
+
377
+ def mask_config_value(key: str, value: Any) -> Any:
378
+ """Return a display-safe copy of a config value.
379
+
380
+ Secrets (API keys, tokens, cookie files) are fully masked; proxy URLs keep
381
+ host/port but hide the password (``http://user:***@host:port``).
382
+ """
383
+ if value is None:
384
+ return value
385
+ if key in SENSITIVE_KEYS or key.endswith(("_api_key", "_token", "_insttoken")):
386
+ return "***" if value else value
387
+ if "proxy" in key.lower() and isinstance(value, str) and "@" in value:
388
+ return _PROXY_URL_CREDS_RE.sub(r"\1***@", value)
389
+ return value
390
+
391
+
392
+ def get_config_safe() -> dict[str, Any]:
393
+ config = load_config()
394
+ return {key: mask_config_value(key, value) for key, value in config.items()}
395
+
396
+
397
+ def parse_proxy_pool(value: str | None) -> list[str]:
398
+ """Parse a comma-separated proxy list into a deduplicated list.
399
+
400
+ Accepts forms like ``"socks5://1.1.1.1:1080, http://2.2.2.2:8080"`` and
401
+ returns ``["socks5://1.1.1.1:1080", "http://2.2.2.2:8080"]``. Empty/blank
402
+ entries are dropped. Order is preserved; duplicates removed.
403
+ """
404
+ if not value:
405
+ return []
406
+ seen: set[str] = set()
407
+ proxies: list[str] = []
408
+ for token in str(value).split(","):
409
+ proxy = token.strip()
410
+ if proxy and proxy not in seen:
411
+ seen.add(proxy)
412
+ proxies.append(proxy)
413
+ return proxies
@@ -0,0 +1,62 @@
1
+ The spelling tables spelling_us.tsv.gz and spelling_search.tsv.gz in this
2
+ directory are derived from VarCon (Variant Conversion Info), taken from
3
+ https://github.com/en-wl/wordlist (varcon/varcon.txt), by
4
+ scripts/build_spelling_table.py. Its copyright and permission notices follow.
5
+
6
+ Copyright
7
+ =========
8
+
9
+ Copyright 2000-2019 by Kevin Atkinson
10
+
11
+ Permission to use, copy, modify, distribute and sell this array, the
12
+ associated software, and its documentation for any purpose is hereby
13
+ granted without fee, provided that the above copyright notice appears
14
+ in all copies and that both that copyright notice and this permission
15
+ notice appear in supporting documentation. Kevin Atkinson makes no
16
+ representations about the suitability of this array for any
17
+ purpose. It is provided "as is" without express or implied warranty.
18
+
19
+ Copyright 2016 by Benjamin Titze
20
+
21
+ Permission to use, copy, modify, distribute and sell this array, the
22
+ associated software, and its documentation for any purpose is hereby
23
+ granted without fee, provided that the above copyright notice appears
24
+ in all copies and that both that copyright notice and this permission
25
+ notice appear in supporting documentation. Benjamin Titze makes no
26
+ representations about the suitability of this array for any
27
+ purpose. It is provided "as is" without express or implied warranty.
28
+
29
+ Since the original words lists come from the Ispell distribution:
30
+
31
+ Copyright 1993, Geoff Kuenning, Granada Hills, CA
32
+ All rights reserved.
33
+
34
+ Redistribution and use in source and binary forms, with or without
35
+ modification, are permitted provided that the following conditions
36
+ are met:
37
+
38
+ 1. Redistributions of source code must retain the above copyright
39
+ notice, this list of conditions and the following disclaimer.
40
+ 2. Redistributions in binary form must reproduce the above copyright
41
+ notice, this list of conditions and the following disclaimer in the
42
+ documentation and/or other materials provided with the distribution.
43
+ 3. All modifications to the source code must be clearly marked as
44
+ such. Binary redistributions based on modified source code
45
+ must be clearly marked as modified versions in the documentation
46
+ and/or other materials provided with the distribution.
47
+ (clause 4 removed with permission from Geoff Kuenning)
48
+ 5. The name of Geoff Kuenning may not be used to endorse or promote
49
+ products derived from this software without specific prior
50
+ written permission.
51
+
52
+ THIS SOFTWARE IS PROVIDED BY GEOFF KUENNING AND CONTRIBUTORS ``AS IS'' AND
53
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
54
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
55
+ ARE DISCLAIMED. IN NO EVENT SHALL GEOFF KUENNING OR CONTRIBUTORS BE LIABLE
56
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
57
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
58
+ OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
59
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
60
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
61
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62
+ SUCH DAMAGE.