litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,571 @@
1
+ """Download jobs started from the desktop app.
2
+
3
+ A job is a list of papers downloaded by the normal batch engine
4
+ (``server._batch_download_impl``: lanes, racing, institutional cascade) in a
5
+ background thread of the app backend. Its state lives in one file,
6
+ ``<litloop home>/downloads/<job>.json``, written through ``store`` (locked,
7
+ replaced in one step) so the app can poll it while the job runs::
8
+
9
+ {"schema": 1, "job": "20261017-101500", "title": "...", "session": "...",
10
+ "state": "running" | "queued" | "done" | "cancelled" | "failed" | "interrupted",
11
+ "created": "...", "finished": "...", "output_dir": "...", "error": "...",
12
+ "items": [{"doi", "title", "year", "journal", "publisher",
13
+ "status": "waiting" | "queued" | "done" | "failed",
14
+ "source", "file", "error",
15
+ "zotero": {"status": "pushing" | "added" | "reused" | "no_pdf" | "failed",
16
+ "key", "error"}}],
17
+ "zotero": {"state": "running" | "done" | "failed", "collection", "key",
18
+ "started", "finished", "error"}}
19
+
20
+ Papers are marked done/failed one by one as the engine reports them
21
+ (``progress_reporter.advance``); source, file and error are filled in when
22
+ the batch returns. One job runs at a time: the engine's progress report and
23
+ cancel switch are process-wide.
24
+
25
+ A long list (more than ``batch_small_max`` papers) runs only the rounds that
26
+ need nobody, open access and Sci-Hub; the rest is ``queued`` and waits in one
27
+ queue per publisher (``queues``), which the user starts when they can log in
28
+ or pass a check (``start_queue``): its papers then go one by one through the
29
+ publisher's site and the school's routes, so one publisher's logins and checks
30
+ come together. The job is ``queued`` while any paper waits in a queue.
31
+
32
+ Downloaded papers are pushed to Zotero (zotero.push_via_zotero_mcp) by hand
33
+ from the job page, or right after the download when ``zotero_auto_push`` is
34
+ on, into ``<zotero_collection_root>/<job title>`` unless another collection
35
+ is chosen. A push runs in its own thread, one at a time, and can overlap a
36
+ download.
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ import json
42
+ import re
43
+ import threading
44
+ import time
45
+ from pathlib import Path
46
+ from typing import Any
47
+
48
+ from .. import store
49
+ from ..identifiers import canonical_doi
50
+
51
+ _RUN_LOCK = threading.Lock()
52
+ _CURRENT: dict[str, Any] = {"job": None, "cancelled": False}
53
+ _PUSH_LOCK = threading.Lock()
54
+ _PUSHING: dict[str, Any] = {"job": None}
55
+
56
+
57
+ class Busy(RuntimeError):
58
+ """Another download job is still running."""
59
+
60
+
61
+ def jobs_dir() -> Path:
62
+ from ..config import DATA_DIR
63
+ d = Path(DATA_DIR) / "downloads"
64
+ d.mkdir(parents=True, exist_ok=True)
65
+ return d
66
+
67
+
68
+ def job_path(job: str) -> Path:
69
+ return jobs_dir() / f"{job}.json"
70
+
71
+
72
+ def _settled(d: dict[str, Any]) -> dict[str, Any]:
73
+ """A job still marked running that this process is not running was cut off (app closed)."""
74
+ if d.get("state") == "running" and d.get("job") != _CURRENT["job"]:
75
+ d["state"] = "interrupted"
76
+ z = d.get("zotero")
77
+ if z and z.get("state") == "running" and d.get("job") != _PUSHING["job"]:
78
+ z["state"] = "interrupted"
79
+ for it in d.get("items", []):
80
+ if (it.get("zotero") or {}).get("status") == "pushing":
81
+ it["zotero"]["status"] = "failed"
82
+ return d
83
+
84
+
85
+ OTHER_PUBLISHER = "其他出版商"
86
+ _CORPORATE = re.compile(r"[\s,]+(?:\([A-Z]{2,}\)|B\.?V\.?|Ltd\.?|Limited|Inc\.?|LLC|GmbH|plc|AG)$", re.I)
87
+
88
+
89
+ def publisher_name(raw: str | None) -> str:
90
+ """OpenAlex's host organization without its company form ("Elsevier BV" -> "Elsevier")."""
91
+ name, before = (raw or "").strip(), None
92
+ while name != before:
93
+ before, name = name, _CORPORATE.sub("", name).strip()
94
+ return name
95
+
96
+
97
+ def queues(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
98
+ """The papers waiting in a queue, one queue per publisher, the largest first."""
99
+ count: dict[str, int] = {}
100
+ for it in items:
101
+ if it.get("status") == "queued":
102
+ name = it.get("publisher") or OTHER_PUBLISHER
103
+ count[name] = count.get(name, 0) + 1
104
+ return [{"publisher": k, "count": n}
105
+ for k, n in sorted(count.items(), key=lambda kv: (kv[0] == OTHER_PUBLISHER, -kv[1], kv[0]))]
106
+
107
+
108
+ def list_jobs(limit: int = 50) -> list[dict[str, Any]]:
109
+ """Newest first, without the item lists (counts instead)."""
110
+ out = []
111
+ for p in sorted(jobs_dir().glob("*.json"), reverse=True)[:limit]:
112
+ try:
113
+ d = _settled(store.read_json(p))
114
+ except (OSError, ValueError):
115
+ continue
116
+ items = d.get("items", [])
117
+ out.append({k: d.get(k) for k in ("job", "title", "session", "state", "created", "finished")}
118
+ | {"total": len(items), "done": sum(1 for i in items if i["status"] == "done"),
119
+ "failed": sum(1 for i in items if i["status"] == "failed"),
120
+ "queued": sum(1 for i in items if i["status"] == "queued")})
121
+ return out
122
+
123
+
124
+ def get(job: str) -> dict[str, Any]:
125
+ d = _settled(store.read_json(job_path(job)))
126
+ d["queues"] = queues(d.get("items", []))
127
+ return d
128
+
129
+
130
+ def running() -> str | None:
131
+ return _CURRENT["job"]
132
+
133
+
134
+ LOCAL = "本机已有" # the source of a paper found in the save folder, not downloaded again
135
+ _ARXIV_DOI = "10.48550/arxiv."
136
+
137
+
138
+ def same_paper(doi: str) -> set[str]:
139
+ """The keys one paper can go by: an arXiv id and its arXiv DOI are the same paper."""
140
+ d = canonical_doi(doi or "")
141
+ if d.startswith(_ARXIV_DOI):
142
+ return {d, d[len(_ARXIV_DOI):]}
143
+ if d and not d.startswith("10."): # a bare arXiv id
144
+ return {d, _ARXIV_DOI + d}
145
+ return {d} if d else set()
146
+
147
+
148
+ def local_files(dois: list[str] | set[str] | None = None) -> dict[str, str]:
149
+ """DOI -> PDF already in the save folder, from its DOI index (.doi_index.json).
150
+
151
+ Only files that are still there count; age does not matter (a PDF on disk
152
+ does not expire, whatever cache_ttl_hours says about re-checking sources).
153
+ """
154
+ from ..config import load_config
155
+ folder = Path(str(load_config().get("output_dir") or "")).expanduser()
156
+ try:
157
+ idx = json.loads((folder / ".doi_index.json").read_text(encoding="utf-8"))
158
+ except (OSError, ValueError):
159
+ return {}
160
+ found = {}
161
+ for key, entry in idx.items():
162
+ f = entry.get("file") if isinstance(entry, dict) else entry
163
+ if f and Path(f).is_file():
164
+ for k in same_paper(key):
165
+ found[k] = str(f)
166
+ if dois is None:
167
+ return found
168
+ return {canonical_doi(d): found[k] for d in dois for k in same_paper(d) if k in found}
169
+
170
+
171
+ def downloaded_dois() -> set[str]:
172
+ """DOIs some job has already got, or whose PDF is already in the save folder:
173
+ they are not offered again."""
174
+ got = {k for d in local_files() for k in same_paper(d)}
175
+ for p in jobs_dir().glob("*.json"):
176
+ try:
177
+ got |= {i["doi"] for i in store.read_json(p).get("items", []) if i.get("status") == "done"}
178
+ except (OSError, ValueError):
179
+ continue
180
+ return got
181
+
182
+
183
+ def queued_dois() -> set[str]:
184
+ """DOIs waiting in a long list's publisher queue: not offered again either."""
185
+ out: set[str] = set()
186
+ for p in jobs_dir().glob("*.json"):
187
+ try:
188
+ out |= {i["doi"] for i in store.read_json(p).get("items", []) if i.get("status") == "queued"}
189
+ except (OSError, ValueError):
190
+ continue
191
+ return out
192
+
193
+
194
+ def start(papers: list[dict[str, Any]], title: str, session: str | None = None,
195
+ zotero_target: str | None = None) -> dict[str, Any]:
196
+ """Create a job for papers ({doi, title, year, journal}) and start it in the background.
197
+
198
+ zotero_target: the collection a later push goes to by default (the attachment
199
+ check sets it to the collection being repaired).
200
+ """
201
+ items, seen = [], set()
202
+ for p in papers:
203
+ doi = canonical_doi(p.get("doi") or "")
204
+ if not doi or doi in seen:
205
+ continue
206
+ seen.add(doi)
207
+ items.append({"doi": doi, "title": p.get("title") or doi, "year": p.get("year"),
208
+ "journal": p.get("journal"), "status": "waiting", "source": None, "file": None, "error": None})
209
+ if not items:
210
+ raise ValueError("无法下载:没有带 DOI 的文献")
211
+ # Already in the save folder: done at once, the engine is not asked again.
212
+ for it, f in ((it, local_files([it["doi"]]).get(it["doi"])) for it in items):
213
+ if f:
214
+ it.update(status="done", source=LOCAL, file=f)
215
+ if not _RUN_LOCK.acquire(blocking=False):
216
+ raise Busy("another download is still running")
217
+ try:
218
+ from ..config import load_config
219
+ job = time.strftime("%Y%m%d-%H%M%S")
220
+ while job_path(job).exists():
221
+ time.sleep(1)
222
+ job = time.strftime("%Y%m%d-%H%M%S")
223
+ data = {"schema": store.SCHEMA, "job": job, "title": title, "session": session, "state": "running",
224
+ "created": time.strftime("%Y-%m-%d %H:%M:%S"), "finished": None,
225
+ "output_dir": str(load_config().get("output_dir") or ""), "error": None, "items": items,
226
+ "zotero_target": (zotero_target or "").strip().strip("/") or None}
227
+ store.write_json(job_path(job), data, indent=1)
228
+ _CURRENT.update(job=job, cancelled=False)
229
+ todo = sum(1 for it in items if it["status"] != "done")
230
+ rounds = "free" if todo > int(load_config().get("batch_small_max", 30) or 0) else "all"
231
+ threading.Thread(target=_run, args=(job,), kwargs={"rounds": rounds}, name=f"download-{job}", daemon=True).start()
232
+ return data
233
+ except BaseException:
234
+ _CURRENT.update(job=None)
235
+ _RUN_LOCK.release()
236
+ raise
237
+
238
+
239
+ def start_queue(job: str, publisher: str) -> dict[str, Any]:
240
+ """Download the papers of one publisher's queue (see the module doc)."""
241
+ path = job_path(job)
242
+ dois = [it["doi"] for it in get(job)["items"]
243
+ if it["status"] == "queued" and (it.get("publisher") or OTHER_PUBLISHER) == publisher]
244
+ if not dois:
245
+ raise ValueError("无法开始:此队列里没有待下载的文献")
246
+ if not _RUN_LOCK.acquire(blocking=False):
247
+ raise Busy("another download is still running")
248
+ try:
249
+ with store.update(path, indent=1) as data:
250
+ for it in data["items"]:
251
+ if it["doi"] in dois:
252
+ it.update(status="waiting", error=None)
253
+ data.update(state="running", finished=None, error=None)
254
+ _CURRENT.update(job=job, cancelled=False)
255
+ threading.Thread(target=_run, args=(job,), kwargs={"only": dois, "rounds": "single"},
256
+ name=f"download-{job}", daemon=True).start()
257
+ return get(job)
258
+ except BaseException:
259
+ _CURRENT.update(job=None)
260
+ _RUN_LOCK.release()
261
+ raise
262
+
263
+
264
+ def cancel() -> bool:
265
+ """Ask the running job to stop: pending papers are skipped, browser windows closed."""
266
+ if not _CURRENT["job"]:
267
+ return False
268
+ from ..sources import cancel_downloads
269
+ _CURRENT["cancelled"] = True
270
+ cancel_downloads()
271
+ return True
272
+
273
+
274
+ def _mark(job: str, doi: str, **fields: Any) -> None:
275
+ with store.update(job_path(job), indent=1) as data:
276
+ for it in data["items"]:
277
+ if it["doi"] == doi:
278
+ it.update(fields)
279
+
280
+
281
+ def _fill_meta(path: Path, wait_s: float = 10.0, publishers: bool = False) -> None:
282
+ """Papers without a publication year (a pasted list has none) get it from
283
+ OpenAlex, so the Sci-Hub year rule (scihub_skip_from_year) applies to them
284
+ too; the years also go into the job file. With ``publishers`` (a long
285
+ list) every paper also gets its publisher, which names its queue. The job waits at most
286
+ ``wait_s``; years that come later still reach the papers not yet started."""
287
+ missing = [i["doi"] for i in store.read_json(path)["items"]
288
+ if i.get("doi") and (not i.get("year") or publishers and "publisher" not in i)]
289
+ if not missing:
290
+ return
291
+
292
+ def run() -> None:
293
+ from ..search.deep import _get
294
+ from ..sources.scihub import note_years
295
+ for k in range(0, len(missing), 50):
296
+ chunk = missing[k:k + 50]
297
+ try:
298
+ data = _get({"filter": "doi:" + "|".join(chunk), "select": "doi,publication_year,primary_location",
299
+ "per_page": 50}, tries=1)
300
+ except Exception: # no network, no budget: the papers just stay without a year
301
+ return
302
+ works = {canonical_doi(w.get("doi")): w for w in data.get("results", []) if w.get("doi")}
303
+ years = {d: int(w["publication_year"]) for d, w in works.items() if w.get("publication_year")}
304
+ if years:
305
+ note_years(years)
306
+ if not works:
307
+ continue
308
+ with store.update(path, indent=1) as data_file:
309
+ for it in data_file["items"]:
310
+ w = works.get(it.get("doi"))
311
+ if w is None:
312
+ continue
313
+ if not it.get("year") and it["doi"] in years:
314
+ it["year"] = years[it["doi"]]
315
+ if publishers and "publisher" not in it:
316
+ source = (w.get("primary_location") or {}).get("source") or {}
317
+ it["publisher"] = publisher_name(source.get("host_organization_name")) or None
318
+
319
+ t = threading.Thread(target=run, name="job-years", daemon=True)
320
+ t.start()
321
+ t.join(wait_s)
322
+
323
+
324
+ def _alias_map(dois: list[str]) -> dict[str, str]:
325
+ """Every key a paper answers to (an arXiv id or its DOI) -> the item's DOI.
326
+
327
+ The engine reports progress under whatever identifier it was handed, which
328
+ for a pasted arXiv id is the bare number while the job item is stored under
329
+ ``10.48550/arxiv.<id>``; without the mapping those rows never update.
330
+ """
331
+ aliases: dict[str, str] = {}
332
+ for d in dois:
333
+ for key in same_paper(d):
334
+ aliases.setdefault(key, d)
335
+ return aliases
336
+
337
+
338
+ def _run(job: str, only: list[str] | None = None, rounds: str = "all") -> None:
339
+ """Download the job's papers not yet done (``only``: these), with the
340
+ engine's ``rounds`` (``server._batch_download_impl``)."""
341
+ path = job_path(job)
342
+ dois = only or [i["doi"] for i in store.read_json(path)["items"] if i["status"] != "done"] # not the ones already here
343
+ wanted = _alias_map(dois)
344
+
345
+ def item_of(current: str) -> str | None:
346
+ return wanted.get(canonical_doi(current or ""))
347
+
348
+ def on_item(ok: bool, current: str) -> None:
349
+ # A lane that fails hands the paper to the next one (grey, institutional,
350
+ # a cooled-down retry), so a failure here is not final: the engine's
351
+ # results at the end say what failed. Until then the paper is retrying.
352
+ doi = item_of(current)
353
+ if doi:
354
+ if ok:
355
+ _mark(job, doi, status="done", retrying=False, stage=None)
356
+ else:
357
+ _mark(job, doi, retrying=True)
358
+
359
+ def on_stage(current: str, text: str) -> None:
360
+ doi = item_of(current)
361
+ if doi:
362
+ _mark(job, doi, stage=text)
363
+
364
+ def on_result(current: str, r: dict) -> None:
365
+ # Final for this paper: show it now, as the end of the job would.
366
+ doi = item_of(current)
367
+ if doi:
368
+ ok = bool(r.get("success"))
369
+ from . import error_text
370
+ _mark(job, doi, status="done" if ok else "failed", retrying=False, stage=None,
371
+ source=r.get("source"), file=r.get("file") or None, error=error_text.pick_failure(r),
372
+ note=r.get("note"), attempts=error_text.steps_for_app(r.get("attempts")))
373
+
374
+ from ..download import progress as progress_reporter
375
+ from ..sources import _SKIPPED
376
+ from ..sources.scihub import note_years
377
+ _SKIPPED.clear() # skips asked for in an earlier job do not carry over
378
+ # The publication year decides whether Sci-Hub is asked (scihub_skip_from_year).
379
+ note_years({i["doi"]: i.get("year") for i in store.read_json(path)["items"] if i.get("year")})
380
+ _fill_meta(path, publishers=rounds == "free")
381
+ progress_reporter.add_listener(on_item)
382
+ progress_reporter.add_stage_listener(on_stage)
383
+ progress_reporter.add_result_listener(on_result)
384
+ state, error, results = "done", None, []
385
+ try:
386
+ if dois:
387
+ from ..server import _batch_download_impl
388
+ out = json.loads(_batch_download_impl(dois, batch_id=f"app-{job}", keep_attempts=True, # the app shows them
389
+ rounds=rounds))
390
+ results = out.get("results", [])
391
+ except Exception as exc: # the job file says what went wrong; the app shows it
392
+ state, error = "failed", f"{type(exc).__name__}: {exc}"
393
+ finally:
394
+ progress_reporter.remove_listener(on_item)
395
+ progress_reporter.remove_stage_listener(on_stage)
396
+ progress_reporter.remove_result_listener(on_result)
397
+ try:
398
+ with store.update(path, indent=1) as data:
399
+ by_doi: dict[str, dict[str, Any]] = {}
400
+ for r in results: # an arXiv id and its DOI are the same paper
401
+ for key in same_paper(r.get("doi") or r.get("identifier") or ""):
402
+ by_doi.setdefault(key, r)
403
+ for it in data["items"]:
404
+ r = by_doi.get(it["doi"])
405
+ if r and r.get("queued"):
406
+ it.update(status="queued", retrying=False, stage=None, error=None)
407
+ elif r:
408
+ from . import error_text
409
+ note = r.get("note") or it.get("note")
410
+ if r.get("preprint"):
411
+ # The file is a preprint of the paper, not the published
412
+ # PDF: say so next to a green "done" (2026-11-03).
413
+ note = note or f"{r['preprint']} 预印本,不是正式发表版"
414
+ it.update(status="done" if r.get("success") else "failed", retrying=False, stage=None, source=r.get("source"),
415
+ file=r.get("file") or None,
416
+ error=error_text.pick_failure(r) or (None if r.get("success") else it.get("error")),
417
+ note=note,
418
+ attempts=error_text.steps_for_app(r.get("attempts")) or it.get("attempts"))
419
+ elif it["status"] == "waiting" and state == "done":
420
+ if rounds == "single" and _CURRENT["cancelled"]:
421
+ it.update(status="queued", retrying=False, stage=None) # back in its queue
422
+ else:
423
+ it.update(status="failed", error="cancelled" if _CURRENT["cancelled"] else "no result")
424
+ data["state"] = ("cancelled" if _CURRENT["cancelled"] and state == "done"
425
+ else "queued" if state == "done" and any(i["status"] == "queued" for i in data["items"])
426
+ else state)
427
+ data["error"] = error
428
+ data["finished"] = time.strftime("%Y-%m-%d %H:%M:%S")
429
+ finally:
430
+ _CURRENT.update(job=None, cancelled=False)
431
+ _RUN_LOCK.release()
432
+ from ..config import load_config
433
+ if load_config().get("zotero_auto_push"):
434
+ try:
435
+ push_zotero(job)
436
+ except (Busy, ValueError):
437
+ pass # another push runs, or nothing was downloaded: the user can push later
438
+
439
+
440
+ def default_collection(title: str) -> str:
441
+ """<zotero_collection_root>/<title>, shortened; '/' would start a sub-collection."""
442
+ from ..config import load_config
443
+ root = str(load_config().get("zotero_collection_root") or "LitLoop").strip().strip("/")
444
+ name = " ".join(title.replace("/", "-").split())
445
+ name = name if len(name) <= 40 else name[:39] + "…"
446
+ return f"{root}/{name}" if root else name
447
+
448
+
449
+ def pushing() -> str | None:
450
+ return _PUSHING["job"]
451
+
452
+
453
+ def find_local_pdf(doi: str) -> Path | None:
454
+ """A PDF of this DOI on this computer: one a job downloaded, else a file in the
455
+ download folder whose name carries the DOI (the default file-name format)."""
456
+ from ..config import load_config
457
+ from ..identifiers import safe_filename
458
+ doi = canonical_doi(doi or "")
459
+ if not doi:
460
+ return None
461
+ for d in _jobs_full():
462
+ for it in d["items"]:
463
+ if it["doi"] == doi and it.get("file") and Path(it["file"]).is_file():
464
+ return Path(it["file"])
465
+ out = Path(str(load_config().get("output_dir") or ""))
466
+ tag = safe_filename(doi)
467
+ if tag and out.is_dir():
468
+ for p in out.glob("*.pdf"):
469
+ if tag in p.name.lower():
470
+ return p
471
+ return None
472
+
473
+
474
+ def _jobs_full(limit: int = 200) -> list[dict[str, Any]]:
475
+ out = []
476
+ for j in list_jobs(limit):
477
+ try:
478
+ out.append(get(j["job"]))
479
+ except (OSError, ValueError):
480
+ continue
481
+ return out
482
+
483
+
484
+ def zotero_pending() -> list[dict[str, Any]]:
485
+ """Downloaded papers not in Zotero yet (never pushed, PDF not attached, or failed), by job."""
486
+ out = []
487
+ for d in _jobs_full():
488
+ papers = [{k: i.get(k) for k in ("doi", "title", "year", "journal")} | {"zotero": (i.get("zotero") or {}).get("status")}
489
+ for i in d["items"] if i["status"] == "done" and i.get("file")
490
+ and (i.get("zotero") or {}).get("status") not in ("added", "reused")]
491
+ if papers:
492
+ out.append({"job": d["job"], "title": d["title"], "created": d["created"], "papers": papers,
493
+ "collection": (d.get("zotero") or {}).get("collection") or d.get("zotero_target")
494
+ or default_collection(d["title"])})
495
+ return out
496
+
497
+
498
+ def zotero_history() -> list[dict[str, Any]]:
499
+ """Jobs that were pushed, newest first, with counts per outcome."""
500
+ out = []
501
+ for d in _jobs_full():
502
+ z = d.get("zotero")
503
+ if not z:
504
+ continue
505
+ counts = {k: sum(1 for i in d["items"] if (i.get("zotero") or {}).get("status") == k)
506
+ for k in ("added", "reused", "no_pdf", "failed")}
507
+ out.append({"job": d["job"], "title": d["title"], **{k: z.get(k) for k in ("state", "collection", "started", "finished", "error")}, **counts})
508
+ return out
509
+
510
+
511
+ def push_zotero(job: str, collection: str | None = None, tags: list[str] | None = None) -> dict[str, Any]:
512
+ """Push the job's downloaded papers to Zotero in the background (those not pushed yet)."""
513
+ data = get(job)
514
+ todo = [i for i in data["items"] if i["status"] == "done" and i.get("file")
515
+ and (i.get("zotero") or {}).get("status") not in ("added", "reused")]
516
+ if not todo:
517
+ raise ValueError("无法推送:没有待推送的文献")
518
+ collection = (collection or "").strip().strip("/") or data.get("zotero_target") or default_collection(data["title"])
519
+ if not _PUSH_LOCK.acquire(blocking=False):
520
+ raise Busy("another Zotero push is still running")
521
+ try:
522
+ with store.update(job_path(job), indent=1) as d:
523
+ d["zotero"] = {"state": "running", "collection": collection, "key": None,
524
+ "started": time.strftime("%Y-%m-%d %H:%M:%S"), "finished": None, "error": None}
525
+ _PUSHING["job"] = job
526
+ threading.Thread(target=_push, args=(job, collection, list(tags or [])),
527
+ name=f"zotero-{job}", daemon=True).start()
528
+ return get(job)
529
+ except BaseException:
530
+ _PUSHING["job"] = None
531
+ _PUSH_LOCK.release()
532
+ raise
533
+
534
+
535
+ def _set_zotero(job: str, **fields: Any) -> None:
536
+ with store.update(job_path(job), indent=1) as d:
537
+ d["zotero"] = {**(d.get("zotero") or {}), **fields}
538
+
539
+
540
+ def _push(job: str, collection: str, tags: list[str]) -> None:
541
+ from ..config import load_config
542
+ from ..zotero.client import ensure_collection, push_via_zotero_mcp
543
+ cfg = load_config()
544
+ try:
545
+ found = ensure_collection(collection, cfg)
546
+ if not found.get("success"):
547
+ _set_zotero(job, state="failed", error=found.get("error") or "cannot create the collection")
548
+ return
549
+ _set_zotero(job, key=found["key"])
550
+ for it in get(job)["items"]:
551
+ if it["status"] != "done" or not it.get("file") or (it.get("zotero") or {}).get("status") in ("added", "reused"):
552
+ continue
553
+ _mark(job, it["doi"], zotero={"status": "pushing", "key": None, "error": None})
554
+ try:
555
+ r = push_via_zotero_mcp(it["doi"], Path(it["file"]), cfg, collections=[found["key"]], tags=tags)
556
+ except Exception as exc:
557
+ r = {"success": False, "error": f"{type(exc).__name__}: {exc}"}
558
+ if not r.get("success"):
559
+ z = {"status": "failed", "key": None, "error": r.get("error")}
560
+ elif not r.get("pdf_attached"):
561
+ z = {"status": "no_pdf", "key": r.get("zotero_key"), "error": r.get("attach_message")}
562
+ else:
563
+ z = {"status": "reused" if r.get("reused") else "added", "key": r.get("zotero_key"), "error": None}
564
+ _mark(job, it["doi"], zotero=z)
565
+ _set_zotero(job, state="done", finished=time.strftime("%Y-%m-%d %H:%M:%S"))
566
+ except Exception as exc:
567
+ _set_zotero(job, state="failed", error=f"{type(exc).__name__}: {exc}",
568
+ finished=time.strftime("%Y-%m-%d %H:%M:%S"))
569
+ finally:
570
+ _PUSHING["job"] = None
571
+ _PUSH_LOCK.release()