litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,571 @@
|
|
|
1
|
+
"""Download jobs started from the desktop app.
|
|
2
|
+
|
|
3
|
+
A job is a list of papers downloaded by the normal batch engine
|
|
4
|
+
(``server._batch_download_impl``: lanes, racing, institutional cascade) in a
|
|
5
|
+
background thread of the app backend. Its state lives in one file,
|
|
6
|
+
``<litloop home>/downloads/<job>.json``, written through ``store`` (locked,
|
|
7
|
+
replaced in one step) so the app can poll it while the job runs::
|
|
8
|
+
|
|
9
|
+
{"schema": 1, "job": "20261017-101500", "title": "...", "session": "...",
|
|
10
|
+
"state": "running" | "queued" | "done" | "cancelled" | "failed" | "interrupted",
|
|
11
|
+
"created": "...", "finished": "...", "output_dir": "...", "error": "...",
|
|
12
|
+
"items": [{"doi", "title", "year", "journal", "publisher",
|
|
13
|
+
"status": "waiting" | "queued" | "done" | "failed",
|
|
14
|
+
"source", "file", "error",
|
|
15
|
+
"zotero": {"status": "pushing" | "added" | "reused" | "no_pdf" | "failed",
|
|
16
|
+
"key", "error"}}],
|
|
17
|
+
"zotero": {"state": "running" | "done" | "failed", "collection", "key",
|
|
18
|
+
"started", "finished", "error"}}
|
|
19
|
+
|
|
20
|
+
Papers are marked done/failed one by one as the engine reports them
|
|
21
|
+
(``progress_reporter.advance``); source, file and error are filled in when
|
|
22
|
+
the batch returns. One job runs at a time: the engine's progress report and
|
|
23
|
+
cancel switch are process-wide.
|
|
24
|
+
|
|
25
|
+
A long list (more than ``batch_small_max`` papers) runs only the rounds that
|
|
26
|
+
need nobody, open access and Sci-Hub; the rest is ``queued`` and waits in one
|
|
27
|
+
queue per publisher (``queues``), which the user starts when they can log in
|
|
28
|
+
or pass a check (``start_queue``): its papers then go one by one through the
|
|
29
|
+
publisher's site and the school's routes, so one publisher's logins and checks
|
|
30
|
+
come together. The job is ``queued`` while any paper waits in a queue.
|
|
31
|
+
|
|
32
|
+
Downloaded papers are pushed to Zotero (zotero.push_via_zotero_mcp) by hand
|
|
33
|
+
from the job page, or right after the download when ``zotero_auto_push`` is
|
|
34
|
+
on, into ``<zotero_collection_root>/<job title>`` unless another collection
|
|
35
|
+
is chosen. A push runs in its own thread, one at a time, and can overlap a
|
|
36
|
+
download.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import json
|
|
42
|
+
import re
|
|
43
|
+
import threading
|
|
44
|
+
import time
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any
|
|
47
|
+
|
|
48
|
+
from .. import store
|
|
49
|
+
from ..identifiers import canonical_doi
|
|
50
|
+
|
|
51
|
+
_RUN_LOCK = threading.Lock()
|
|
52
|
+
_CURRENT: dict[str, Any] = {"job": None, "cancelled": False}
|
|
53
|
+
_PUSH_LOCK = threading.Lock()
|
|
54
|
+
_PUSHING: dict[str, Any] = {"job": None}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Busy(RuntimeError):
|
|
58
|
+
"""Another download job is still running."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def jobs_dir() -> Path:
|
|
62
|
+
from ..config import DATA_DIR
|
|
63
|
+
d = Path(DATA_DIR) / "downloads"
|
|
64
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
return d
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def job_path(job: str) -> Path:
|
|
69
|
+
return jobs_dir() / f"{job}.json"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _settled(d: dict[str, Any]) -> dict[str, Any]:
|
|
73
|
+
"""A job still marked running that this process is not running was cut off (app closed)."""
|
|
74
|
+
if d.get("state") == "running" and d.get("job") != _CURRENT["job"]:
|
|
75
|
+
d["state"] = "interrupted"
|
|
76
|
+
z = d.get("zotero")
|
|
77
|
+
if z and z.get("state") == "running" and d.get("job") != _PUSHING["job"]:
|
|
78
|
+
z["state"] = "interrupted"
|
|
79
|
+
for it in d.get("items", []):
|
|
80
|
+
if (it.get("zotero") or {}).get("status") == "pushing":
|
|
81
|
+
it["zotero"]["status"] = "failed"
|
|
82
|
+
return d
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
OTHER_PUBLISHER = "其他出版商"
|
|
86
|
+
_CORPORATE = re.compile(r"[\s,]+(?:\([A-Z]{2,}\)|B\.?V\.?|Ltd\.?|Limited|Inc\.?|LLC|GmbH|plc|AG)$", re.I)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def publisher_name(raw: str | None) -> str:
|
|
90
|
+
"""OpenAlex's host organization without its company form ("Elsevier BV" -> "Elsevier")."""
|
|
91
|
+
name, before = (raw or "").strip(), None
|
|
92
|
+
while name != before:
|
|
93
|
+
before, name = name, _CORPORATE.sub("", name).strip()
|
|
94
|
+
return name
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def queues(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
98
|
+
"""The papers waiting in a queue, one queue per publisher, the largest first."""
|
|
99
|
+
count: dict[str, int] = {}
|
|
100
|
+
for it in items:
|
|
101
|
+
if it.get("status") == "queued":
|
|
102
|
+
name = it.get("publisher") or OTHER_PUBLISHER
|
|
103
|
+
count[name] = count.get(name, 0) + 1
|
|
104
|
+
return [{"publisher": k, "count": n}
|
|
105
|
+
for k, n in sorted(count.items(), key=lambda kv: (kv[0] == OTHER_PUBLISHER, -kv[1], kv[0]))]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def list_jobs(limit: int = 50) -> list[dict[str, Any]]:
|
|
109
|
+
"""Newest first, without the item lists (counts instead)."""
|
|
110
|
+
out = []
|
|
111
|
+
for p in sorted(jobs_dir().glob("*.json"), reverse=True)[:limit]:
|
|
112
|
+
try:
|
|
113
|
+
d = _settled(store.read_json(p))
|
|
114
|
+
except (OSError, ValueError):
|
|
115
|
+
continue
|
|
116
|
+
items = d.get("items", [])
|
|
117
|
+
out.append({k: d.get(k) for k in ("job", "title", "session", "state", "created", "finished")}
|
|
118
|
+
| {"total": len(items), "done": sum(1 for i in items if i["status"] == "done"),
|
|
119
|
+
"failed": sum(1 for i in items if i["status"] == "failed"),
|
|
120
|
+
"queued": sum(1 for i in items if i["status"] == "queued")})
|
|
121
|
+
return out
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def get(job: str) -> dict[str, Any]:
|
|
125
|
+
d = _settled(store.read_json(job_path(job)))
|
|
126
|
+
d["queues"] = queues(d.get("items", []))
|
|
127
|
+
return d
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def running() -> str | None:
|
|
131
|
+
return _CURRENT["job"]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
LOCAL = "本机已有" # the source of a paper found in the save folder, not downloaded again
|
|
135
|
+
_ARXIV_DOI = "10.48550/arxiv."
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def same_paper(doi: str) -> set[str]:
|
|
139
|
+
"""The keys one paper can go by: an arXiv id and its arXiv DOI are the same paper."""
|
|
140
|
+
d = canonical_doi(doi or "")
|
|
141
|
+
if d.startswith(_ARXIV_DOI):
|
|
142
|
+
return {d, d[len(_ARXIV_DOI):]}
|
|
143
|
+
if d and not d.startswith("10."): # a bare arXiv id
|
|
144
|
+
return {d, _ARXIV_DOI + d}
|
|
145
|
+
return {d} if d else set()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def local_files(dois: list[str] | set[str] | None = None) -> dict[str, str]:
|
|
149
|
+
"""DOI -> PDF already in the save folder, from its DOI index (.doi_index.json).
|
|
150
|
+
|
|
151
|
+
Only files that are still there count; age does not matter (a PDF on disk
|
|
152
|
+
does not expire, whatever cache_ttl_hours says about re-checking sources).
|
|
153
|
+
"""
|
|
154
|
+
from ..config import load_config
|
|
155
|
+
folder = Path(str(load_config().get("output_dir") or "")).expanduser()
|
|
156
|
+
try:
|
|
157
|
+
idx = json.loads((folder / ".doi_index.json").read_text(encoding="utf-8"))
|
|
158
|
+
except (OSError, ValueError):
|
|
159
|
+
return {}
|
|
160
|
+
found = {}
|
|
161
|
+
for key, entry in idx.items():
|
|
162
|
+
f = entry.get("file") if isinstance(entry, dict) else entry
|
|
163
|
+
if f and Path(f).is_file():
|
|
164
|
+
for k in same_paper(key):
|
|
165
|
+
found[k] = str(f)
|
|
166
|
+
if dois is None:
|
|
167
|
+
return found
|
|
168
|
+
return {canonical_doi(d): found[k] for d in dois for k in same_paper(d) if k in found}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def downloaded_dois() -> set[str]:
|
|
172
|
+
"""DOIs some job has already got, or whose PDF is already in the save folder:
|
|
173
|
+
they are not offered again."""
|
|
174
|
+
got = {k for d in local_files() for k in same_paper(d)}
|
|
175
|
+
for p in jobs_dir().glob("*.json"):
|
|
176
|
+
try:
|
|
177
|
+
got |= {i["doi"] for i in store.read_json(p).get("items", []) if i.get("status") == "done"}
|
|
178
|
+
except (OSError, ValueError):
|
|
179
|
+
continue
|
|
180
|
+
return got
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def queued_dois() -> set[str]:
|
|
184
|
+
"""DOIs waiting in a long list's publisher queue: not offered again either."""
|
|
185
|
+
out: set[str] = set()
|
|
186
|
+
for p in jobs_dir().glob("*.json"):
|
|
187
|
+
try:
|
|
188
|
+
out |= {i["doi"] for i in store.read_json(p).get("items", []) if i.get("status") == "queued"}
|
|
189
|
+
except (OSError, ValueError):
|
|
190
|
+
continue
|
|
191
|
+
return out
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def start(papers: list[dict[str, Any]], title: str, session: str | None = None,
|
|
195
|
+
zotero_target: str | None = None) -> dict[str, Any]:
|
|
196
|
+
"""Create a job for papers ({doi, title, year, journal}) and start it in the background.
|
|
197
|
+
|
|
198
|
+
zotero_target: the collection a later push goes to by default (the attachment
|
|
199
|
+
check sets it to the collection being repaired).
|
|
200
|
+
"""
|
|
201
|
+
items, seen = [], set()
|
|
202
|
+
for p in papers:
|
|
203
|
+
doi = canonical_doi(p.get("doi") or "")
|
|
204
|
+
if not doi or doi in seen:
|
|
205
|
+
continue
|
|
206
|
+
seen.add(doi)
|
|
207
|
+
items.append({"doi": doi, "title": p.get("title") or doi, "year": p.get("year"),
|
|
208
|
+
"journal": p.get("journal"), "status": "waiting", "source": None, "file": None, "error": None})
|
|
209
|
+
if not items:
|
|
210
|
+
raise ValueError("无法下载:没有带 DOI 的文献")
|
|
211
|
+
# Already in the save folder: done at once, the engine is not asked again.
|
|
212
|
+
for it, f in ((it, local_files([it["doi"]]).get(it["doi"])) for it in items):
|
|
213
|
+
if f:
|
|
214
|
+
it.update(status="done", source=LOCAL, file=f)
|
|
215
|
+
if not _RUN_LOCK.acquire(blocking=False):
|
|
216
|
+
raise Busy("another download is still running")
|
|
217
|
+
try:
|
|
218
|
+
from ..config import load_config
|
|
219
|
+
job = time.strftime("%Y%m%d-%H%M%S")
|
|
220
|
+
while job_path(job).exists():
|
|
221
|
+
time.sleep(1)
|
|
222
|
+
job = time.strftime("%Y%m%d-%H%M%S")
|
|
223
|
+
data = {"schema": store.SCHEMA, "job": job, "title": title, "session": session, "state": "running",
|
|
224
|
+
"created": time.strftime("%Y-%m-%d %H:%M:%S"), "finished": None,
|
|
225
|
+
"output_dir": str(load_config().get("output_dir") or ""), "error": None, "items": items,
|
|
226
|
+
"zotero_target": (zotero_target or "").strip().strip("/") or None}
|
|
227
|
+
store.write_json(job_path(job), data, indent=1)
|
|
228
|
+
_CURRENT.update(job=job, cancelled=False)
|
|
229
|
+
todo = sum(1 for it in items if it["status"] != "done")
|
|
230
|
+
rounds = "free" if todo > int(load_config().get("batch_small_max", 30) or 0) else "all"
|
|
231
|
+
threading.Thread(target=_run, args=(job,), kwargs={"rounds": rounds}, name=f"download-{job}", daemon=True).start()
|
|
232
|
+
return data
|
|
233
|
+
except BaseException:
|
|
234
|
+
_CURRENT.update(job=None)
|
|
235
|
+
_RUN_LOCK.release()
|
|
236
|
+
raise
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def start_queue(job: str, publisher: str) -> dict[str, Any]:
|
|
240
|
+
"""Download the papers of one publisher's queue (see the module doc)."""
|
|
241
|
+
path = job_path(job)
|
|
242
|
+
dois = [it["doi"] for it in get(job)["items"]
|
|
243
|
+
if it["status"] == "queued" and (it.get("publisher") or OTHER_PUBLISHER) == publisher]
|
|
244
|
+
if not dois:
|
|
245
|
+
raise ValueError("无法开始:此队列里没有待下载的文献")
|
|
246
|
+
if not _RUN_LOCK.acquire(blocking=False):
|
|
247
|
+
raise Busy("another download is still running")
|
|
248
|
+
try:
|
|
249
|
+
with store.update(path, indent=1) as data:
|
|
250
|
+
for it in data["items"]:
|
|
251
|
+
if it["doi"] in dois:
|
|
252
|
+
it.update(status="waiting", error=None)
|
|
253
|
+
data.update(state="running", finished=None, error=None)
|
|
254
|
+
_CURRENT.update(job=job, cancelled=False)
|
|
255
|
+
threading.Thread(target=_run, args=(job,), kwargs={"only": dois, "rounds": "single"},
|
|
256
|
+
name=f"download-{job}", daemon=True).start()
|
|
257
|
+
return get(job)
|
|
258
|
+
except BaseException:
|
|
259
|
+
_CURRENT.update(job=None)
|
|
260
|
+
_RUN_LOCK.release()
|
|
261
|
+
raise
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def cancel() -> bool:
|
|
265
|
+
"""Ask the running job to stop: pending papers are skipped, browser windows closed."""
|
|
266
|
+
if not _CURRENT["job"]:
|
|
267
|
+
return False
|
|
268
|
+
from ..sources import cancel_downloads
|
|
269
|
+
_CURRENT["cancelled"] = True
|
|
270
|
+
cancel_downloads()
|
|
271
|
+
return True
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _mark(job: str, doi: str, **fields: Any) -> None:
|
|
275
|
+
with store.update(job_path(job), indent=1) as data:
|
|
276
|
+
for it in data["items"]:
|
|
277
|
+
if it["doi"] == doi:
|
|
278
|
+
it.update(fields)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _fill_meta(path: Path, wait_s: float = 10.0, publishers: bool = False) -> None:
|
|
282
|
+
"""Papers without a publication year (a pasted list has none) get it from
|
|
283
|
+
OpenAlex, so the Sci-Hub year rule (scihub_skip_from_year) applies to them
|
|
284
|
+
too; the years also go into the job file. With ``publishers`` (a long
|
|
285
|
+
list) every paper also gets its publisher, which names its queue. The job waits at most
|
|
286
|
+
``wait_s``; years that come later still reach the papers not yet started."""
|
|
287
|
+
missing = [i["doi"] for i in store.read_json(path)["items"]
|
|
288
|
+
if i.get("doi") and (not i.get("year") or publishers and "publisher" not in i)]
|
|
289
|
+
if not missing:
|
|
290
|
+
return
|
|
291
|
+
|
|
292
|
+
def run() -> None:
|
|
293
|
+
from ..search.deep import _get
|
|
294
|
+
from ..sources.scihub import note_years
|
|
295
|
+
for k in range(0, len(missing), 50):
|
|
296
|
+
chunk = missing[k:k + 50]
|
|
297
|
+
try:
|
|
298
|
+
data = _get({"filter": "doi:" + "|".join(chunk), "select": "doi,publication_year,primary_location",
|
|
299
|
+
"per_page": 50}, tries=1)
|
|
300
|
+
except Exception: # no network, no budget: the papers just stay without a year
|
|
301
|
+
return
|
|
302
|
+
works = {canonical_doi(w.get("doi")): w for w in data.get("results", []) if w.get("doi")}
|
|
303
|
+
years = {d: int(w["publication_year"]) for d, w in works.items() if w.get("publication_year")}
|
|
304
|
+
if years:
|
|
305
|
+
note_years(years)
|
|
306
|
+
if not works:
|
|
307
|
+
continue
|
|
308
|
+
with store.update(path, indent=1) as data_file:
|
|
309
|
+
for it in data_file["items"]:
|
|
310
|
+
w = works.get(it.get("doi"))
|
|
311
|
+
if w is None:
|
|
312
|
+
continue
|
|
313
|
+
if not it.get("year") and it["doi"] in years:
|
|
314
|
+
it["year"] = years[it["doi"]]
|
|
315
|
+
if publishers and "publisher" not in it:
|
|
316
|
+
source = (w.get("primary_location") or {}).get("source") or {}
|
|
317
|
+
it["publisher"] = publisher_name(source.get("host_organization_name")) or None
|
|
318
|
+
|
|
319
|
+
t = threading.Thread(target=run, name="job-years", daemon=True)
|
|
320
|
+
t.start()
|
|
321
|
+
t.join(wait_s)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _alias_map(dois: list[str]) -> dict[str, str]:
|
|
325
|
+
"""Every key a paper answers to (an arXiv id or its DOI) -> the item's DOI.
|
|
326
|
+
|
|
327
|
+
The engine reports progress under whatever identifier it was handed, which
|
|
328
|
+
for a pasted arXiv id is the bare number while the job item is stored under
|
|
329
|
+
``10.48550/arxiv.<id>``; without the mapping those rows never update.
|
|
330
|
+
"""
|
|
331
|
+
aliases: dict[str, str] = {}
|
|
332
|
+
for d in dois:
|
|
333
|
+
for key in same_paper(d):
|
|
334
|
+
aliases.setdefault(key, d)
|
|
335
|
+
return aliases
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _run(job: str, only: list[str] | None = None, rounds: str = "all") -> None:
|
|
339
|
+
"""Download the job's papers not yet done (``only``: these), with the
|
|
340
|
+
engine's ``rounds`` (``server._batch_download_impl``)."""
|
|
341
|
+
path = job_path(job)
|
|
342
|
+
dois = only or [i["doi"] for i in store.read_json(path)["items"] if i["status"] != "done"] # not the ones already here
|
|
343
|
+
wanted = _alias_map(dois)
|
|
344
|
+
|
|
345
|
+
def item_of(current: str) -> str | None:
|
|
346
|
+
return wanted.get(canonical_doi(current or ""))
|
|
347
|
+
|
|
348
|
+
def on_item(ok: bool, current: str) -> None:
|
|
349
|
+
# A lane that fails hands the paper to the next one (grey, institutional,
|
|
350
|
+
# a cooled-down retry), so a failure here is not final: the engine's
|
|
351
|
+
# results at the end say what failed. Until then the paper is retrying.
|
|
352
|
+
doi = item_of(current)
|
|
353
|
+
if doi:
|
|
354
|
+
if ok:
|
|
355
|
+
_mark(job, doi, status="done", retrying=False, stage=None)
|
|
356
|
+
else:
|
|
357
|
+
_mark(job, doi, retrying=True)
|
|
358
|
+
|
|
359
|
+
def on_stage(current: str, text: str) -> None:
|
|
360
|
+
doi = item_of(current)
|
|
361
|
+
if doi:
|
|
362
|
+
_mark(job, doi, stage=text)
|
|
363
|
+
|
|
364
|
+
def on_result(current: str, r: dict) -> None:
|
|
365
|
+
# Final for this paper: show it now, as the end of the job would.
|
|
366
|
+
doi = item_of(current)
|
|
367
|
+
if doi:
|
|
368
|
+
ok = bool(r.get("success"))
|
|
369
|
+
from . import error_text
|
|
370
|
+
_mark(job, doi, status="done" if ok else "failed", retrying=False, stage=None,
|
|
371
|
+
source=r.get("source"), file=r.get("file") or None, error=error_text.pick_failure(r),
|
|
372
|
+
note=r.get("note"), attempts=error_text.steps_for_app(r.get("attempts")))
|
|
373
|
+
|
|
374
|
+
from ..download import progress as progress_reporter
|
|
375
|
+
from ..sources import _SKIPPED
|
|
376
|
+
from ..sources.scihub import note_years
|
|
377
|
+
_SKIPPED.clear() # skips asked for in an earlier job do not carry over
|
|
378
|
+
# The publication year decides whether Sci-Hub is asked (scihub_skip_from_year).
|
|
379
|
+
note_years({i["doi"]: i.get("year") for i in store.read_json(path)["items"] if i.get("year")})
|
|
380
|
+
_fill_meta(path, publishers=rounds == "free")
|
|
381
|
+
progress_reporter.add_listener(on_item)
|
|
382
|
+
progress_reporter.add_stage_listener(on_stage)
|
|
383
|
+
progress_reporter.add_result_listener(on_result)
|
|
384
|
+
state, error, results = "done", None, []
|
|
385
|
+
try:
|
|
386
|
+
if dois:
|
|
387
|
+
from ..server import _batch_download_impl
|
|
388
|
+
out = json.loads(_batch_download_impl(dois, batch_id=f"app-{job}", keep_attempts=True, # the app shows them
|
|
389
|
+
rounds=rounds))
|
|
390
|
+
results = out.get("results", [])
|
|
391
|
+
except Exception as exc: # the job file says what went wrong; the app shows it
|
|
392
|
+
state, error = "failed", f"{type(exc).__name__}: {exc}"
|
|
393
|
+
finally:
|
|
394
|
+
progress_reporter.remove_listener(on_item)
|
|
395
|
+
progress_reporter.remove_stage_listener(on_stage)
|
|
396
|
+
progress_reporter.remove_result_listener(on_result)
|
|
397
|
+
try:
|
|
398
|
+
with store.update(path, indent=1) as data:
|
|
399
|
+
by_doi: dict[str, dict[str, Any]] = {}
|
|
400
|
+
for r in results: # an arXiv id and its DOI are the same paper
|
|
401
|
+
for key in same_paper(r.get("doi") or r.get("identifier") or ""):
|
|
402
|
+
by_doi.setdefault(key, r)
|
|
403
|
+
for it in data["items"]:
|
|
404
|
+
r = by_doi.get(it["doi"])
|
|
405
|
+
if r and r.get("queued"):
|
|
406
|
+
it.update(status="queued", retrying=False, stage=None, error=None)
|
|
407
|
+
elif r:
|
|
408
|
+
from . import error_text
|
|
409
|
+
note = r.get("note") or it.get("note")
|
|
410
|
+
if r.get("preprint"):
|
|
411
|
+
# The file is a preprint of the paper, not the published
|
|
412
|
+
# PDF: say so next to a green "done" (2026-11-03).
|
|
413
|
+
note = note or f"{r['preprint']} 预印本,不是正式发表版"
|
|
414
|
+
it.update(status="done" if r.get("success") else "failed", retrying=False, stage=None, source=r.get("source"),
|
|
415
|
+
file=r.get("file") or None,
|
|
416
|
+
error=error_text.pick_failure(r) or (None if r.get("success") else it.get("error")),
|
|
417
|
+
note=note,
|
|
418
|
+
attempts=error_text.steps_for_app(r.get("attempts")) or it.get("attempts"))
|
|
419
|
+
elif it["status"] == "waiting" and state == "done":
|
|
420
|
+
if rounds == "single" and _CURRENT["cancelled"]:
|
|
421
|
+
it.update(status="queued", retrying=False, stage=None) # back in its queue
|
|
422
|
+
else:
|
|
423
|
+
it.update(status="failed", error="cancelled" if _CURRENT["cancelled"] else "no result")
|
|
424
|
+
data["state"] = ("cancelled" if _CURRENT["cancelled"] and state == "done"
|
|
425
|
+
else "queued" if state == "done" and any(i["status"] == "queued" for i in data["items"])
|
|
426
|
+
else state)
|
|
427
|
+
data["error"] = error
|
|
428
|
+
data["finished"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
429
|
+
finally:
|
|
430
|
+
_CURRENT.update(job=None, cancelled=False)
|
|
431
|
+
_RUN_LOCK.release()
|
|
432
|
+
from ..config import load_config
|
|
433
|
+
if load_config().get("zotero_auto_push"):
|
|
434
|
+
try:
|
|
435
|
+
push_zotero(job)
|
|
436
|
+
except (Busy, ValueError):
|
|
437
|
+
pass # another push runs, or nothing was downloaded: the user can push later
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def default_collection(title: str) -> str:
|
|
441
|
+
"""<zotero_collection_root>/<title>, shortened; '/' would start a sub-collection."""
|
|
442
|
+
from ..config import load_config
|
|
443
|
+
root = str(load_config().get("zotero_collection_root") or "LitLoop").strip().strip("/")
|
|
444
|
+
name = " ".join(title.replace("/", "-").split())
|
|
445
|
+
name = name if len(name) <= 40 else name[:39] + "…"
|
|
446
|
+
return f"{root}/{name}" if root else name
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def pushing() -> str | None:
|
|
450
|
+
return _PUSHING["job"]
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def find_local_pdf(doi: str) -> Path | None:
|
|
454
|
+
"""A PDF of this DOI on this computer: one a job downloaded, else a file in the
|
|
455
|
+
download folder whose name carries the DOI (the default file-name format)."""
|
|
456
|
+
from ..config import load_config
|
|
457
|
+
from ..identifiers import safe_filename
|
|
458
|
+
doi = canonical_doi(doi or "")
|
|
459
|
+
if not doi:
|
|
460
|
+
return None
|
|
461
|
+
for d in _jobs_full():
|
|
462
|
+
for it in d["items"]:
|
|
463
|
+
if it["doi"] == doi and it.get("file") and Path(it["file"]).is_file():
|
|
464
|
+
return Path(it["file"])
|
|
465
|
+
out = Path(str(load_config().get("output_dir") or ""))
|
|
466
|
+
tag = safe_filename(doi)
|
|
467
|
+
if tag and out.is_dir():
|
|
468
|
+
for p in out.glob("*.pdf"):
|
|
469
|
+
if tag in p.name.lower():
|
|
470
|
+
return p
|
|
471
|
+
return None
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _jobs_full(limit: int = 200) -> list[dict[str, Any]]:
|
|
475
|
+
out = []
|
|
476
|
+
for j in list_jobs(limit):
|
|
477
|
+
try:
|
|
478
|
+
out.append(get(j["job"]))
|
|
479
|
+
except (OSError, ValueError):
|
|
480
|
+
continue
|
|
481
|
+
return out
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def zotero_pending() -> list[dict[str, Any]]:
|
|
485
|
+
"""Downloaded papers not in Zotero yet (never pushed, PDF not attached, or failed), by job."""
|
|
486
|
+
out = []
|
|
487
|
+
for d in _jobs_full():
|
|
488
|
+
papers = [{k: i.get(k) for k in ("doi", "title", "year", "journal")} | {"zotero": (i.get("zotero") or {}).get("status")}
|
|
489
|
+
for i in d["items"] if i["status"] == "done" and i.get("file")
|
|
490
|
+
and (i.get("zotero") or {}).get("status") not in ("added", "reused")]
|
|
491
|
+
if papers:
|
|
492
|
+
out.append({"job": d["job"], "title": d["title"], "created": d["created"], "papers": papers,
|
|
493
|
+
"collection": (d.get("zotero") or {}).get("collection") or d.get("zotero_target")
|
|
494
|
+
or default_collection(d["title"])})
|
|
495
|
+
return out
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def zotero_history() -> list[dict[str, Any]]:
|
|
499
|
+
"""Jobs that were pushed, newest first, with counts per outcome."""
|
|
500
|
+
out = []
|
|
501
|
+
for d in _jobs_full():
|
|
502
|
+
z = d.get("zotero")
|
|
503
|
+
if not z:
|
|
504
|
+
continue
|
|
505
|
+
counts = {k: sum(1 for i in d["items"] if (i.get("zotero") or {}).get("status") == k)
|
|
506
|
+
for k in ("added", "reused", "no_pdf", "failed")}
|
|
507
|
+
out.append({"job": d["job"], "title": d["title"], **{k: z.get(k) for k in ("state", "collection", "started", "finished", "error")}, **counts})
|
|
508
|
+
return out
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def push_zotero(job: str, collection: str | None = None, tags: list[str] | None = None) -> dict[str, Any]:
|
|
512
|
+
"""Push the job's downloaded papers to Zotero in the background (those not pushed yet)."""
|
|
513
|
+
data = get(job)
|
|
514
|
+
todo = [i for i in data["items"] if i["status"] == "done" and i.get("file")
|
|
515
|
+
and (i.get("zotero") or {}).get("status") not in ("added", "reused")]
|
|
516
|
+
if not todo:
|
|
517
|
+
raise ValueError("无法推送:没有待推送的文献")
|
|
518
|
+
collection = (collection or "").strip().strip("/") or data.get("zotero_target") or default_collection(data["title"])
|
|
519
|
+
if not _PUSH_LOCK.acquire(blocking=False):
|
|
520
|
+
raise Busy("another Zotero push is still running")
|
|
521
|
+
try:
|
|
522
|
+
with store.update(job_path(job), indent=1) as d:
|
|
523
|
+
d["zotero"] = {"state": "running", "collection": collection, "key": None,
|
|
524
|
+
"started": time.strftime("%Y-%m-%d %H:%M:%S"), "finished": None, "error": None}
|
|
525
|
+
_PUSHING["job"] = job
|
|
526
|
+
threading.Thread(target=_push, args=(job, collection, list(tags or [])),
|
|
527
|
+
name=f"zotero-{job}", daemon=True).start()
|
|
528
|
+
return get(job)
|
|
529
|
+
except BaseException:
|
|
530
|
+
_PUSHING["job"] = None
|
|
531
|
+
_PUSH_LOCK.release()
|
|
532
|
+
raise
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _set_zotero(job: str, **fields: Any) -> None:
|
|
536
|
+
with store.update(job_path(job), indent=1) as d:
|
|
537
|
+
d["zotero"] = {**(d.get("zotero") or {}), **fields}
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _push(job: str, collection: str, tags: list[str]) -> None:
|
|
541
|
+
from ..config import load_config
|
|
542
|
+
from ..zotero.client import ensure_collection, push_via_zotero_mcp
|
|
543
|
+
cfg = load_config()
|
|
544
|
+
try:
|
|
545
|
+
found = ensure_collection(collection, cfg)
|
|
546
|
+
if not found.get("success"):
|
|
547
|
+
_set_zotero(job, state="failed", error=found.get("error") or "cannot create the collection")
|
|
548
|
+
return
|
|
549
|
+
_set_zotero(job, key=found["key"])
|
|
550
|
+
for it in get(job)["items"]:
|
|
551
|
+
if it["status"] != "done" or not it.get("file") or (it.get("zotero") or {}).get("status") in ("added", "reused"):
|
|
552
|
+
continue
|
|
553
|
+
_mark(job, it["doi"], zotero={"status": "pushing", "key": None, "error": None})
|
|
554
|
+
try:
|
|
555
|
+
r = push_via_zotero_mcp(it["doi"], Path(it["file"]), cfg, collections=[found["key"]], tags=tags)
|
|
556
|
+
except Exception as exc:
|
|
557
|
+
r = {"success": False, "error": f"{type(exc).__name__}: {exc}"}
|
|
558
|
+
if not r.get("success"):
|
|
559
|
+
z = {"status": "failed", "key": None, "error": r.get("error")}
|
|
560
|
+
elif not r.get("pdf_attached"):
|
|
561
|
+
z = {"status": "no_pdf", "key": r.get("zotero_key"), "error": r.get("attach_message")}
|
|
562
|
+
else:
|
|
563
|
+
z = {"status": "reused" if r.get("reused") else "added", "key": r.get("zotero_key"), "error": None}
|
|
564
|
+
_mark(job, it["doi"], zotero=z)
|
|
565
|
+
_set_zotero(job, state="done", finished=time.strftime("%Y-%m-%d %H:%M:%S"))
|
|
566
|
+
except Exception as exc:
|
|
567
|
+
_set_zotero(job, state="failed", error=f"{type(exc).__name__}: {exc}",
|
|
568
|
+
finished=time.strftime("%Y-%m-%d %H:%M:%S"))
|
|
569
|
+
finally:
|
|
570
|
+
_PUSHING["job"] = None
|
|
571
|
+
_PUSH_LOCK.release()
|