litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Task progress reporting via a well-known JSON file.
|
|
2
|
+
|
|
3
|
+
Writers (batch / lanes / fetch) report per-paper progress here; the desktop
|
|
4
|
+
app reads it: its own download jobs, and downloads the agent (MCP) or the
|
|
5
|
+
command line run in other processes (``running_elsewhere``). The floating
|
|
6
|
+
progress bar that also watched it was removed on 2026-10-19 (the user's call).
|
|
7
|
+
|
|
8
|
+
File: <data_dir>/progress/current.json — atomically replaced (tmp + rename)
|
|
9
|
+
on every update, so readers never see a torn write.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import tempfile
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from ..store import pid_alive as _pid_alive
|
|
24
|
+
|
|
25
|
+
_LOCK = threading.Lock()
|
|
26
|
+
_STATE: dict[str, Any] = {}
|
|
27
|
+
_STARTED_MONO: float = 0.0
|
|
28
|
+
|
|
29
|
+
STALE_SECONDS = 120 # reader shows "idle" when no update for this long
|
|
30
|
+
|
|
31
|
+
# In-process listeners told about every finished item: (ok, current). The
|
|
32
|
+
# desktop app's download jobs use this to mark papers one by one.
|
|
33
|
+
_LISTENERS: list[Any] = []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def add_listener(fn: Any) -> None:
|
|
37
|
+
_LISTENERS.append(fn)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def remove_listener(fn: Any) -> None:
|
|
41
|
+
try:
|
|
42
|
+
_LISTENERS.remove(fn)
|
|
43
|
+
except ValueError:
|
|
44
|
+
pass
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# Listeners told where one paper is now: (identifier, text). The app shows it
|
|
48
|
+
# under the paper ("已在 Sci-Hub 找到,正在下载"); nothing is written to disk here.
|
|
49
|
+
_STAGE_LISTENERS: list[Any] = []
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def add_stage_listener(fn: Any) -> None:
|
|
53
|
+
_STAGE_LISTENERS.append(fn)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def remove_stage_listener(fn: Any) -> None:
|
|
57
|
+
try:
|
|
58
|
+
_STAGE_LISTENERS.remove(fn)
|
|
59
|
+
except ValueError:
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# Listeners told a paper's final result: (identifier, result dict). Sent when
|
|
64
|
+
# the engine knows nothing more will be tried for it (a short list's single
|
|
65
|
+
# downloads), so the app shows it at once instead of when the job ends.
|
|
66
|
+
_RESULT_LISTENERS: list[Any] = []
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def add_result_listener(fn: Any) -> None:
|
|
70
|
+
_RESULT_LISTENERS.append(fn)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def remove_result_listener(fn: Any) -> None:
|
|
74
|
+
try:
|
|
75
|
+
_RESULT_LISTENERS.remove(fn)
|
|
76
|
+
except ValueError:
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def item_result(identifier: str, result: dict) -> None:
|
|
81
|
+
"""A paper is finished for good (downloaded or given up); never raises."""
|
|
82
|
+
for fn in list(_RESULT_LISTENERS):
|
|
83
|
+
try:
|
|
84
|
+
fn(identifier, result)
|
|
85
|
+
except Exception:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def item_stage(identifier: str, text: str) -> None:
|
|
90
|
+
"""Say what is happening to one paper; never raises."""
|
|
91
|
+
for fn in list(_STAGE_LISTENERS):
|
|
92
|
+
try:
|
|
93
|
+
fn(identifier, text)
|
|
94
|
+
except Exception:
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _progress_dir() -> Path:
|
|
99
|
+
data_dir = Path(os.environ.get("LITLOOP_DATA_DIR", str(Path.home() / ".litloop")))
|
|
100
|
+
return data_dir / "progress"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def progress_path() -> Path:
|
|
104
|
+
return _progress_dir() / "current.json"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _atomic_write(payload: dict[str, Any]) -> None:
|
|
108
|
+
path = progress_path()
|
|
109
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
|
|
111
|
+
try:
|
|
112
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
113
|
+
json.dump(payload, fh, ensure_ascii=False, indent=2)
|
|
114
|
+
os.replace(tmp, path)
|
|
115
|
+
except Exception:
|
|
116
|
+
try:
|
|
117
|
+
os.unlink(tmp)
|
|
118
|
+
except OSError:
|
|
119
|
+
pass
|
|
120
|
+
raise
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def start_task(task: str, total: int, **extra: Any) -> None:
|
|
124
|
+
"""Begin (or restart) the reported task with a known total."""
|
|
125
|
+
global _STATE, _STARTED_MONO
|
|
126
|
+
with _LOCK:
|
|
127
|
+
_STARTED_MONO = time.monotonic()
|
|
128
|
+
_STATE = {
|
|
129
|
+
"task": task,
|
|
130
|
+
"total": int(total),
|
|
131
|
+
"done": 0,
|
|
132
|
+
"success": 0,
|
|
133
|
+
"failed": 0,
|
|
134
|
+
"current": "",
|
|
135
|
+
"phase": "",
|
|
136
|
+
"status": "running",
|
|
137
|
+
"attention": {},
|
|
138
|
+
"started_at": datetime.now().isoformat(timespec="seconds"),
|
|
139
|
+
"pid": os.getpid(), # readers can tell a live task from a crashed one
|
|
140
|
+
**extra,
|
|
141
|
+
}
|
|
142
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
143
|
+
try:
|
|
144
|
+
_atomic_write(dict(_STATE))
|
|
145
|
+
except Exception:
|
|
146
|
+
pass
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def update(**kw: Any) -> None:
|
|
150
|
+
"""Merge fields into the current report (done/success/failed/current/...)."""
|
|
151
|
+
with _LOCK:
|
|
152
|
+
if not _STATE:
|
|
153
|
+
return
|
|
154
|
+
_STATE.update(kw)
|
|
155
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
156
|
+
try:
|
|
157
|
+
_atomic_write(dict(_STATE))
|
|
158
|
+
except Exception:
|
|
159
|
+
pass
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def advance(ok: bool, current: str = "", phase: str = "") -> None:
|
|
163
|
+
"""Record one finished item (the hot path for per-paper lanes)."""
|
|
164
|
+
for fn in list(_LISTENERS):
|
|
165
|
+
try:
|
|
166
|
+
fn(ok, current)
|
|
167
|
+
except Exception:
|
|
168
|
+
pass
|
|
169
|
+
with _LOCK:
|
|
170
|
+
if not _STATE:
|
|
171
|
+
return
|
|
172
|
+
_STATE["done"] = int(_STATE.get("done", 0)) + 1
|
|
173
|
+
_STATE["success"] = int(_STATE.get("success", 0)) + (1 if ok else 0)
|
|
174
|
+
_STATE["failed"] = int(_STATE.get("failed", 0)) + (0 if ok else 1)
|
|
175
|
+
if current:
|
|
176
|
+
_STATE["current"] = current
|
|
177
|
+
if phase:
|
|
178
|
+
_STATE["phase"] = phase
|
|
179
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
180
|
+
try:
|
|
181
|
+
_atomic_write(dict(_STATE))
|
|
182
|
+
except Exception:
|
|
183
|
+
pass
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def set_attention(key: str, message: str, *, current: str = "", phase: str = "") -> None:
|
|
187
|
+
"""Show a manual-action reminder without stopping other workers."""
|
|
188
|
+
with _LOCK:
|
|
189
|
+
if not _STATE:
|
|
190
|
+
return
|
|
191
|
+
attention = _STATE.setdefault("attention", {})
|
|
192
|
+
if not isinstance(attention, dict):
|
|
193
|
+
attention = {}
|
|
194
|
+
_STATE["attention"] = attention
|
|
195
|
+
before = attention.get(str(key)) or {}
|
|
196
|
+
attention[str(key)] = {
|
|
197
|
+
"message": str(message),
|
|
198
|
+
"current": str(current or ""),
|
|
199
|
+
"phase": str(phase or ""),
|
|
200
|
+
# When the notice first appeared: a waiting source repeats it on
|
|
201
|
+
# every poll, which must not move this.
|
|
202
|
+
"since": before.get("since") or datetime.now().isoformat(timespec="seconds"),
|
|
203
|
+
}
|
|
204
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
205
|
+
try:
|
|
206
|
+
_atomic_write(dict(_STATE))
|
|
207
|
+
except Exception:
|
|
208
|
+
pass
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def set_output_dir(path) -> None:
|
|
212
|
+
"""Record the task's output folder (the report carries it for readers)."""
|
|
213
|
+
with _LOCK:
|
|
214
|
+
if not _STATE:
|
|
215
|
+
return
|
|
216
|
+
_STATE["output_dir"] = str(path)
|
|
217
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
218
|
+
try:
|
|
219
|
+
_atomic_write(dict(_STATE))
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def clear_attention(key: str) -> None:
|
|
225
|
+
"""Remove one manual-action reminder after its browser page clears."""
|
|
226
|
+
with _LOCK:
|
|
227
|
+
if not _STATE:
|
|
228
|
+
return
|
|
229
|
+
attention = _STATE.get("attention")
|
|
230
|
+
if not isinstance(attention, dict):
|
|
231
|
+
return
|
|
232
|
+
attention.pop(str(key), None)
|
|
233
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
234
|
+
try:
|
|
235
|
+
_atomic_write(dict(_STATE))
|
|
236
|
+
except Exception:
|
|
237
|
+
pass
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def finish() -> None:
|
|
241
|
+
"""Mark the task done (the bar keeps the final state briefly, then idles)."""
|
|
242
|
+
with _LOCK:
|
|
243
|
+
if not _STATE:
|
|
244
|
+
return
|
|
245
|
+
_STATE["status"] = "done"
|
|
246
|
+
_STATE["attention"] = {}
|
|
247
|
+
_STATE["updated_at"] = datetime.now().isoformat(timespec="seconds")
|
|
248
|
+
_STATE["finished_at"] = datetime.now().isoformat(timespec="seconds")
|
|
249
|
+
try:
|
|
250
|
+
_atomic_write(dict(_STATE))
|
|
251
|
+
except Exception:
|
|
252
|
+
pass
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def elapsed_sec() -> float:
|
|
256
|
+
return max(0.0, time.monotonic() - _STARTED_MONO)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def running_elsewhere() -> dict[str, Any] | None:
|
|
260
|
+
"""A download another process is running now (the agent through MCP, the
|
|
261
|
+
command line), for the desktop app to show; None when there is none, when
|
|
262
|
+
the report is this process's own, or when its writer has died."""
|
|
263
|
+
state = read_state()
|
|
264
|
+
if not state or state.get("status") != "running":
|
|
265
|
+
return None
|
|
266
|
+
try:
|
|
267
|
+
pid = int(state.get("pid") or 0)
|
|
268
|
+
except (TypeError, ValueError):
|
|
269
|
+
pid = 0
|
|
270
|
+
if pid == os.getpid() or (pid and not _pid_alive(pid)):
|
|
271
|
+
return None
|
|
272
|
+
return state
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def latest_notice() -> dict[str, Any] | None:
|
|
276
|
+
"""The latest download report, running or finished, whichever process wrote
|
|
277
|
+
it: what the desktop app's system notifications need (a new request for the
|
|
278
|
+
user's action, a finished list). None when there is none, or when a running
|
|
279
|
+
report's writer has died."""
|
|
280
|
+
state = read_state()
|
|
281
|
+
if not state:
|
|
282
|
+
return None
|
|
283
|
+
if state.get("status") == "running":
|
|
284
|
+
try:
|
|
285
|
+
pid = int(state.get("pid") or 0)
|
|
286
|
+
except (TypeError, ValueError):
|
|
287
|
+
pid = 0
|
|
288
|
+
if pid and pid != os.getpid() and not _pid_alive(pid):
|
|
289
|
+
return None
|
|
290
|
+
keys = ("task", "status", "total", "done", "success", "failed", "started_at", "finished_at", "attention")
|
|
291
|
+
return {k: state.get(k) for k in keys}
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def read_state() -> dict[str, Any] | None:
|
|
295
|
+
"""Reader side: latest report, or None when absent/stale."""
|
|
296
|
+
try:
|
|
297
|
+
raw = progress_path().read_text(encoding="utf-8")
|
|
298
|
+
state = json.loads(raw)
|
|
299
|
+
except Exception:
|
|
300
|
+
return None
|
|
301
|
+
if not isinstance(state, dict):
|
|
302
|
+
return None
|
|
303
|
+
if state.get("status") == "done":
|
|
304
|
+
return state # finished states stay visible a while (reader decides)
|
|
305
|
+
try:
|
|
306
|
+
ts = datetime.fromisoformat(str(state.get("updated_at")))
|
|
307
|
+
age = (datetime.now() - ts).total_seconds()
|
|
308
|
+
if age > STALE_SECONDS:
|
|
309
|
+
return None
|
|
310
|
+
except Exception:
|
|
311
|
+
pass
|
|
312
|
+
return state
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""PDF auto-rename based on metadata, in the format the user picked (``rename_format``)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import re
|
|
7
|
+
import unicodedata
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..identifiers import safe_filename
|
|
12
|
+
from ..log import get_logger
|
|
13
|
+
|
|
14
|
+
log = get_logger()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# rename_format values -> what the settings page shows. "off" keeps the name the source gave.
|
|
18
|
+
FORMATS = {
|
|
19
|
+
"author_year_title_doi": "作者年份_标题_DOI",
|
|
20
|
+
"author_year_title": "作者年份_标题",
|
|
21
|
+
"zotero": "作者 - 年份 - 完整标题",
|
|
22
|
+
"year_author_title": "年份_作者_标题",
|
|
23
|
+
"title": "完整标题",
|
|
24
|
+
"doi": "DOI",
|
|
25
|
+
"off": "保持原名",
|
|
26
|
+
}
|
|
27
|
+
DEFAULT_FORMAT = "author_year_title_doi"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def rename_format(config: dict[str, Any]) -> str:
|
|
31
|
+
"""The configured format; the older ``auto_rename: false`` means off."""
|
|
32
|
+
if config.get("auto_rename") is False:
|
|
33
|
+
return "off"
|
|
34
|
+
fmt = config.get("rename_format") or DEFAULT_FORMAT
|
|
35
|
+
return fmt if fmt in FORMATS else DEFAULT_FORMAT
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
_ILLEGAL = re.compile('[' + re.escape('<>:"/' + chr(92) + '|?*') + chr(0) + '-' + chr(31) + ']')
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _full_title(metadata: dict[str, Any], max_len: int = 120) -> str:
|
|
42
|
+
"""The whole title, kept in its own script, with characters Windows forbids removed."""
|
|
43
|
+
titles = metadata.get("title", metadata.get("titles", []))
|
|
44
|
+
title = titles[0] if isinstance(titles, list) and titles else titles
|
|
45
|
+
if not isinstance(title, str):
|
|
46
|
+
return ""
|
|
47
|
+
title = re.sub(r"<[^>]+>", "", title)
|
|
48
|
+
title = " ".join(_ILLEGAL.sub(" ", title).split()).strip(" .")
|
|
49
|
+
if len(title) > max_len:
|
|
50
|
+
title = title[:max_len].rsplit(" ", 1)[0].rstrip(" .,;")
|
|
51
|
+
return title
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def generate_filename(metadata: dict[str, Any], doi: str | None = None,
|
|
55
|
+
max_len: int = 60, max_words: int = 4, fmt: str = DEFAULT_FORMAT) -> str:
|
|
56
|
+
"""Generate a clean filename from paper metadata in format ``fmt`` (see FORMATS).
|
|
57
|
+
|
|
58
|
+
Format: LastNameYear_First_Title_Words_<DOI>.pdf. The readable part is
|
|
59
|
+
capped at ``max_words`` words / ``max_len`` chars; the DOI (from ``doi``
|
|
60
|
+
or the metadata) makes the name unique to the paper.
|
|
61
|
+
Example: Jiang2024_Lightweight_Thermally_Insulating_SiBCN_10.1016_j.cej.2024.157656.pdf
|
|
62
|
+
"""
|
|
63
|
+
# Extract year
|
|
64
|
+
year = ""
|
|
65
|
+
for key in ("year", "published-print", "published-online", "created"):
|
|
66
|
+
val = metadata.get(key)
|
|
67
|
+
if isinstance(val, str) and val.isdigit():
|
|
68
|
+
year = val
|
|
69
|
+
break
|
|
70
|
+
elif isinstance(val, dict):
|
|
71
|
+
parts = val.get("date-parts", [[]])
|
|
72
|
+
if parts and parts[0]:
|
|
73
|
+
year = str(parts[0][0])
|
|
74
|
+
break
|
|
75
|
+
elif isinstance(val, (int, float)):
|
|
76
|
+
year = str(int(val))
|
|
77
|
+
break
|
|
78
|
+
|
|
79
|
+
# Extract first author last name
|
|
80
|
+
authors = metadata.get("author", metadata.get("authors", []))
|
|
81
|
+
first_author = ""
|
|
82
|
+
if authors and isinstance(authors, list):
|
|
83
|
+
a = authors[0]
|
|
84
|
+
if isinstance(a, dict):
|
|
85
|
+
first_author = a.get("family", a.get("given", a.get("name", "")))
|
|
86
|
+
elif isinstance(a, str):
|
|
87
|
+
# "Last, First" or "First Last"
|
|
88
|
+
parts = a.split(",")
|
|
89
|
+
first_author = parts[0].strip().split()[-1] if parts else a
|
|
90
|
+
|
|
91
|
+
# Extract the first meaningful words of the title
|
|
92
|
+
titles = metadata.get("title", metadata.get("titles", []))
|
|
93
|
+
title_words: list[str] = []
|
|
94
|
+
if titles:
|
|
95
|
+
title = titles[0] if isinstance(titles, list) else titles
|
|
96
|
+
if isinstance(title, str):
|
|
97
|
+
title = re.sub(r"<[^>]+>", "", _ascii(title))
|
|
98
|
+
stop_words = {"a", "an", "the", "on", "in", "of", "for", "to", "and", "with", "from",
|
|
99
|
+
"by", "via", "at", "as", "its", "their", "into", "using"}
|
|
100
|
+
for w in re.findall(r"[A-Za-z0-9]+", title):
|
|
101
|
+
if w.lower() in stop_words or (len(w) < 3 and not any(c.isdigit() for c in w)):
|
|
102
|
+
continue
|
|
103
|
+
title_words.append(w if any(c.isupper() for c in w[1:]) else w.capitalize())
|
|
104
|
+
if len(title_words) == max_words:
|
|
105
|
+
break
|
|
106
|
+
|
|
107
|
+
# Clean components
|
|
108
|
+
first_author = re.sub(r"[^A-Za-z]", "", _ascii(first_author))[:20]
|
|
109
|
+
year = re.sub(r"[^0-9]", "", year)[:4]
|
|
110
|
+
title_word = "_".join(title_words)
|
|
111
|
+
|
|
112
|
+
# Build filename: LastNameYear_TitleWord
|
|
113
|
+
parts = []
|
|
114
|
+
if first_author:
|
|
115
|
+
parts.append(first_author)
|
|
116
|
+
if year:
|
|
117
|
+
parts.append(year)
|
|
118
|
+
if title_word:
|
|
119
|
+
parts.append(title_word)
|
|
120
|
+
|
|
121
|
+
if not parts and fmt not in ("doi", "title"):
|
|
122
|
+
doi = doi or metadata.get("DOI") or metadata.get("doi")
|
|
123
|
+
return safe_filename(str(doi).strip().lower()) if doi and fmt != "author_year_title_doi" else ""
|
|
124
|
+
|
|
125
|
+
doi = doi or metadata.get("DOI") or metadata.get("doi")
|
|
126
|
+
doi_part = safe_filename(str(doi).strip().lower()) if doi else ""
|
|
127
|
+
if fmt == "doi":
|
|
128
|
+
return doi_part
|
|
129
|
+
if fmt == "title":
|
|
130
|
+
return _full_title(metadata) or doi_part
|
|
131
|
+
if fmt == "zotero":
|
|
132
|
+
# Zotero's own pattern keeps names in their script (李, Müller).
|
|
133
|
+
raw = authors[0] if authors and isinstance(authors, list) else ""
|
|
134
|
+
raw = (raw.get("family") or raw.get("name") or "") if isinstance(raw, dict) else str(raw).split(",")[0]
|
|
135
|
+
author = " ".join(_ILLEGAL.sub(" ", raw).split())[:30] or first_author
|
|
136
|
+
named = " - ".join(x for x in (author, year, _full_title(metadata)) if x)
|
|
137
|
+
return named or doi_part
|
|
138
|
+
if fmt == "year_author_title":
|
|
139
|
+
name = "_".join(x for x in (year, first_author, title_word) if x)
|
|
140
|
+
if len(name) > max_len:
|
|
141
|
+
name = name[:max_len].rsplit("_", 1)[0]
|
|
142
|
+
return name or doi_part
|
|
143
|
+
|
|
144
|
+
# Join first two parts directly (AuthorYear), then underscore with title
|
|
145
|
+
if len(parts) >= 3:
|
|
146
|
+
name = f"{parts[0]}{parts[1]}_{parts[2]}"
|
|
147
|
+
elif len(parts) == 2:
|
|
148
|
+
name = f"{parts[0]}{parts[1]}"
|
|
149
|
+
else:
|
|
150
|
+
name = parts[0]
|
|
151
|
+
# Truncate at a word boundary if too long
|
|
152
|
+
if len(name) > max_len:
|
|
153
|
+
name = name[:max_len].rsplit("_", 1)[0] if "_" in name[:max_len] else name[:max_len]
|
|
154
|
+
if fmt == "author_year_title_doi" and doi_part:
|
|
155
|
+
name = f"{name}_{doi_part}"
|
|
156
|
+
return name
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _ascii(text: str) -> str:
|
|
160
|
+
"""Fold accents so names stay readable (Müller -> Muller)."""
|
|
161
|
+
return unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _md5(path: Path) -> str:
|
|
165
|
+
h = hashlib.md5()
|
|
166
|
+
with path.open("rb") as fh:
|
|
167
|
+
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
168
|
+
h.update(chunk)
|
|
169
|
+
return h.hexdigest()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def rename_pdf(
|
|
173
|
+
file_path: Path,
|
|
174
|
+
metadata: dict[str, Any],
|
|
175
|
+
target_dir: Path | None = None,
|
|
176
|
+
doi: str | None = None,
|
|
177
|
+
fmt: str = DEFAULT_FORMAT,
|
|
178
|
+
) -> Path | None:
|
|
179
|
+
"""Rename a downloaded PDF based on metadata. Returns new path or None if failed."""
|
|
180
|
+
if not file_path.exists() or fmt == "off":
|
|
181
|
+
return None
|
|
182
|
+
|
|
183
|
+
new_name = generate_filename(metadata, doi=doi, fmt=fmt)
|
|
184
|
+
if not new_name:
|
|
185
|
+
return None
|
|
186
|
+
|
|
187
|
+
new_name = f"{new_name}.pdf"
|
|
188
|
+
target = (target_dir or file_path.parent) / new_name
|
|
189
|
+
|
|
190
|
+
# Never overwrite. Identical content (md5) is reused; anything else
|
|
191
|
+
# (another copy of the same DOI, or a legacy name) gets a numeric suffix.
|
|
192
|
+
if target.exists() and target != file_path:
|
|
193
|
+
try:
|
|
194
|
+
if _md5(target) == _md5(file_path):
|
|
195
|
+
file_path.unlink(missing_ok=True)
|
|
196
|
+
return target
|
|
197
|
+
except OSError:
|
|
198
|
+
pass
|
|
199
|
+
stem, counter = target.stem, 2
|
|
200
|
+
while target.exists() and target != file_path:
|
|
201
|
+
target = target.parent / f"{stem}_{counter}.pdf"
|
|
202
|
+
counter += 1
|
|
203
|
+
|
|
204
|
+
try:
|
|
205
|
+
if target != file_path:
|
|
206
|
+
file_path.rename(target)
|
|
207
|
+
log.info(f"Renamed: {file_path.name} -> {target.name}")
|
|
208
|
+
return target
|
|
209
|
+
except OSError as e:
|
|
210
|
+
log.warning(f"Rename failed: {e}")
|
|
211
|
+
return None
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""HTML extraction router - dispatches to publisher-specific adapters."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from .publisher_adapters import acs, elsevier, generic, nature, rsc, tandfonline, wiley
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
# Ordered list of publisher adapters (checked in order)
|
|
10
|
+
_ADAPTERS = [nature, elsevier, wiley, acs, rsc, tandfonline]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def extract(html: str, url: str = "") -> dict:
|
|
14
|
+
"""Extract paper content from HTML, routing to the best adapter.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
html: Raw HTML content.
|
|
18
|
+
url: The URL the HTML was fetched from (used for publisher detection).
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
Dict with title, authors, abstract, full_text, figures, references.
|
|
22
|
+
"""
|
|
23
|
+
# Try publisher-specific adapters first
|
|
24
|
+
for adapter in _ADAPTERS:
|
|
25
|
+
if adapter.can_handle(url):
|
|
26
|
+
logger.info("Using %s adapter for %s", adapter.__name__, url)
|
|
27
|
+
return adapter.extract(html, url)
|
|
28
|
+
|
|
29
|
+
# Fallback to generic
|
|
30
|
+
logger.info("Using generic adapter for %s", url)
|
|
31
|
+
return generic.extract(html, url)
|