litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/optional.py
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""Optional parts of LitLoop that the installer leaves out.
|
|
2
|
+
|
|
3
|
+
The layout model (`pymupdf-layout`, `onnxruntime`, `networkx`, about 65 MB) only
|
|
4
|
+
makes PDF-to-markdown tidier: without it `download/markdown.py` falls back to
|
|
5
|
+
plain reading order. The installer ships without it; the settings page installs
|
|
6
|
+
it on demand with uv into `<data dir>/packages`, and this module puts that
|
|
7
|
+
directory on `sys.path` so the rest of LitLoop can import it as if it had been
|
|
8
|
+
installed normally.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import importlib.util
|
|
14
|
+
import os
|
|
15
|
+
import shutil
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
#: Where on-demand components go. One directory for all of them: removing one
|
|
24
|
+
#: file is enough to uninstall, and a component never shadows a bundled module.
|
|
25
|
+
COMPONENTS: dict[str, dict[str, Any]] = {
|
|
26
|
+
"layout": {
|
|
27
|
+
"label": "版面分析",
|
|
28
|
+
"packages": ["pymupdf-layout", "onnxruntime", "networkx"],
|
|
29
|
+
"module": "onnxruntime",
|
|
30
|
+
"size_mb": 65,
|
|
31
|
+
},
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
_INSTALL_LOCK = threading.Lock()
|
|
35
|
+
_STATUS_LOCK = threading.Lock()
|
|
36
|
+
_STATUS: dict[str, dict[str, Any]] = {}
|
|
37
|
+
|
|
38
|
+
_STAGES = ("准备环境", "正在下载依赖", "正在安装", "正在校验")
|
|
39
|
+
_STAGE_ORDER = {name: idx for idx, name in enumerate(_STAGES)}
|
|
40
|
+
_NETWORK_ERROR_HINTS = (
|
|
41
|
+
"timed out",
|
|
42
|
+
"timeout",
|
|
43
|
+
"connection",
|
|
44
|
+
"ssl",
|
|
45
|
+
"certificate",
|
|
46
|
+
"dns",
|
|
47
|
+
"name or service not known",
|
|
48
|
+
"temporary failure in name resolution",
|
|
49
|
+
"failed to fetch",
|
|
50
|
+
"network",
|
|
51
|
+
"proxy",
|
|
52
|
+
"403",
|
|
53
|
+
"404",
|
|
54
|
+
"502",
|
|
55
|
+
"503",
|
|
56
|
+
"index",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def packages_dir() -> Path:
|
|
61
|
+
"""`<data dir>/packages`, next to the config file (LITLOOP_DATA_DIR moves both)."""
|
|
62
|
+
root = Path(os.environ.get("LITLOOP_DATA_DIR") or (Path.home() / ".litloop"))
|
|
63
|
+
return root / "packages"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def ensure_on_path() -> None:
|
|
67
|
+
"""Make installed components importable. Called once, when LitLoop starts."""
|
|
68
|
+
folder = packages_dir()
|
|
69
|
+
entry = str(folder)
|
|
70
|
+
try:
|
|
71
|
+
if folder.is_dir() and entry not in sys.path:
|
|
72
|
+
sys.path.insert(0, entry)
|
|
73
|
+
except OSError:
|
|
74
|
+
pass
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def installed(key: str) -> bool:
|
|
78
|
+
"""Whether the component can be imported right now."""
|
|
79
|
+
spec = COMPONENTS.get(key) or {}
|
|
80
|
+
module = str(spec.get("module") or "")
|
|
81
|
+
if not module:
|
|
82
|
+
return False
|
|
83
|
+
ensure_on_path()
|
|
84
|
+
try:
|
|
85
|
+
return importlib.util.find_spec(module) is not None
|
|
86
|
+
except (ImportError, ValueError):
|
|
87
|
+
return False
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def uv_command() -> str | None:
|
|
91
|
+
"""uv, if it can be found: shipped next to Python, on PATH, or configured."""
|
|
92
|
+
for candidate in (os.environ.get("LITLOOP_UV", ""),
|
|
93
|
+
str(Path(sys.executable).with_name("uv.exe")),
|
|
94
|
+
str(Path(sys.executable).with_name("uv")),
|
|
95
|
+
shutil.which("uv") or ""):
|
|
96
|
+
if candidate and Path(candidate).is_file():
|
|
97
|
+
return candidate
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _set_status(
|
|
102
|
+
key: str,
|
|
103
|
+
*,
|
|
104
|
+
state: str,
|
|
105
|
+
stage: str = "",
|
|
106
|
+
elapsed: float = 0.0,
|
|
107
|
+
line: str = "",
|
|
108
|
+
ok: bool | None = None,
|
|
109
|
+
log: str = "",
|
|
110
|
+
started_at: float | None = None,
|
|
111
|
+
) -> None:
|
|
112
|
+
"""Record the latest install progress for *key* under ``_STATUS_LOCK``."""
|
|
113
|
+
with _STATUS_LOCK:
|
|
114
|
+
_STATUS[key] = {
|
|
115
|
+
"key": key,
|
|
116
|
+
"state": state,
|
|
117
|
+
"stage": stage,
|
|
118
|
+
"elapsed": round(float(elapsed), 1),
|
|
119
|
+
"line": line[:160],
|
|
120
|
+
"ok": ok,
|
|
121
|
+
"log": log,
|
|
122
|
+
"started_at": started_at,
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def install_status(key: str) -> dict[str, Any]:
|
|
127
|
+
"""Return the current install status for an optional component."""
|
|
128
|
+
if key not in COMPONENTS:
|
|
129
|
+
raise KeyError(f"no such component: {key}")
|
|
130
|
+
with _STATUS_LOCK:
|
|
131
|
+
entry = dict(_STATUS.get(key) or {})
|
|
132
|
+
if not entry:
|
|
133
|
+
return {
|
|
134
|
+
"key": key,
|
|
135
|
+
"installed": installed(key),
|
|
136
|
+
"state": "idle",
|
|
137
|
+
"stage": "",
|
|
138
|
+
"elapsed": 0.0,
|
|
139
|
+
"line": "",
|
|
140
|
+
"ok": None,
|
|
141
|
+
"log": "",
|
|
142
|
+
}
|
|
143
|
+
elapsed = float(entry.get("elapsed") or 0.0)
|
|
144
|
+
started_at = entry.get("started_at")
|
|
145
|
+
if entry.get("state") == "running" and isinstance(started_at, (int, float)):
|
|
146
|
+
elapsed = round(max(0.0, time.monotonic() - started_at), 1)
|
|
147
|
+
return {
|
|
148
|
+
"key": key,
|
|
149
|
+
"installed": installed(key),
|
|
150
|
+
"state": str(entry.get("state") or "idle"),
|
|
151
|
+
"stage": str(entry.get("stage") or ""),
|
|
152
|
+
"elapsed": elapsed,
|
|
153
|
+
"line": str(entry.get("line") or ""),
|
|
154
|
+
"ok": entry.get("ok"),
|
|
155
|
+
"log": str(entry.get("log") or ""),
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _stage_from_line(line: str, current: str) -> str:
|
|
160
|
+
"""Advance the install stage monotonically from a single line of uv output."""
|
|
161
|
+
low = line.lower()
|
|
162
|
+
detected = current
|
|
163
|
+
if any(tok in low for tok in ("installing", "unpacking", "installed", "built ")):
|
|
164
|
+
detected = "正在安装"
|
|
165
|
+
elif any(tok in low for tok in ("downloading", "fetching", "resolved ", "prepared ")):
|
|
166
|
+
detected = "正在下载依赖"
|
|
167
|
+
if _STAGE_ORDER.get(detected, 0) >= _STAGE_ORDER.get(current, 0):
|
|
168
|
+
return detected
|
|
169
|
+
return current
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _format_failure_log(returncode: int, tail: str) -> str:
|
|
173
|
+
"""Build a clear Chinese failure reason with trailing uv output."""
|
|
174
|
+
low = tail.lower()
|
|
175
|
+
if any(hint in low for hint in _NETWORK_ERROR_HINTS):
|
|
176
|
+
prefix = f"网络或镜像连接失败(退出码 {returncode}),请在设置中检查网络或更换镜像源后重试"
|
|
177
|
+
else:
|
|
178
|
+
prefix = f"安装失败(uv 退出码 {returncode})"
|
|
179
|
+
return f"{prefix}\n{tail}" if tail else prefix
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def install(key: str, *, index_url: str = "", timeout: float = 1800.0) -> dict[str, Any]:
|
|
183
|
+
"""Download a component into `packages_dir()`. Blocking; run it in a thread.
|
|
184
|
+
|
|
185
|
+
``index_url`` overrides the package index (a domestic mirror), since the
|
|
186
|
+
official PyPI is slow from some networks. Returns ``{ok, log}``: the log is
|
|
187
|
+
the tail of uv's output, which the settings page shows when it fails.
|
|
188
|
+
"""
|
|
189
|
+
spec = COMPONENTS.get(key)
|
|
190
|
+
if not spec:
|
|
191
|
+
raise KeyError(f"no such component: {key}")
|
|
192
|
+
uv = uv_command()
|
|
193
|
+
if not uv:
|
|
194
|
+
msg = "找不到 uv:安装包应当自带,或在 PATH 里放一个 uv"
|
|
195
|
+
_set_status(key, state="failed", stage="", elapsed=0.0, line=msg, ok=False, log=msg)
|
|
196
|
+
return {"ok": False, "log": msg}
|
|
197
|
+
target = packages_dir()
|
|
198
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
199
|
+
cmd = [uv, "pip", "install", "--python", sys.executable, "--target", str(target),
|
|
200
|
+
*spec["packages"]]
|
|
201
|
+
env = {**os.environ, "UV_LINK_MODE": "copy"}
|
|
202
|
+
if index_url:
|
|
203
|
+
env["UV_INDEX_URL"] = index_url
|
|
204
|
+
if not _INSTALL_LOCK.acquire(timeout=5):
|
|
205
|
+
return {"ok": False, "log": "另一个组件正在安装,稍后再试"}
|
|
206
|
+
|
|
207
|
+
started = time.monotonic()
|
|
208
|
+
stage = "准备环境"
|
|
209
|
+
lines: list[str] = []
|
|
210
|
+
last_line = ""
|
|
211
|
+
timed_out = False
|
|
212
|
+
_set_status(key, state="running", stage=stage, elapsed=0.0, line="", ok=None, log="", started_at=started)
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
proc = subprocess.Popen(
|
|
216
|
+
cmd,
|
|
217
|
+
stdout=subprocess.PIPE,
|
|
218
|
+
stderr=subprocess.STDOUT,
|
|
219
|
+
text=True,
|
|
220
|
+
encoding="utf-8",
|
|
221
|
+
errors="replace",
|
|
222
|
+
bufsize=1,
|
|
223
|
+
env=env,
|
|
224
|
+
)
|
|
225
|
+
except OSError as exc:
|
|
226
|
+
_INSTALL_LOCK.release()
|
|
227
|
+
elapsed = round(time.monotonic() - started, 1)
|
|
228
|
+
msg = f"无法运行 uv:{exc}"
|
|
229
|
+
_set_status(key, state="failed", stage=stage, elapsed=elapsed, line=msg, ok=False, log=msg)
|
|
230
|
+
return {"ok": False, "log": msg}
|
|
231
|
+
|
|
232
|
+
def _on_timeout() -> None:
|
|
233
|
+
nonlocal timed_out
|
|
234
|
+
timed_out = True
|
|
235
|
+
try:
|
|
236
|
+
proc.kill()
|
|
237
|
+
except OSError:
|
|
238
|
+
pass
|
|
239
|
+
|
|
240
|
+
timer = threading.Timer(timeout, _on_timeout)
|
|
241
|
+
timer.daemon = True
|
|
242
|
+
timer.start()
|
|
243
|
+
try:
|
|
244
|
+
if proc.stdout is not None:
|
|
245
|
+
for raw_line in proc.stdout:
|
|
246
|
+
text_line = raw_line.rstrip("\r\n")
|
|
247
|
+
stripped = text_line.strip()
|
|
248
|
+
if stripped:
|
|
249
|
+
lines.append(stripped)
|
|
250
|
+
last_line = stripped[:160]
|
|
251
|
+
stage = _stage_from_line(stripped, stage)
|
|
252
|
+
elapsed = round(time.monotonic() - started, 1)
|
|
253
|
+
_set_status(
|
|
254
|
+
key,
|
|
255
|
+
state="running",
|
|
256
|
+
stage=stage,
|
|
257
|
+
elapsed=elapsed,
|
|
258
|
+
line=last_line,
|
|
259
|
+
ok=None,
|
|
260
|
+
log="\n".join(lines[-20:]),
|
|
261
|
+
started_at=started,
|
|
262
|
+
)
|
|
263
|
+
returncode = proc.wait()
|
|
264
|
+
finally:
|
|
265
|
+
timer.cancel()
|
|
266
|
+
if proc.stdout is not None:
|
|
267
|
+
try:
|
|
268
|
+
proc.stdout.close()
|
|
269
|
+
except OSError:
|
|
270
|
+
pass
|
|
271
|
+
_INSTALL_LOCK.release()
|
|
272
|
+
|
|
273
|
+
tail = "\n".join(lines[-20:]).strip()
|
|
274
|
+
elapsed = round(time.monotonic() - started, 1)
|
|
275
|
+
|
|
276
|
+
if timed_out:
|
|
277
|
+
msg = f"安装超过了 {int(timeout)} 秒,已放弃"
|
|
278
|
+
full_log = f"{msg}\n{tail}" if tail else msg
|
|
279
|
+
_set_status(key, state="failed", stage=stage, elapsed=elapsed, line=msg, ok=False, log=full_log)
|
|
280
|
+
return {"ok": False, "log": full_log}
|
|
281
|
+
|
|
282
|
+
if returncode != 0:
|
|
283
|
+
msg = _format_failure_log(returncode, tail)
|
|
284
|
+
_set_status(key, state="failed", stage=stage, elapsed=elapsed, line=last_line or msg, ok=False, log=msg)
|
|
285
|
+
return {"ok": False, "log": msg}
|
|
286
|
+
|
|
287
|
+
stage = "正在校验"
|
|
288
|
+
_set_status(
|
|
289
|
+
key,
|
|
290
|
+
state="running",
|
|
291
|
+
stage=stage,
|
|
292
|
+
elapsed=elapsed,
|
|
293
|
+
line=last_line,
|
|
294
|
+
ok=None,
|
|
295
|
+
log=tail,
|
|
296
|
+
started_at=started,
|
|
297
|
+
)
|
|
298
|
+
# The new directory may be new to the import system.
|
|
299
|
+
importlib.invalidate_caches()
|
|
300
|
+
ensure_on_path()
|
|
301
|
+
ok = installed(key)
|
|
302
|
+
elapsed = round(time.monotonic() - started, 1)
|
|
303
|
+
if ok:
|
|
304
|
+
log = tail or "已安装"
|
|
305
|
+
_set_status(key, state="done", stage=stage, elapsed=elapsed, line=last_line or "已安装", ok=True, log=log)
|
|
306
|
+
return {"ok": True, "log": log}
|
|
307
|
+
|
|
308
|
+
msg = f"安装命令已完成,但仍无法导入组件。\n{tail}" if tail else "安装命令已完成,但仍无法导入组件"
|
|
309
|
+
_set_status(key, state="failed", stage=stage, elapsed=elapsed, line=last_line or msg, ok=False, log=msg)
|
|
310
|
+
return {"ok": False, "log": msg}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Publisher access rules, profiles, PDF routers, and download strategies."""
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""Reusable publisher access catalog and live verification helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from .pdf_router import build_pdf_candidates, discover_pdf_candidates_from_html
|
|
12
|
+
from .profiles import get_publisher_profile, list_publisher_profiles
|
|
13
|
+
|
|
14
|
+
CATALOG_PATH = Path(__file__).parent.parent / "data" / "publisher_access_catalog.json"
|
|
15
|
+
INSTITUTIONAL_IDENTITY_POLICY_PATH = Path(__file__).parent.parent / "data" / "institutional_identity_policy.json"
|
|
16
|
+
|
|
17
|
+
DEFAULT_HEADERS = {
|
|
18
|
+
"User-Agent": (
|
|
19
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
20
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
|
|
21
|
+
),
|
|
22
|
+
"Accept": "text/html,application/xhtml+xml,application/pdf;q=0.9,*/*;q=0.8",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def load_publisher_access_catalog(path: str | Path | None = None) -> dict[str, Any]:
|
|
27
|
+
"""Load the machine-readable publisher access catalog."""
|
|
28
|
+
catalog_path = Path(path) if path else CATALOG_PATH
|
|
29
|
+
return json.loads(catalog_path.read_text(encoding="utf-8"))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def load_institutional_identity_policy(path: str | Path | None = None) -> dict[str, Any]:
|
|
33
|
+
"""Load reusable policy for choosing institutional identity routes."""
|
|
34
|
+
policy_path = Path(path) if path else INSTITUTIONAL_IDENTITY_POLICY_PATH
|
|
35
|
+
return json.loads(policy_path.read_text(encoding="utf-8"))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def verify_publisher_access(
|
|
39
|
+
publisher: str,
|
|
40
|
+
*,
|
|
41
|
+
session: requests.Session | None = None,
|
|
42
|
+
probe_pdf: bool = True,
|
|
43
|
+
max_candidates: int = 4,
|
|
44
|
+
timeout: int = 20,
|
|
45
|
+
) -> dict[str, Any]:
|
|
46
|
+
"""Resolve a sample DOI and classify official PDF candidate behavior."""
|
|
47
|
+
catalog = load_publisher_access_catalog()
|
|
48
|
+
profile = get_publisher_profile(publisher)
|
|
49
|
+
key = _profile_key(publisher)
|
|
50
|
+
entry = catalog["publishers"][key]
|
|
51
|
+
doi = entry["verification"]["sample_doi"]
|
|
52
|
+
http = session or requests.Session()
|
|
53
|
+
|
|
54
|
+
landing = _get(
|
|
55
|
+
http,
|
|
56
|
+
f"https://doi.org/{doi}",
|
|
57
|
+
timeout=timeout,
|
|
58
|
+
headers=DEFAULT_HEADERS,
|
|
59
|
+
allow_redirects=True,
|
|
60
|
+
)
|
|
61
|
+
landing_url = getattr(landing, "url", "")
|
|
62
|
+
history_urls = [getattr(item, "url", "") for item in getattr(landing, "history", [])]
|
|
63
|
+
chain = [url for url in [*history_urls, landing_url] if url]
|
|
64
|
+
landing_status = int(getattr(landing, "status_code", 0) or 0)
|
|
65
|
+
landing_text = _response_text(landing)
|
|
66
|
+
|
|
67
|
+
discovered = discover_pdf_candidates_from_html(landing_text, landing_url) if landing_text else []
|
|
68
|
+
candidates = build_pdf_candidates(
|
|
69
|
+
profile,
|
|
70
|
+
doi,
|
|
71
|
+
source_url=landing_url,
|
|
72
|
+
discovered_urls=discovered,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
candidate_probes: list[dict[str, Any]] = []
|
|
76
|
+
if probe_pdf:
|
|
77
|
+
for url in candidates[:max(0, max_candidates)]:
|
|
78
|
+
candidate_probes.append(_probe_candidate(http, url, timeout=timeout))
|
|
79
|
+
|
|
80
|
+
expected_domains = [domain.lower() for domain in entry["verification"].get("expected_domains", [])]
|
|
81
|
+
chain_haystack = " ".join(chain).lower()
|
|
82
|
+
resolved_to_expected_domain = any(domain in chain_haystack for domain in expected_domains)
|
|
83
|
+
observed_access = _observed_access(landing, candidate_probes)
|
|
84
|
+
return {
|
|
85
|
+
"profile_key": key,
|
|
86
|
+
"display_name": entry["display_name"],
|
|
87
|
+
"sample_doi": doi,
|
|
88
|
+
"landing_status": landing_status,
|
|
89
|
+
"landing_url": landing_url,
|
|
90
|
+
"redirect_chain": chain,
|
|
91
|
+
"resolved_to_expected_domain": resolved_to_expected_domain,
|
|
92
|
+
"pdf_candidates": candidates,
|
|
93
|
+
"candidate_probes": candidate_probes,
|
|
94
|
+
"observed_access": observed_access,
|
|
95
|
+
"identity": entry["identity"],
|
|
96
|
+
"persistence": entry["persistence"],
|
|
97
|
+
"link_characteristics": entry["link_characteristics"],
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def verify_publishers(
|
|
102
|
+
publishers: list[str] | None = None,
|
|
103
|
+
*,
|
|
104
|
+
probe_pdf: bool = True,
|
|
105
|
+
max_candidates: int = 4,
|
|
106
|
+
timeout: int = 20,
|
|
107
|
+
) -> list[dict[str, Any]]:
|
|
108
|
+
keys = publishers or list_publisher_profiles()
|
|
109
|
+
session = requests.Session()
|
|
110
|
+
return [
|
|
111
|
+
verify_publisher_access(
|
|
112
|
+
key,
|
|
113
|
+
session=session,
|
|
114
|
+
probe_pdf=probe_pdf,
|
|
115
|
+
max_candidates=max_candidates,
|
|
116
|
+
timeout=timeout,
|
|
117
|
+
)
|
|
118
|
+
for key in keys
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _profile_key(publisher: str) -> str:
|
|
123
|
+
profile = get_publisher_profile(publisher)
|
|
124
|
+
for key in list_publisher_profiles():
|
|
125
|
+
if get_publisher_profile(key) is profile:
|
|
126
|
+
return key
|
|
127
|
+
raise ValueError(f"Unknown publisher profile: {publisher}")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _get(session: requests.Session, url: str, **kwargs):
|
|
131
|
+
return session.get(url, **kwargs)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _probe_candidate(session: requests.Session, url: str, *, timeout: int) -> dict[str, Any]:
|
|
135
|
+
headers = {
|
|
136
|
+
**DEFAULT_HEADERS,
|
|
137
|
+
"Accept": "application/pdf,text/html;q=0.8,*/*;q=0.5",
|
|
138
|
+
"Range": "bytes=0-4095",
|
|
139
|
+
}
|
|
140
|
+
try:
|
|
141
|
+
response = _get(
|
|
142
|
+
session,
|
|
143
|
+
url,
|
|
144
|
+
headers=headers,
|
|
145
|
+
timeout=timeout,
|
|
146
|
+
allow_redirects=True,
|
|
147
|
+
stream=True,
|
|
148
|
+
)
|
|
149
|
+
result = {
|
|
150
|
+
"url": url,
|
|
151
|
+
"final_url": getattr(response, "url", url),
|
|
152
|
+
"status": int(getattr(response, "status_code", 0) or 0),
|
|
153
|
+
"content_type": _header(response, "content-type"),
|
|
154
|
+
"classification": _classify_response(response),
|
|
155
|
+
}
|
|
156
|
+
close = getattr(response, "close", None)
|
|
157
|
+
if close:
|
|
158
|
+
close()
|
|
159
|
+
return result
|
|
160
|
+
except requests.RequestException as exc:
|
|
161
|
+
return {
|
|
162
|
+
"url": url,
|
|
163
|
+
"final_url": "",
|
|
164
|
+
"status": 0,
|
|
165
|
+
"content_type": "",
|
|
166
|
+
"classification": "network_error",
|
|
167
|
+
"error": str(exc),
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _classify_response(response) -> str:
|
|
172
|
+
status = int(getattr(response, "status_code", 0) or 0)
|
|
173
|
+
content_type = _header(response, "content-type").lower()
|
|
174
|
+
final_url = str(getattr(response, "url", "")).lower()
|
|
175
|
+
text = _response_text(response, limit=4000).lower()
|
|
176
|
+
login_markers = (
|
|
177
|
+
"login", "signin", "sign in", "openathens",
|
|
178
|
+
"institution", "shibboleth", "access through",
|
|
179
|
+
)
|
|
180
|
+
challenge_markers = ("captcha", "robot", "challenge", "perfdrive", "cloudflare", "validate.perfdrive")
|
|
181
|
+
if 200 <= status < 300 and "pdf" in content_type:
|
|
182
|
+
return "pdf_accessible"
|
|
183
|
+
if status in {401, 402, 403}:
|
|
184
|
+
if any(marker in final_url or marker in text for marker in challenge_markers):
|
|
185
|
+
return "challenge_or_bot_check"
|
|
186
|
+
return "identity_required"
|
|
187
|
+
if status == 404:
|
|
188
|
+
return "not_found"
|
|
189
|
+
if status >= 500:
|
|
190
|
+
return "server_error"
|
|
191
|
+
if any(marker in final_url or marker in text for marker in challenge_markers):
|
|
192
|
+
return "challenge_or_bot_check"
|
|
193
|
+
if any(marker in final_url or marker in text for marker in login_markers):
|
|
194
|
+
return "identity_required"
|
|
195
|
+
if 200 <= status < 300 and "html" in content_type:
|
|
196
|
+
return "html_or_reader"
|
|
197
|
+
if 300 <= status < 400:
|
|
198
|
+
return "redirect"
|
|
199
|
+
return "unknown"
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _observed_access(landing, candidate_probes: list[dict[str, Any]]) -> str:
|
|
203
|
+
classifications = [probe["classification"] for probe in candidate_probes]
|
|
204
|
+
if "pdf_accessible" in classifications:
|
|
205
|
+
return "pdf_accessible"
|
|
206
|
+
if "identity_required" in classifications:
|
|
207
|
+
return "identity_required"
|
|
208
|
+
if "challenge_or_bot_check" in classifications or _classify_response(landing) == "challenge_or_bot_check":
|
|
209
|
+
return "challenge_or_bot_check"
|
|
210
|
+
if not candidate_probes:
|
|
211
|
+
return _classify_response(landing)
|
|
212
|
+
if all(item == "not_found" for item in classifications):
|
|
213
|
+
return "not_found"
|
|
214
|
+
return classifications[0] if classifications else "unknown"
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _response_text(response, *, limit: int = 200_000) -> str:
|
|
218
|
+
content_type = _header(response, "content-type").lower()
|
|
219
|
+
if content_type and "html" not in content_type and "text" not in content_type and "xml" not in content_type:
|
|
220
|
+
return ""
|
|
221
|
+
try:
|
|
222
|
+
return str(getattr(response, "text", "") or "")[:limit]
|
|
223
|
+
except Exception:
|
|
224
|
+
return ""
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _header(response, name: str) -> str:
|
|
228
|
+
headers = getattr(response, "headers", {}) or {}
|
|
229
|
+
try:
|
|
230
|
+
return str(headers.get(name) or headers.get(name.title()) or "")
|
|
231
|
+
except AttributeError:
|
|
232
|
+
return ""
|