litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/cli/main.py
ADDED
|
@@ -0,0 +1,1173 @@
|
|
|
1
|
+
"""CLI entrypoint for LitLoop server."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from enum import Enum
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Any
|
|
10
|
+
|
|
11
|
+
import typer
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(help="LitLoop server")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ServerMode(str, Enum):
|
|
17
|
+
STDIO = "stdio"
|
|
18
|
+
HTTP = "streamable_http"
|
|
19
|
+
WEB = "web"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _default_output_dir(output: str) -> str:
|
|
23
|
+
"""CLI downloads land where MCP downloads do (config output_dir), not in the cwd."""
|
|
24
|
+
if output:
|
|
25
|
+
return output
|
|
26
|
+
from ..config import load_config
|
|
27
|
+
return str(Path(load_config().get("output_dir") or ".").expanduser())
|
|
28
|
+
|
|
29
|
+
def _version_callback(value: bool) -> None:
|
|
30
|
+
if value:
|
|
31
|
+
from .. import __version__
|
|
32
|
+
|
|
33
|
+
typer.echo(f"litloop {__version__}")
|
|
34
|
+
raise typer.Exit()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@app.callback()
|
|
38
|
+
def _options(
|
|
39
|
+
version: bool = typer.Option(
|
|
40
|
+
False, "--version", callback=_version_callback, is_eager=True,
|
|
41
|
+
help="Show the version and exit.",
|
|
42
|
+
),
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Search, screen, download and file papers."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@app.command("run")
|
|
48
|
+
def run_server(
|
|
49
|
+
mode: ServerMode = typer.Option(ServerMode.STDIO, help="Transport mode"),
|
|
50
|
+
host: str = typer.Option("127.0.0.1", help="HTTP host (default localhost; bind 0.0.0.0 only behind auth)"),
|
|
51
|
+
port: int = typer.Option(8000, help="HTTP port"),
|
|
52
|
+
) -> None:
|
|
53
|
+
"""Start the LitLoop server."""
|
|
54
|
+
from ..deps import print_status
|
|
55
|
+
from ..log import get_logger
|
|
56
|
+
log = get_logger()
|
|
57
|
+
|
|
58
|
+
# Check dependencies before starting
|
|
59
|
+
print_status()
|
|
60
|
+
|
|
61
|
+
from ..server import mcp_app
|
|
62
|
+
|
|
63
|
+
if mode == ServerMode.STDIO:
|
|
64
|
+
log.info("Starting in stdio mode")
|
|
65
|
+
mcp_app.run(transport="stdio")
|
|
66
|
+
elif mode == ServerMode.WEB:
|
|
67
|
+
try:
|
|
68
|
+
import uvicorn
|
|
69
|
+
from .web import app as web_app
|
|
70
|
+
except ModuleNotFoundError as e:
|
|
71
|
+
typer.echo(f" Missing dependency: {e.name}. Install with: pip install 'litloop[web]'")
|
|
72
|
+
raise typer.Exit(1)
|
|
73
|
+
log.info(f"Starting web UI on http://{host}:{port}")
|
|
74
|
+
uvicorn.run(web_app, host=host, port=port)
|
|
75
|
+
else:
|
|
76
|
+
import uvicorn
|
|
77
|
+
log.info(f"Starting HTTP server on {host}:{port}")
|
|
78
|
+
asgi_app = mcp_app.streamable_http_app()
|
|
79
|
+
uvicorn.run(asgi_app, host=host, port=port)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@app.command("check")
|
|
83
|
+
def check_deps() -> None:
|
|
84
|
+
"""Check dependency status."""
|
|
85
|
+
from ..deps import print_status
|
|
86
|
+
print_status()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@app.command("diagnose")
|
|
90
|
+
def diagnose(
|
|
91
|
+
out: str = typer.Option("", "--out", "-o", help="Where to write the zip (default: ~/.litloop/exports/)"),
|
|
92
|
+
) -> None:
|
|
93
|
+
"""Write a diagnostic bundle: version, system, settings (keys masked), recent log."""
|
|
94
|
+
from ..diagnostics import write_bundle
|
|
95
|
+
path = write_bundle(Path(out) if out else None)
|
|
96
|
+
print(f" {path}")
|
|
97
|
+
print(" 设置里的密钥已经打码;包里没有 PDF、没有 Zotero 数据。")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@app.command("web")
|
|
101
|
+
def web_server(
|
|
102
|
+
# It downloads with the user's school session and has no login: never on
|
|
103
|
+
# every interface by default (2026-10-23 review).
|
|
104
|
+
host: str = typer.Option("127.0.0.1", help="Web server host (default localhost; bind 0.0.0.0 only behind auth)"),
|
|
105
|
+
port: int = typer.Option(8080, help="Web server port"),
|
|
106
|
+
) -> None:
|
|
107
|
+
"""Start the LitLoop web UI for browser-based paper downloading."""
|
|
108
|
+
try:
|
|
109
|
+
import uvicorn
|
|
110
|
+
from .web import app as web_app
|
|
111
|
+
except ModuleNotFoundError as e:
|
|
112
|
+
typer.echo(f" Missing dependency: {e.name}. Install with: pip install 'litloop[web]'")
|
|
113
|
+
raise typer.Exit(1)
|
|
114
|
+
print(f" Starting LitLoop Web UI on http://{host}:{port}")
|
|
115
|
+
print(f" Open http://localhost:{port} in your browser")
|
|
116
|
+
uvicorn.run(web_app, host=host, port=port)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@app.command("login")
|
|
120
|
+
def login(
|
|
121
|
+
login_type: str = typer.Option("cookies", help="Login type: cookies, webvpn, carsi, ezproxy, custom"),
|
|
122
|
+
url: str = typer.Option("", help="URL to open (for cookies/custom type)"),
|
|
123
|
+
) -> None:
|
|
124
|
+
"""Log in to your institution via stealth browser. Cookies are saved for all future downloads."""
|
|
125
|
+
from ..config import load_config
|
|
126
|
+
config = load_config()
|
|
127
|
+
|
|
128
|
+
if login_type == "cookies":
|
|
129
|
+
from ..browser.cookies import extract_via_browser
|
|
130
|
+
target_url = url or "https://www.sciencedirect.com/"
|
|
131
|
+
result = extract_via_browser(config, url=target_url)
|
|
132
|
+
if result["success"]:
|
|
133
|
+
print(f" {result['message']}")
|
|
134
|
+
else:
|
|
135
|
+
print(f" {result.get('message') or result.get('error', 'Failed')}")
|
|
136
|
+
raise typer.Exit(1)
|
|
137
|
+
elif login_type == "webvpn":
|
|
138
|
+
from ..browser.login import webvpn_login
|
|
139
|
+
success = webvpn_login(config)
|
|
140
|
+
raise typer.Exit(0 if success else 1)
|
|
141
|
+
elif login_type == "ezproxy":
|
|
142
|
+
from ..browser.login import ezproxy_login
|
|
143
|
+
success = ezproxy_login(config)
|
|
144
|
+
raise typer.Exit(0 if success else 1)
|
|
145
|
+
elif login_type == "custom":
|
|
146
|
+
if not url:
|
|
147
|
+
print(" Error: --url is required for login_type=custom")
|
|
148
|
+
raise typer.Exit(1)
|
|
149
|
+
from ..browser.login import open_login_browser
|
|
150
|
+
from ..config import DATA_DIR
|
|
151
|
+
cookie_file = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "custom_cookies.json"
|
|
152
|
+
success = open_login_browser(url, config, cookie_file=cookie_file)
|
|
153
|
+
raise typer.Exit(0 if success else 1)
|
|
154
|
+
else:
|
|
155
|
+
print(f" Unknown login type: {login_type}")
|
|
156
|
+
raise typer.Exit(1)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@app.command("get")
|
|
160
|
+
def get_paper(
|
|
161
|
+
identifier: str = typer.Argument(help="DOI or arXiv ID"),
|
|
162
|
+
output: str = typer.Option("", help="Output directory (default: output_dir in config, ~/.litloop/papers)"),
|
|
163
|
+
no_bibtex: bool = typer.Option(False, help="Skip BibTeX citation"),
|
|
164
|
+
strategy: str = typer.Option("", help="Override download strategy: fastest, grey_only(all 3 grey sources), scihub_only(Sci-Hub only), scihub_first, oa_first, legal_only"),
|
|
165
|
+
si: bool = typer.Option(False, "--si", help="Also fetch Supplementary Information attachments"),
|
|
166
|
+
md: bool = typer.Option(False, "--md", help="Also export the PDF as agent-ready markdown"),
|
|
167
|
+
) -> None:
|
|
168
|
+
"""Download a paper with zero configuration. Just give a DOI."""
|
|
169
|
+
output = _default_output_dir(output)
|
|
170
|
+
from ..sources import download
|
|
171
|
+
from ..config import load_config, update_config
|
|
172
|
+
|
|
173
|
+
result = download(
|
|
174
|
+
identifier, output,
|
|
175
|
+
scihub_enabled=None, use_tor=True, use_vpnsci=True, # None: follow scihub_enabled in the config
|
|
176
|
+
bibtex=not no_bibtex,
|
|
177
|
+
strategy=strategy if strategy else None,
|
|
178
|
+
)
|
|
179
|
+
if result.get("success"):
|
|
180
|
+
print(f" OK: {result.get('file', '')}")
|
|
181
|
+
print(f" Source: {result.get('source', '?')}")
|
|
182
|
+
if md:
|
|
183
|
+
from ..download.markdown import pdf_to_markdown
|
|
184
|
+
|
|
185
|
+
md_path = pdf_to_markdown(result["file"])
|
|
186
|
+
print(f" Markdown: {md_path}")
|
|
187
|
+
if si:
|
|
188
|
+
from ..publishers.supplementary import fetch_supplementary
|
|
189
|
+
|
|
190
|
+
out_dir = str(Path(result.get("file", output)).parent) if result.get("file") else output
|
|
191
|
+
si_files = fetch_supplementary(identifier, out_dir, load_config())
|
|
192
|
+
print(f" SI files: {len(si_files)}")
|
|
193
|
+
for f in si_files:
|
|
194
|
+
print(f" {f}")
|
|
195
|
+
else:
|
|
196
|
+
reason = (result.get("reason") or result.get("error")
|
|
197
|
+
or result.get("error_type") or "unknown")
|
|
198
|
+
print(f" FAILED: {reason}")
|
|
199
|
+
source_failures = result.get("source_failures") or []
|
|
200
|
+
if source_failures:
|
|
201
|
+
print(f" 渠道明细 ({len(source_failures)} 个渠道失败,部分可能只是临时不可用):")
|
|
202
|
+
for f in source_failures:
|
|
203
|
+
note = f" — {f['reason']}" if f.get("reason") else ""
|
|
204
|
+
print(f" {f.get('source', '?')}: {f.get('error_type') or 'failed'}{note}")
|
|
205
|
+
hint = result.get('agent_hint', '')
|
|
206
|
+
if hint:
|
|
207
|
+
print(f" Hint: {hint}")
|
|
208
|
+
else:
|
|
209
|
+
print(f" Hint: 运行 litloop login 配置机构代理,或检查网络连接")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@app.command("browser-status")
|
|
213
|
+
def browser_status() -> None:
|
|
214
|
+
"""Check browser backend availability and which browser kernel is in effect."""
|
|
215
|
+
from ..config import load_config
|
|
216
|
+
from ..browser.engine import is_available
|
|
217
|
+
from ..browser.backend import browser_info
|
|
218
|
+
import importlib.metadata as _md
|
|
219
|
+
|
|
220
|
+
config = load_config()
|
|
221
|
+
available = is_available(config)
|
|
222
|
+
print(f" Browser backend: {'available' if available else 'not installed'}")
|
|
223
|
+
if not available:
|
|
224
|
+
return
|
|
225
|
+
info = browser_info(config)
|
|
226
|
+
print(f" backend: {info.get('backend', '?')}")
|
|
227
|
+
for pkg in ("patchright", "cloakbrowser"):
|
|
228
|
+
try:
|
|
229
|
+
print(f" {pkg} package: {_md.version(pkg)}")
|
|
230
|
+
except Exception:
|
|
231
|
+
pass
|
|
232
|
+
if info.get("binary"):
|
|
233
|
+
print(f" browser kernel: {info['binary']} (version {info.get('version') or '?'})")
|
|
234
|
+
else:
|
|
235
|
+
print(f" browser kernel: {info.get('binary') or '?'}")
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@app.command("browser-doctor")
|
|
239
|
+
def browser_doctor_cmd() -> None:
|
|
240
|
+
"""Report reusable shared browser runtime options without installing anything."""
|
|
241
|
+
import json as _json
|
|
242
|
+
|
|
243
|
+
from ..browser.runtime import doctor
|
|
244
|
+
|
|
245
|
+
print(_json.dumps(doctor(), ensure_ascii=False))
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@app.command("import-cookies")
|
|
249
|
+
def import_cookies_cmd(cookie_file: str = typer.Argument(help="Netscape-format cookie file path")) -> None:
|
|
250
|
+
"""Import Netscape cookies into browser context."""
|
|
251
|
+
from ..config import load_config
|
|
252
|
+
from ..browser.engine import import_cookies, is_available
|
|
253
|
+
|
|
254
|
+
config = load_config()
|
|
255
|
+
if not is_available(config):
|
|
256
|
+
print("Error: CloakBrowser not available")
|
|
257
|
+
raise typer.Exit(1)
|
|
258
|
+
try:
|
|
259
|
+
count = import_cookies(cookie_file, config)
|
|
260
|
+
print(f"Imported {count} cookies from {cookie_file}")
|
|
261
|
+
except Exception as exc:
|
|
262
|
+
print(f"Error: {exc}")
|
|
263
|
+
raise typer.Exit(1)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@app.command("coverage")
|
|
267
|
+
def coverage_report(
|
|
268
|
+
input_file: str = typer.Argument(help="File with one DOI per line"),
|
|
269
|
+
json_output: str = typer.Option("", help="Save JSON coverage report to file"),
|
|
270
|
+
no_browser: bool = typer.Option(False, help="Disable browser-based sources"),
|
|
271
|
+
) -> None:
|
|
272
|
+
"""Dry-run coverage audit: test DOI routing without downloading PDFs.
|
|
273
|
+
|
|
274
|
+
Reports which sources would be attempted for each DOI and how
|
|
275
|
+
publishers map to source tiers.
|
|
276
|
+
"""
|
|
277
|
+
from pathlib import Path
|
|
278
|
+
from ..config import load_config
|
|
279
|
+
from ..sources.publishers import get_publisher, get_publisher_fast_sources, DOI_PREFIX_TO_PUBLISHER
|
|
280
|
+
|
|
281
|
+
config = load_config()
|
|
282
|
+
if no_browser:
|
|
283
|
+
config["browser_headless"] = True
|
|
284
|
+
config["vpnsci_enabled"] = False
|
|
285
|
+
config["carsi_enabled"] = False
|
|
286
|
+
|
|
287
|
+
dois = [line.strip() for line in Path(input_file).read_text(encoding="utf-8").splitlines() if line.strip() and not line.startswith("#")]
|
|
288
|
+
|
|
289
|
+
by_publisher: dict[str, dict[str, int]] = {}
|
|
290
|
+
items: list[dict] = []
|
|
291
|
+
|
|
292
|
+
for doi in dois:
|
|
293
|
+
publisher = get_publisher(doi) or "Unknown"
|
|
294
|
+
sources = get_publisher_fast_sources(doi)
|
|
295
|
+
source_names = [name for _, name in sources]
|
|
296
|
+
|
|
297
|
+
tier_info = {"doi": doi, "publisher": publisher, "sources": source_names}
|
|
298
|
+
|
|
299
|
+
if not publisher:
|
|
300
|
+
tier_info["status"] = "not_routed"
|
|
301
|
+
tier_info["action"] = f"No publisher mapping for DOI prefix. Add to DOI_PREFIX_TO_PUBLISHER."
|
|
302
|
+
elif not sources:
|
|
303
|
+
tier_info["status"] = "no_sources"
|
|
304
|
+
tier_info["action"] = f"Publisher '{publisher}' has no registered sources."
|
|
305
|
+
else:
|
|
306
|
+
tier_info["status"] = "routed"
|
|
307
|
+
|
|
308
|
+
items.append(tier_info)
|
|
309
|
+
|
|
310
|
+
if publisher not in by_publisher:
|
|
311
|
+
by_publisher[publisher] = {"count": 0, "routed": 0, "not_routed": 0}
|
|
312
|
+
by_publisher[publisher]["count"] += 1
|
|
313
|
+
if tier_info["status"] == "routed":
|
|
314
|
+
by_publisher[publisher]["routed"] += 1
|
|
315
|
+
else:
|
|
316
|
+
by_publisher[publisher]["not_routed"] += 1
|
|
317
|
+
|
|
318
|
+
report = {
|
|
319
|
+
"total": len(dois),
|
|
320
|
+
"by_publisher": by_publisher,
|
|
321
|
+
"items": items,
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
# Print summary
|
|
325
|
+
print(f"\n Coverage Report: {len(dois)} DOIs")
|
|
326
|
+
print(f" {'='*50}")
|
|
327
|
+
for pub, stats in sorted(by_publisher.items(), key=lambda x: -x[1]["count"]):
|
|
328
|
+
pct = stats["routed"] / stats["count"] * 100 if stats["count"] else 0
|
|
329
|
+
print(f" {pub:25s} {stats['count']:3d} DOIs {stats['routed']:3d} routed {stats['not_routed']:3d} gaps ({pct:.0f}%)")
|
|
330
|
+
|
|
331
|
+
routed = sum(s["routed"] for s in by_publisher.values())
|
|
332
|
+
not_routed = sum(s["not_routed"] for s in by_publisher.values())
|
|
333
|
+
print(f" {'='*50}")
|
|
334
|
+
print(f" {'TOTAL':25s} {len(dois):3d} DOIs {routed:3d} routed {not_routed:3d} gaps ({routed/len(dois)*100:.0f}%)\n")
|
|
335
|
+
|
|
336
|
+
if not_routed > 0:
|
|
337
|
+
print(" Unrouted DOIs (no publisher mapping):")
|
|
338
|
+
for item in items:
|
|
339
|
+
if item["status"] != "routed":
|
|
340
|
+
print(f" {item['doi']:45s} {item['action']}")
|
|
341
|
+
print()
|
|
342
|
+
|
|
343
|
+
if json_output:
|
|
344
|
+
import json as _json
|
|
345
|
+
Path(json_output).write_text(_json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
346
|
+
print(f" JSON report saved to: {json_output}")
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
# ── Institutional access commands ─────────────────────────────────────────────
|
|
350
|
+
|
|
351
|
+
@app.command("setup")
|
|
352
|
+
def setup_school(
|
|
353
|
+
school: str = typer.Argument("", help="School name to configure"),
|
|
354
|
+
show: bool = typer.Option(False, "--show", help="Show current configuration"),
|
|
355
|
+
) -> None:
|
|
356
|
+
"""Configure institutional access (WebVPN/EZproxy/CARSI)."""
|
|
357
|
+
from ..config import load_config, save_config
|
|
358
|
+
|
|
359
|
+
config = load_config()
|
|
360
|
+
|
|
361
|
+
if show:
|
|
362
|
+
print(f" School: {config.get('vpnsci_school', '(not set)')}")
|
|
363
|
+
print(f" WebVPN base URL: {config.get('vpnsci_base_url', '(not set)')}")
|
|
364
|
+
print(f" EZproxy URL: {config.get('ezproxy_login_url', '(not set)')}")
|
|
365
|
+
print(f" CARSI enabled: {config.get('carsi_enabled', False)}")
|
|
366
|
+
print(f" CARSI IdP: {config.get('carsi_idp_name', '(not set)')}")
|
|
367
|
+
print(f" Elsevier API key: {'set' if config.get('elsevier_api_key') else '(not set)'}")
|
|
368
|
+
print(f" Elsevier inst: {'set' if config.get('elsevier_insttoken') else '(not set)'}")
|
|
369
|
+
print(f" Proxy: {config.get('network_proxy', '(not set)')}")
|
|
370
|
+
return
|
|
371
|
+
|
|
372
|
+
if not school:
|
|
373
|
+
from ..institutional.schools import list_schools
|
|
374
|
+
schools = list_schools()
|
|
375
|
+
print(f" Available schools ({len(schools)} total):\n")
|
|
376
|
+
for s in schools[:30]:
|
|
377
|
+
print(f" {s.name:30s} [{s.school_type}] {s.host}")
|
|
378
|
+
if len(schools) > 30:
|
|
379
|
+
print(f"\n ... and {len(schools) - 30} more. Use: litloop schools <query>")
|
|
380
|
+
return
|
|
381
|
+
|
|
382
|
+
from ..institutional.schools import search_schools
|
|
383
|
+
matches = search_schools(school)
|
|
384
|
+
if not matches:
|
|
385
|
+
print(f" No school matching '{school}' found.")
|
|
386
|
+
print(f" Run 'litloop setup' to list available schools.")
|
|
387
|
+
raise typer.Exit(1)
|
|
388
|
+
|
|
389
|
+
chosen = matches[0]
|
|
390
|
+
config["vpnsci_school"] = chosen.name
|
|
391
|
+
config["vpnsci_base_url"] = chosen.host
|
|
392
|
+
save_config(config)
|
|
393
|
+
print(f" Configured: {chosen.name}")
|
|
394
|
+
print(f" Type: {chosen.school_type}")
|
|
395
|
+
print(f" Gateway: {chosen.host}")
|
|
396
|
+
if len(matches) > 1:
|
|
397
|
+
others = ", ".join(m.name for m in matches[1:5])
|
|
398
|
+
print(f" Other matches: {others}")
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
@app.command("schools")
|
|
402
|
+
def list_schools_cmd(
|
|
403
|
+
query: str = typer.Argument("", help="Search query (name, province, or host)"),
|
|
404
|
+
) -> None:
|
|
405
|
+
"""List or search available schools/institutions."""
|
|
406
|
+
from ..institutional.schools import list_schools, search_schools
|
|
407
|
+
|
|
408
|
+
if query:
|
|
409
|
+
results = search_schools(query)
|
|
410
|
+
if not results:
|
|
411
|
+
print(f" No schools matching '{query}'.")
|
|
412
|
+
return
|
|
413
|
+
print(f" Found {len(results)} school(s):\n")
|
|
414
|
+
for s in results:
|
|
415
|
+
print(f" {s.name:30s} [{s.school_type}] {s.host}")
|
|
416
|
+
else:
|
|
417
|
+
schools = list_schools()
|
|
418
|
+
print(f" Available schools ({len(schools)} total):\n")
|
|
419
|
+
for s in schools:
|
|
420
|
+
print(f" {s.name:30s} [{s.school_type}] {s.host}")
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
@app.command("fetch")
|
|
424
|
+
def fetch_paper_cmd(
|
|
425
|
+
identifier: str = typer.Argument(help="DOI or article URL"),
|
|
426
|
+
output: str = typer.Option("", help="Output directory (default: output_dir in config, ~/.litloop/papers)"),
|
|
427
|
+
format: str = typer.Option("markdown", help="Output format: markdown, json, text"),
|
|
428
|
+
no_cache: bool = typer.Option(False, "--no-cache", help="Skip cache"),
|
|
429
|
+
) -> None:
|
|
430
|
+
"""Fetch a paper via 7-step institutional cascade.
|
|
431
|
+
|
|
432
|
+
Cascade: cache → OA → Elsevier API → DOI resolve → CARSI → publisher → browser → gateway.
|
|
433
|
+
"""
|
|
434
|
+
output = _default_output_dir(output)
|
|
435
|
+
from ..institutional.config_adapter import ConfigAdapter
|
|
436
|
+
from ..download.fetcher import PaperFetcher
|
|
437
|
+
|
|
438
|
+
config = ConfigAdapter.load()
|
|
439
|
+
config._config["output_dir"] = output
|
|
440
|
+
|
|
441
|
+
fetcher = PaperFetcher(config)
|
|
442
|
+
result = fetcher.fetch_with_result(identifier, use_cache=not no_cache)
|
|
443
|
+
|
|
444
|
+
if format == "json":
|
|
445
|
+
print(result.to_json())
|
|
446
|
+
elif format == "text":
|
|
447
|
+
print(result.to_text())
|
|
448
|
+
else:
|
|
449
|
+
print(result.to_markdown(include_pdf_path=True))
|
|
450
|
+
|
|
451
|
+
fetcher.close()
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _atomic_write_json(path: Path, data: Any) -> None:
|
|
455
|
+
"""Write JSON atomically (temp file then replace) so a crash mid-write
|
|
456
|
+
never leaves a truncated batch report."""
|
|
457
|
+
tmp = path.with_suffix(str(path.suffix) + ".tmp")
|
|
458
|
+
tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
459
|
+
tmp.replace(path)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _write_retry_file(output: str, results: list) -> None:
|
|
463
|
+
"""Persist failed identifiers as retry.txt so 'batch --retry' can pick them up."""
|
|
464
|
+
try:
|
|
465
|
+
from ..download.pipeline import collect_failures
|
|
466
|
+
|
|
467
|
+
failed = collect_failures(results)
|
|
468
|
+
if failed:
|
|
469
|
+
p = Path(output) / "retry.txt"
|
|
470
|
+
p.write_text("\n".join(failed) + "\n", encoding="utf-8")
|
|
471
|
+
print(f" {len(failed)} failed → retry list: {p}")
|
|
472
|
+
except Exception:
|
|
473
|
+
pass
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
@app.command("batch")
|
|
477
|
+
def batch_fetch_cmd(
|
|
478
|
+
input_file: str = typer.Argument(help="File with one DOI/URL per line, a queue TSV, a csv/xlsx table, APA or BibTeX"),
|
|
479
|
+
output: str = typer.Option("", help="Output directory (default: output_dir in config, ~/.litloop/papers)"),
|
|
480
|
+
format: str = typer.Option("json", help="Output format: json, text"),
|
|
481
|
+
scihub: bool = typer.Option(False, "--scihub", help="Use Sci-Hub racing engine (includes grey sources) instead of institutional cascade"),
|
|
482
|
+
runs_dir: str = typer.Option("", "--runs-dir", help="ScanSci Find run directory: use its download queue as input and its preprint arXiv IDs as fallbacks for failed DOIs"),
|
|
483
|
+
lanes: bool = typer.Option(True, "--lanes/--no-lanes", help="Channel-lane scheduling (default on): pretriage + Elsevier API/OA/MDPI fast lane -> grey racing -> institutional cascade. --no-lanes falls back to per-item racing"),
|
|
484
|
+
retry: str = typer.Option("", "--retry", help="Retry failed identifiers from a previous batch_results.json"),
|
|
485
|
+
) -> None:
|
|
486
|
+
"""Batch fetch papers. Default: institutional cascade. Use --scihub for grey-source racing."""
|
|
487
|
+
output = _default_output_dir(output)
|
|
488
|
+
import json as _json
|
|
489
|
+
from ..config import load_config as _load_config
|
|
490
|
+
|
|
491
|
+
fallbacks: dict[str, list[str]] | None = None
|
|
492
|
+
entries = None
|
|
493
|
+
if retry:
|
|
494
|
+
from ..download.pipeline import collect_failures
|
|
495
|
+
|
|
496
|
+
prev = _json.loads(Path(retry).read_text(encoding="utf-8"))
|
|
497
|
+
dois = collect_failures(prev)
|
|
498
|
+
print(f" Retrying {len(dois)} failed identifiers from {retry}")
|
|
499
|
+
elif runs_dir:
|
|
500
|
+
from ..search.scansci_find import build_download_queue, build_preprint_fallbacks
|
|
501
|
+
run_identifiers = build_download_queue(runs_dir)
|
|
502
|
+
fallbacks = build_preprint_fallbacks(runs_dir) or None
|
|
503
|
+
if not run_identifiers:
|
|
504
|
+
print(f" No identifiers found in {runs_dir}/download_queue.json")
|
|
505
|
+
return
|
|
506
|
+
dois = run_identifiers
|
|
507
|
+
print(f" Using ScanSci Find queue from {runs_dir}: {len(dois)} identifiers")
|
|
508
|
+
else:
|
|
509
|
+
from ..download.pipeline import load_job
|
|
510
|
+
|
|
511
|
+
entries = load_job(input_file)
|
|
512
|
+
dois = [e.identifier for e in entries if e.identifier and not e.unresolved]
|
|
513
|
+
unresolved = sum(1 for e in entries if e.unresolved)
|
|
514
|
+
if unresolved:
|
|
515
|
+
print(f" {unresolved} 行无法识别为 DOI/arXiv,已跳过(可用标题检索补全后重试)")
|
|
516
|
+
if not dois:
|
|
517
|
+
print(" No DOIs/URLs found in input file.")
|
|
518
|
+
return
|
|
519
|
+
|
|
520
|
+
# Channel-lane scheduling (default; --no-lanes falls back to per-item
|
|
521
|
+
# racing). Grey-oriented strategies (scihub_only/grey_only/scihub_first)
|
|
522
|
+
# express a lane-ORDER preference the fast->grey->institutional schedule
|
|
523
|
+
# would invert, and an explicit --scihub asks for the racing engine —
|
|
524
|
+
# both stay on racing.
|
|
525
|
+
from ..download.pipeline import collect_failures, grey_allowed, run_lanes
|
|
526
|
+
|
|
527
|
+
_cfg = _load_config()
|
|
528
|
+
_grey_oriented = (
|
|
529
|
+
scihub
|
|
530
|
+
or _cfg.get("download_strategy", "fastest") in ("scihub_only", "grey_only", "scihub_first")
|
|
531
|
+
)
|
|
532
|
+
use_lanes = lanes and entries is not None and not _grey_oriented
|
|
533
|
+
if use_lanes:
|
|
534
|
+
cfg = _load_config()
|
|
535
|
+
try:
|
|
536
|
+
# Lane scheduling only changes the schedule, never the source
|
|
537
|
+
# authorization: allow_grey is derived from the user's config.
|
|
538
|
+
results = run_lanes(entries, output, config=cfg, allow_grey=grey_allowed(cfg))
|
|
539
|
+
except Exception as exc:
|
|
540
|
+
print(f"\n Lane scheduling error: {exc}")
|
|
541
|
+
results = [
|
|
542
|
+
{"doi": e.identifier, "success": False, "error": f"lane error: {exc}"}
|
|
543
|
+
for e in entries if not e.unresolved
|
|
544
|
+
]
|
|
545
|
+
partial_path = Path(output) / "batch_results.partial.json"
|
|
546
|
+
_atomic_write_json(partial_path, results)
|
|
547
|
+
print(f" Partial results saved to: {partial_path}")
|
|
548
|
+
return
|
|
549
|
+
ok = sum(1 for r in results if r.get("success"))
|
|
550
|
+
print(f"\n Lane results: {ok}/{len(results)} succeeded")
|
|
551
|
+
if format == "json":
|
|
552
|
+
_write_retry_file(output, results)
|
|
553
|
+
out_path = Path(output) / "batch_results.json"
|
|
554
|
+
_atomic_write_json(out_path, results)
|
|
555
|
+
print(f" Results saved to: {out_path}")
|
|
556
|
+
return
|
|
557
|
+
|
|
558
|
+
# Auto-detect: if download_strategy is grey/scihub-oriented, switch to racing engine
|
|
559
|
+
_cfg = _load_config()
|
|
560
|
+
_strategy = _cfg.get("download_strategy", "fastest")
|
|
561
|
+
# A grey-oriented strategy only switches engines when the user also allowed
|
|
562
|
+
# grey sources; scihub_enabled=false is a veto (download.pipeline.grey_allowed).
|
|
563
|
+
# --scihub on this command is itself the user's explicit choice.
|
|
564
|
+
_auto_scihub = scihub or (_strategy in ("scihub_only", "grey_only", "scihub_first")
|
|
565
|
+
and bool(_cfg.get("scihub_enabled", False)))
|
|
566
|
+
if not scihub and _auto_scihub:
|
|
567
|
+
print(f" Auto-switching to Sci-Hub racing engine (download_strategy={_strategy})")
|
|
568
|
+
|
|
569
|
+
if _auto_scihub:
|
|
570
|
+
# Use the source-racing engine (includes Sci-Hub/SciBban/LibGen)
|
|
571
|
+
from ..sources import batch_download
|
|
572
|
+
results = batch_download(dois, output_dir=output, scihub_enabled=True, fallbacks=fallbacks)
|
|
573
|
+
# Verify file existence for each "success" result before writing report
|
|
574
|
+
_verify_batch_results(results, output)
|
|
575
|
+
if format == "json":
|
|
576
|
+
_write_retry_file(output, results)
|
|
577
|
+
out_path = Path(output) / "batch_results.json"
|
|
578
|
+
_atomic_write_json(out_path, results)
|
|
579
|
+
print(f"\n Results saved to: {out_path}")
|
|
580
|
+
return
|
|
581
|
+
|
|
582
|
+
# Default: institutional cascade (PaperFetcher)
|
|
583
|
+
from ..institutional.config_adapter import ConfigAdapter
|
|
584
|
+
from ..download.fetcher import PaperFetcher
|
|
585
|
+
|
|
586
|
+
config = ConfigAdapter.load()
|
|
587
|
+
config._config["output_dir"] = output
|
|
588
|
+
|
|
589
|
+
fetcher = PaperFetcher(config)
|
|
590
|
+
results = []
|
|
591
|
+
|
|
592
|
+
for i, doi in enumerate(dois, 1):
|
|
593
|
+
print(f" [{i}/{len(dois)}] {doi}")
|
|
594
|
+
try:
|
|
595
|
+
result = fetcher.fetch_with_result(doi)
|
|
596
|
+
result_dict = result.to_dict()
|
|
597
|
+
# Verify file actually exists on disk for success status
|
|
598
|
+
if result_dict.get("status") == "success" or result_dict.get("success"):
|
|
599
|
+
pdf_path = result_dict.get("file") or result_dict.get("pdf_path", "")
|
|
600
|
+
if pdf_path and not Path(pdf_path).exists():
|
|
601
|
+
result_dict["status"] = "error"
|
|
602
|
+
result_dict["error"] = "PDF file not found on disk (may have been saved elsewhere)"
|
|
603
|
+
results.append(result_dict)
|
|
604
|
+
status = result.status
|
|
605
|
+
quality = result.quality
|
|
606
|
+
print(f" → {status} ({quality})")
|
|
607
|
+
except Exception as e:
|
|
608
|
+
results.append({"doi": doi, "error": str(e)})
|
|
609
|
+
print(f" → error: {e}")
|
|
610
|
+
|
|
611
|
+
fetcher.close()
|
|
612
|
+
|
|
613
|
+
if format == "json":
|
|
614
|
+
out_path = Path(output) / "batch_results.json"
|
|
615
|
+
_atomic_write_json(out_path, results)
|
|
616
|
+
print(f"\n Results saved to: {out_path}")
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _verify_batch_results(results: dict, output_dir: str) -> None:
|
|
620
|
+
"""Verify that 'success' results have actual files on disk; fix stale entries."""
|
|
621
|
+
output_path = Path(output_dir)
|
|
622
|
+
for r in results.get("results", []):
|
|
623
|
+
if r.get("success"):
|
|
624
|
+
file_path = r.get("file", "")
|
|
625
|
+
if file_path and not Path(file_path).exists():
|
|
626
|
+
# Check if file exists under a different name in the output dir
|
|
627
|
+
doi = r.get("doi", "")
|
|
628
|
+
if doi:
|
|
629
|
+
from ..identifiers import safe_filename
|
|
630
|
+
safe = safe_filename(doi)
|
|
631
|
+
found = list(output_path.glob(f"{safe}*.pdf"))
|
|
632
|
+
if found:
|
|
633
|
+
r["file"] = str(found[0])
|
|
634
|
+
else:
|
|
635
|
+
r["success"] = False
|
|
636
|
+
r["error"] = "File missing from disk"
|
|
637
|
+
results["succeeded"] = max(0, results.get("succeeded", 1) - 1)
|
|
638
|
+
results["failed"] = results.get("failed", 0) + 1
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
@app.command("elsevier-setup")
|
|
642
|
+
def elsevier_setup(
|
|
643
|
+
api_key: str = typer.Option("", help="Elsevier API key"),
|
|
644
|
+
inst_token: str = typer.Option("", help="Elsevier institutional token(可选,通常无需配置:校园网 + API key 即可)"),
|
|
645
|
+
) -> None:
|
|
646
|
+
"""Configure Elsevier API access for direct full-text retrieval."""
|
|
647
|
+
from ..config import load_config, save_config
|
|
648
|
+
|
|
649
|
+
config = load_config()
|
|
650
|
+
changed = False
|
|
651
|
+
|
|
652
|
+
if api_key:
|
|
653
|
+
config["elsevier_api_key"] = api_key
|
|
654
|
+
changed = True
|
|
655
|
+
print(f" Elsevier API key: saved")
|
|
656
|
+
# 配置即验证:自检样本自动跑一遍,当场给出 key 画像
|
|
657
|
+
try:
|
|
658
|
+
from ..sources.elsevier_check import check_key_profile
|
|
659
|
+
|
|
660
|
+
_rows, profile, advice = check_key_profile(config)
|
|
661
|
+
print(f" key 画像: {profile}")
|
|
662
|
+
print(f" 建议: {advice}")
|
|
663
|
+
except Exception as e:
|
|
664
|
+
print(f" 自检失败(可稍后运行 elsevier-check): {e}")
|
|
665
|
+
if inst_token:
|
|
666
|
+
config["elsevier_insttoken"] = inst_token
|
|
667
|
+
changed = True
|
|
668
|
+
print(f" Elsevier inst token: saved")
|
|
669
|
+
|
|
670
|
+
if not changed:
|
|
671
|
+
has_key = bool(config.get("elsevier_api_key"))
|
|
672
|
+
has_token = bool(config.get("elsevier_insttoken"))
|
|
673
|
+
print(f" Elsevier API key: {'set' if has_key else '(not set)'}")
|
|
674
|
+
print(f" Elsevier inst token: {'set' if has_token else '(not set, 通常不需要)'}")
|
|
675
|
+
print(f"\n Usage: litloop elsevier-setup --api-key YOUR_KEY")
|
|
676
|
+
print(f"\n 申请 key:机构邮箱注册 Elsevier 账号,并在校园网/学校 VPN 下创建 key"
|
|
677
|
+
f"(门户靠注册时的邮箱域 + 出口 IP 识别机构,个人邮箱/校外创建绑不上机构)。")
|
|
678
|
+
print(f" 创建后当场跑 elsevier-check:无 key 基线 406、大刊样本 200 即绑定成功。")
|
|
679
|
+
print(f" 提示:insttoken 通常不需要——API key + 校园网/机构网络出口即可(仅拿不到校园出口时申请)。")
|
|
680
|
+
print(f" NOT_ENTITLED 表示未连校园网或学校未订阅该刊,不是缺 insttoken。")
|
|
681
|
+
print(f" 403 'Requestor configuration settings insufficient':出口不在机构注册段(配 network_proxy 走校园出口),"
|
|
682
|
+
f"或机构未给该 key 开 API 全文权限(联系机构 Elsevier 管理员)。")
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
@app.command("elsevier-check")
|
|
686
|
+
def elsevier_check_cmd(
|
|
687
|
+
doi: str = typer.Option("", "--doi", help="单篇 DOI 逐篇双路由探测"),
|
|
688
|
+
file: str = typer.Option("", "--file", help="DOI 清单(每行一个,逐篇探测)"),
|
|
689
|
+
json_out: bool = typer.Option(False, "--json", help="输出机读 JSON"),
|
|
690
|
+
) -> None:
|
|
691
|
+
"""Elsevier API key 权限自检与逐篇探测(纯查询,零下载)。"""
|
|
692
|
+
from rich.console import Console
|
|
693
|
+
from rich.table import Table
|
|
694
|
+
|
|
695
|
+
console = Console()
|
|
696
|
+
|
|
697
|
+
from ..download import progress as _pr
|
|
698
|
+
from ..config import load_config
|
|
699
|
+
from ..sources.elsevier_check import check_dois, check_key_profile
|
|
700
|
+
|
|
701
|
+
config = load_config()
|
|
702
|
+
console.print("[bold]Elsevier key 权限自检[/bold]")
|
|
703
|
+
profile_rows, profile, advice = check_key_profile(config)
|
|
704
|
+
t = Table(title="key 自检样本")
|
|
705
|
+
for col in ("DOI", "样本类型", "代理路由", "直连路由", "判定"):
|
|
706
|
+
t.add_column(col)
|
|
707
|
+
for r in profile_rows:
|
|
708
|
+
t.add_row(r["doi"][:34], r["sample_type"], r["proxy"], r["direct"], r["verdict"])
|
|
709
|
+
console.print(t)
|
|
710
|
+
console.print(f"[bold]key 画像:[/bold] {profile} — {advice}")
|
|
711
|
+
|
|
712
|
+
dois: list[str] = []
|
|
713
|
+
if doi:
|
|
714
|
+
dois.append(doi.strip())
|
|
715
|
+
elif file:
|
|
716
|
+
from pathlib import Path as _Path
|
|
717
|
+
|
|
718
|
+
p = _Path(file)
|
|
719
|
+
if p.exists():
|
|
720
|
+
dois = [l.strip() for l in p.read_text(encoding="utf-8").splitlines() if l.strip()]
|
|
721
|
+
else:
|
|
722
|
+
console.print(f"[red]清单不存在: {file}[/red]")
|
|
723
|
+
if dois:
|
|
724
|
+
_pr.start_task("Elsevier权限检测", total=len(dois))
|
|
725
|
+
rows = check_dois(dois, config, progress=_pr)
|
|
726
|
+
_pr.finish()
|
|
727
|
+
t2 = Table(title="用户样本双路由探测")
|
|
728
|
+
for col in ("DOI", "代理路由", "直连路由", "判定", "建议通道"):
|
|
729
|
+
t2.add_column(col)
|
|
730
|
+
for r in rows:
|
|
731
|
+
t2.add_row(r["doi"][:34], r["proxy"], r["direct"], r["verdict"], r["route_advice"])
|
|
732
|
+
console.print(t2)
|
|
733
|
+
if json_out:
|
|
734
|
+
import json as _json
|
|
735
|
+
|
|
736
|
+
console.print(_json.dumps({"profile": profile, "advice": advice,
|
|
737
|
+
"rows": profile_rows + rows},
|
|
738
|
+
ensure_ascii=False, indent=2))
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
@app.command("session-doctor")
|
|
742
|
+
def session_doctor() -> None:
|
|
743
|
+
"""Diagnose browser profile sessions and cookie health."""
|
|
744
|
+
from ..institutional.config_adapter import ConfigAdapter
|
|
745
|
+
from ..browser.profile_health import candidate_profile_dirs, inspect_browser_profile
|
|
746
|
+
|
|
747
|
+
config = ConfigAdapter.load()
|
|
748
|
+
profiles = candidate_profile_dirs(config.chrome_profile_dir)
|
|
749
|
+
|
|
750
|
+
domains = [
|
|
751
|
+
"sciencedirect.com", "springer.com", "nature.com", "wiley.com",
|
|
752
|
+
"acs.org", "rsc.org", "ieeexplore.ieee.org", "openathens.net",
|
|
753
|
+
]
|
|
754
|
+
|
|
755
|
+
print(" Browser Profile Diagnostics\n")
|
|
756
|
+
for profile_dir in profiles:
|
|
757
|
+
report = inspect_browser_profile(profile_dir, domains)
|
|
758
|
+
exists = report["exists"]
|
|
759
|
+
cookies_db = report["cookies_db_exists"]
|
|
760
|
+
print(f" Profile: {report['profile_dir']}")
|
|
761
|
+
print(f" Exists: {exists}")
|
|
762
|
+
print(f" Cookies DB: {cookies_db}")
|
|
763
|
+
|
|
764
|
+
if report["error"]:
|
|
765
|
+
print(f" Error: {report['error']}")
|
|
766
|
+
|
|
767
|
+
for domain, info in report.get("domains", {}).items():
|
|
768
|
+
total = info["cookie_count"]
|
|
769
|
+
if total > 0:
|
|
770
|
+
print(f" {domain:25s} {total:3d} cookies (session={info['session_cookie_count']}, persistent={info['persistent_cookie_count']}, expired={info['expired_cookie_count']})")
|
|
771
|
+
print()
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
@app.command("federated-login")
|
|
775
|
+
def federated_login(
|
|
776
|
+
publisher: str = typer.Argument(help="Publisher key (e.g. sciencedirect, springer, wiley)"),
|
|
777
|
+
force: bool = typer.Option(False, "--force", help="Force re-login"),
|
|
778
|
+
) -> None:
|
|
779
|
+
"""Log in to a publisher via CARSI/Shibboleth federation."""
|
|
780
|
+
from ..institutional.carsi_login import CARSIClient
|
|
781
|
+
from ..config import load_config
|
|
782
|
+
|
|
783
|
+
config = load_config()
|
|
784
|
+
client = CARSIClient(config)
|
|
785
|
+
|
|
786
|
+
success = client.login(publisher, force=force)
|
|
787
|
+
if success:
|
|
788
|
+
print(f" Login successful for {publisher}.")
|
|
789
|
+
else:
|
|
790
|
+
print(f" Login failed for {publisher}.")
|
|
791
|
+
raise typer.Exit(1)
|
|
792
|
+
client.close()
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
@app.command("search")
|
|
796
|
+
def search_cmd(
|
|
797
|
+
query: str = typer.Argument("", help="Search query (keywords, author, title)"),
|
|
798
|
+
limit: int = typer.Option(10, help="Max results"),
|
|
799
|
+
year_from: int = typer.Option(None, help="Start year"),
|
|
800
|
+
year_to: int = typer.Option(None, help="End year"),
|
|
801
|
+
sort: str = typer.Option("", help="Sort: cited_by_count, publication_date"),
|
|
802
|
+
json_output: bool = typer.Option(True, help="Output as JSON (default)"),
|
|
803
|
+
author: str = typer.Option("", "--author", help="Search by author name (resolves to OpenAlex author ID)"),
|
|
804
|
+
author_id: str = typer.Option("", "--author-id", help="Search by OpenAlex author ID directly"),
|
|
805
|
+
out: Annotated[str, typer.Option("--out", help="Write results to a channel-annotated queue file (feed to 'batch --lanes')")] = "",
|
|
806
|
+
) -> None:
|
|
807
|
+
"""Search academic papers via OpenAlex, Semantic Scholar, and Crossref.
|
|
808
|
+
|
|
809
|
+
Results include DOI, title, authors, year, and citation count.
|
|
810
|
+
Use the DOIs with 'litloop get' or 'litloop batch' to download.
|
|
811
|
+
|
|
812
|
+
Examples:
|
|
813
|
+
litloop search "carbon cycle" --limit 10 --sort cited_by_count
|
|
814
|
+
litloop search --author "Fang Jingyun" --limit 10 --sort cited_by_count
|
|
815
|
+
litloop search --author-id A5102961214 --limit 10 --sort cited_by_count
|
|
816
|
+
"""
|
|
817
|
+
import json as _json
|
|
818
|
+
from ..search.quick import search_papers_v2 as search_papers
|
|
819
|
+
|
|
820
|
+
if author or author_id:
|
|
821
|
+
# Author-based search
|
|
822
|
+
results = search_papers(
|
|
823
|
+
limit=limit, year_from=year_from, year_to=year_to, sort=sort,
|
|
824
|
+
author=author if author else None,
|
|
825
|
+
author_id=author_id if author_id else None,
|
|
826
|
+
)
|
|
827
|
+
# Keep the resolved author match for the JSON payload; the human
|
|
828
|
+
# readable line is only shown in non-JSON mode.
|
|
829
|
+
author_match = None
|
|
830
|
+
if results and results[0].get("_author_match"):
|
|
831
|
+
author_match = results[0].pop("_author_match")
|
|
832
|
+
else:
|
|
833
|
+
sort_key = sort if sort else None
|
|
834
|
+
stats: dict = {}
|
|
835
|
+
results = search_papers(query, limit=limit, year_from=year_from, year_to=year_to,
|
|
836
|
+
sort=sort_key, stats=stats)
|
|
837
|
+
author_match = None
|
|
838
|
+
if not results:
|
|
839
|
+
_explain_empty_search(stats.get("errors") or [])
|
|
840
|
+
|
|
841
|
+
if out:
|
|
842
|
+
from ..download.pipeline import QueueEntry, predict_channel, write_queue
|
|
843
|
+
|
|
844
|
+
qe = [
|
|
845
|
+
QueueEntry(
|
|
846
|
+
identifier=r.get("doi") or r.get("arxiv_id") or "",
|
|
847
|
+
channel=predict_channel(r.get("doi") or ""),
|
|
848
|
+
title=str(r.get("title", "")),
|
|
849
|
+
)
|
|
850
|
+
for r in results
|
|
851
|
+
if r.get("doi") or r.get("arxiv_id")
|
|
852
|
+
]
|
|
853
|
+
p = write_queue(qe, out)
|
|
854
|
+
print(f" Queue written: {p} ({len(qe)} entries)")
|
|
855
|
+
|
|
856
|
+
if json_output:
|
|
857
|
+
payload = {"results": results}
|
|
858
|
+
if author_match is not None:
|
|
859
|
+
payload["author_match"] = author_match
|
|
860
|
+
print(_json.dumps(payload, indent=2, ensure_ascii=True))
|
|
861
|
+
else:
|
|
862
|
+
if author_match is not None:
|
|
863
|
+
print(f" Author: {author_match['name']} (ID:{author_match['id']}, works:{author_match['works_count']}, cited:{author_match['cited_by_count']})")
|
|
864
|
+
for i, r in enumerate(results, 1):
|
|
865
|
+
authors = ", ".join(r.get("authors", [])[:3] or [])
|
|
866
|
+
cited = r.get("cited_by_count", 0)
|
|
867
|
+
print(f"{i:2d}. {r.get('title', '?')[:80]}")
|
|
868
|
+
print(f" {authors} ({r.get('year', '?')}) cited={cited} doi:{r.get('doi', '?')}")
|
|
869
|
+
if i < len(results):
|
|
870
|
+
print()
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
def _explain_empty_search(errors: list[str]) -> None:
|
|
874
|
+
"""Say why nothing came back instead of printing an empty list."""
|
|
875
|
+
if not errors:
|
|
876
|
+
typer.echo("没有搜到结果:换个关键词,或者放宽年份再试。", err=True)
|
|
877
|
+
return
|
|
878
|
+
typer.echo("没有搜到结果:" + ";".join(errors), err=True)
|
|
879
|
+
if any("429" in e.lower() or "限流" in e or "rate" in e.lower() for e in errors):
|
|
880
|
+
typer.echo(
|
|
881
|
+
"OpenAlex 对不带 Key 的检索限流很紧:可以在 https://openalex.org/settings/api "
|
|
882
|
+
"申请一个免费 Key,再填到 app 的「设置 → 检索接口」或 config 里。",
|
|
883
|
+
err=True,
|
|
884
|
+
)
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
@app.command("plan")
|
|
888
|
+
def plan_cmd(
|
|
889
|
+
query: str = typer.Argument(help="Research topic"),
|
|
890
|
+
domain: str = typer.Option("general", "--domain", help="Domain profile (general/medicine/computer_science/chinese_general/...)"),
|
|
891
|
+
depth: str = typer.Option("standard", "--depth", help="Depth: quick/standard/systematic"),
|
|
892
|
+
question: str = typer.Option("", "--question", help="Research question"),
|
|
893
|
+
year_from: int = typer.Option(None, "--year-from"),
|
|
894
|
+
year_to: int = typer.Option(None, "--year-to"),
|
|
895
|
+
) -> None:
|
|
896
|
+
"""Build an auditable search protocol without running a search (ScanSci Find)."""
|
|
897
|
+
from ..search.scansci_find import find_cli_available, plan
|
|
898
|
+
if not find_cli_available():
|
|
899
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
900
|
+
raise typer.Exit(1)
|
|
901
|
+
print(json.dumps(plan(query, domain=domain, depth=depth, question=question,
|
|
902
|
+
year_from=year_from, year_to=year_to), ensure_ascii=False, indent=2))
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
@app.command("estimate")
|
|
906
|
+
def estimate_cmd(
|
|
907
|
+
query: str = typer.Argument(help="Research topic"),
|
|
908
|
+
domain: str = typer.Option("general", "--domain"),
|
|
909
|
+
depth: str = typer.Option("standard", "--depth"),
|
|
910
|
+
question: str = typer.Option("", "--question"),
|
|
911
|
+
year_from: int = typer.Option(None, "--year-from"),
|
|
912
|
+
year_to: int = typer.Option(None, "--year-to"),
|
|
913
|
+
) -> None:
|
|
914
|
+
"""Estimate result volume before spending a full search budget (ScanSci Find)."""
|
|
915
|
+
from ..search.scansci_find import find_cli_available, estimate
|
|
916
|
+
if not find_cli_available():
|
|
917
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
918
|
+
raise typer.Exit(1)
|
|
919
|
+
print(json.dumps(estimate(query, domain=domain, depth=depth, question=question,
|
|
920
|
+
year_from=year_from, year_to=year_to), ensure_ascii=False, indent=2))
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
@app.command("smoke")
|
|
924
|
+
def smoke_cmd(
|
|
925
|
+
query: str = typer.Argument(help="Research topic"),
|
|
926
|
+
domain: str = typer.Option("general", "--domain"),
|
|
927
|
+
records: int = typer.Option(4, "--records", help="Records per source (default 4)"),
|
|
928
|
+
) -> None:
|
|
929
|
+
"""Fetch a few records per source and validate the candidate contract (ScanSci Find)."""
|
|
930
|
+
from ..search.scansci_find import find_cli_available, smoke
|
|
931
|
+
if not find_cli_available():
|
|
932
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
933
|
+
raise typer.Exit(1)
|
|
934
|
+
print(json.dumps(smoke(query, domain=domain, records_per_source=records), ensure_ascii=False, indent=2))
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
@app.command("calibrate")
|
|
938
|
+
def calibrate_cmd(
|
|
939
|
+
query: str = typer.Argument(help="Research topic"),
|
|
940
|
+
domain: str = typer.Option("general", "--domain"),
|
|
941
|
+
depth: str = typer.Option("standard", "--depth"),
|
|
942
|
+
sample_size: int = typer.Option(100, "--sample-size"),
|
|
943
|
+
) -> None:
|
|
944
|
+
"""Run a bounded calibration sample before a high-recall search (ScanSci Find)."""
|
|
945
|
+
from ..search.scansci_find import find_cli_available, calibrate
|
|
946
|
+
if not find_cli_available():
|
|
947
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
948
|
+
raise typer.Exit(1)
|
|
949
|
+
print(json.dumps(calibrate(query, domain=domain, depth=depth, sample_size=sample_size), ensure_ascii=False, indent=2))
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
@app.command("verify")
|
|
953
|
+
def verify_cmd(
|
|
954
|
+
input_file: str = typer.Argument(help="Candidates JSON file (from discovery search)"),
|
|
955
|
+
limit: int = typer.Option(None, "--limit"),
|
|
956
|
+
) -> None:
|
|
957
|
+
"""Verify DOI/PMID/arXiv identifiers against authoritative APIs (ScanSci Find)."""
|
|
958
|
+
from ..search.scansci_find import find_cli_available, verify
|
|
959
|
+
if not find_cli_available():
|
|
960
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
961
|
+
raise typer.Exit(1)
|
|
962
|
+
print(json.dumps(verify(input_file, limit=limit), ensure_ascii=False, indent=2))
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
@app.command("resolve-oa")
|
|
966
|
+
def resolve_oa_cmd(
|
|
967
|
+
input_file: str = typer.Argument(help="Candidates JSON file (from discovery search)"),
|
|
968
|
+
limit: int = typer.Option(None, "--limit"),
|
|
969
|
+
) -> None:
|
|
970
|
+
"""Resolve DOI open-access locations through Unpaywall (ScanSci Find)."""
|
|
971
|
+
from ..search.scansci_find import find_cli_available, resolve_oa
|
|
972
|
+
if not find_cli_available():
|
|
973
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
974
|
+
raise typer.Exit(1)
|
|
975
|
+
print(json.dumps(resolve_oa(input_file, limit=limit), ensure_ascii=False, indent=2))
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
@app.command("build-queue")
|
|
979
|
+
def build_queue_cmd(
|
|
980
|
+
input_path: str = typer.Argument(help="Discovery output dir (with download_queue.json) or candidates JSON file"),
|
|
981
|
+
out: str = typer.Option("", "--out", help="Write identifier list to file (one per line)"),
|
|
982
|
+
) -> None:
|
|
983
|
+
"""Build a download identifier queue from ScanSci Find output — feed to 'litloop batch'."""
|
|
984
|
+
from pathlib import Path as _Path
|
|
985
|
+
from ..search.scansci_find import build_download_queue
|
|
986
|
+
import json as _json
|
|
987
|
+
|
|
988
|
+
p = _Path(input_path)
|
|
989
|
+
if p.is_dir():
|
|
990
|
+
identifiers = build_download_queue(p)
|
|
991
|
+
else:
|
|
992
|
+
try:
|
|
993
|
+
candidates = _json.loads(p.read_text(encoding="utf-8"))
|
|
994
|
+
except Exception:
|
|
995
|
+
print(f"Error: cannot read {input_path}")
|
|
996
|
+
raise typer.Exit(1)
|
|
997
|
+
if isinstance(candidates, dict):
|
|
998
|
+
candidates = candidates.get("candidates", [])
|
|
999
|
+
identifiers = []
|
|
1000
|
+
for c in candidates:
|
|
1001
|
+
for key in ("doi", "arxiv_id"):
|
|
1002
|
+
if c.get(key):
|
|
1003
|
+
identifiers.append(str(c[key]))
|
|
1004
|
+
break
|
|
1005
|
+
# Resolve any bare titles (no DOI/arXiv) through the built-in resolver
|
|
1006
|
+
unresolved = [i for i in identifiers if i]
|
|
1007
|
+
for i in unresolved:
|
|
1008
|
+
print(i)
|
|
1009
|
+
if out:
|
|
1010
|
+
_Path(out).write_text("\n".join(unresolved) + "\n", encoding="utf-8")
|
|
1011
|
+
print(f"\nWrote {len(unresolved)} identifiers to {out} (use: litloop batch {out})")
|
|
1012
|
+
else:
|
|
1013
|
+
print(f"\n{len(unresolved)} identifiers ready (use: litloop batch <file>)")
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
@app.command("find")
|
|
1017
|
+
def find_cmd(
|
|
1018
|
+
query: str = typer.Argument(help="Research topic"),
|
|
1019
|
+
out: str = typer.Option(..., "--out", help="Output directory for ScanSci Find artifacts"),
|
|
1020
|
+
domain: str = typer.Option("general", "--domain"),
|
|
1021
|
+
depth: str = typer.Option("quick", "--depth", help="quick/standard/systematic"),
|
|
1022
|
+
limit: int = typer.Option(20, "--limit"),
|
|
1023
|
+
expand_citations: bool = typer.Option(False, "--expand-citations"),
|
|
1024
|
+
citation_rounds: int = typer.Option(None, "--citation-rounds"),
|
|
1025
|
+
citation_source: str = typer.Option("semantic", "--citation-source"),
|
|
1026
|
+
verify_identifiers: bool = typer.Option(False, "--verify-identifiers"),
|
|
1027
|
+
resolve_oa: bool = typer.Option(False, "--resolve-oa"),
|
|
1028
|
+
find_preprints: bool = typer.Option(False, "--find-preprints", help="Find lawful preprint copies of paywalled candidates"),
|
|
1029
|
+
code_links: bool = typer.Option(False, "--code-links", help="Attach Papers with Code code-availability metadata"),
|
|
1030
|
+
sort: str = typer.Option("", "--sort", help="relevance | recency | citations"),
|
|
1031
|
+
year_from: int = typer.Option(None, "--year-from"),
|
|
1032
|
+
year_to: int = typer.Option(None, "--year-to"),
|
|
1033
|
+
) -> None:
|
|
1034
|
+
"""Full discovery search via ScanSci Find's 13-source engine.
|
|
1035
|
+
|
|
1036
|
+
Writes candidates.json / download_queue.json / coverage & PRISMA reports
|
|
1037
|
+
into --out. Follow up with 'litloop build-queue <dir>' then
|
|
1038
|
+
'litloop batch <queue file>' to download.
|
|
1039
|
+
"""
|
|
1040
|
+
from ..search.scansci_find import build_download_queue, find_cli_available, search
|
|
1041
|
+
if not find_cli_available():
|
|
1042
|
+
print("Error: scansci-find CLI not available. Install the scansci-find discovery engine (see README, 依赖与 Key)")
|
|
1043
|
+
raise typer.Exit(1)
|
|
1044
|
+
payload = search(
|
|
1045
|
+
query, out, domain=domain, depth=depth, limit=limit,
|
|
1046
|
+
expand_citations=expand_citations, citation_source=citation_source,
|
|
1047
|
+
citation_rounds=citation_rounds,
|
|
1048
|
+
verify_identifiers=verify_identifiers, resolve_oa=resolve_oa,
|
|
1049
|
+
find_preprints=find_preprints, code_links=code_links, sort=sort,
|
|
1050
|
+
year_from=year_from, year_to=year_to,
|
|
1051
|
+
)
|
|
1052
|
+
queue = build_download_queue(out)
|
|
1053
|
+
print(f" Query: {query}")
|
|
1054
|
+
print(f" Domain: {domain} | Depth: {depth} | Total candidates: {payload.get('total', 0)}")
|
|
1055
|
+
print(f" Download queue: {len(queue)} identifiers")
|
|
1056
|
+
print(f" Artifacts: {out}")
|
|
1057
|
+
if queue:
|
|
1058
|
+
print(f" Next: litloop build-queue {out} --out queue.txt && litloop batch queue.txt")
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
@app.command("manifest")
|
|
1062
|
+
def manifest_cmd(
|
|
1063
|
+
input_file: str = typer.Argument(help="oa_manifest.json produced by scansci-find"),
|
|
1064
|
+
output: str = typer.Option(".", help="Output directory"),
|
|
1065
|
+
scihub: bool = typer.Option(False, "--scihub", help="Allow grey sources for non-eligible entries"),
|
|
1066
|
+
use_tor: bool = typer.Option(False, "--use-tor"),
|
|
1067
|
+
use_vpnsci: bool = typer.Option(True, "--use-vpnsci", help="Enable WebVPN/institutional phase for needs_institution entries"),
|
|
1068
|
+
) -> None:
|
|
1069
|
+
"""Download per a ScanSci Find oa_manifest.
|
|
1070
|
+
|
|
1071
|
+
Eligible entries (open_pdf/preprint with a PDF URL) are fetched directly
|
|
1072
|
+
from the manifest URL; needs_institution entries go through the racing
|
|
1073
|
+
engine with the institutional phase enabled. Writes download_results.json
|
|
1074
|
+
so scansci-find reconcile can write results back into the candidate set.
|
|
1075
|
+
"""
|
|
1076
|
+
import json as _json
|
|
1077
|
+
from pathlib import Path as _Path
|
|
1078
|
+
|
|
1079
|
+
from ..identifiers import safe_filename
|
|
1080
|
+
from ..download.pdf import download_pdf, is_pdf_file
|
|
1081
|
+
from ..sources import _update_doi_index, download
|
|
1082
|
+
|
|
1083
|
+
payload = _json.loads(_Path(input_file).read_text(encoding="utf-8"))
|
|
1084
|
+
entries = payload.get("entries") if isinstance(payload, dict) else payload
|
|
1085
|
+
if not isinstance(entries, list):
|
|
1086
|
+
print(" Invalid manifest: expected a list of entries or {'entries': [...]}")
|
|
1087
|
+
raise typer.Exit(1)
|
|
1088
|
+
config = load_config()
|
|
1089
|
+
out_dir = _Path(output)
|
|
1090
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
1091
|
+
|
|
1092
|
+
results = []
|
|
1093
|
+
direct_hits = 0
|
|
1094
|
+
for entry in entries:
|
|
1095
|
+
identifier = str(entry.get("identifier") or "").strip()
|
|
1096
|
+
if not identifier:
|
|
1097
|
+
continue
|
|
1098
|
+
status = str(entry.get("download_status") or "")
|
|
1099
|
+
pdf_url = str(entry.get("pdf_url") or "").strip()
|
|
1100
|
+
if status == "eligible" and pdf_url:
|
|
1101
|
+
out_path = out_dir / f"{safe_filename(identifier)}.pdf"
|
|
1102
|
+
result = download_pdf(pdf_url, out_path, config, source="oa_manifest")
|
|
1103
|
+
if result and result.get("success") and is_pdf_file(out_path):
|
|
1104
|
+
result["identifier"] = identifier
|
|
1105
|
+
result["doi"] = identifier
|
|
1106
|
+
result["cached"] = False
|
|
1107
|
+
_update_doi_index(out_dir, identifier, out_path, source="oa_manifest", strategy=config.get("download_strategy", ""), config=config)
|
|
1108
|
+
results.append(result)
|
|
1109
|
+
direct_hits += 1
|
|
1110
|
+
continue
|
|
1111
|
+
print(f" Direct PDF failed for {identifier}, falling back to engine")
|
|
1112
|
+
result = download(identifier, out_dir, scihub_enabled=scihub, use_tor=use_tor, use_vpnsci=use_vpnsci)
|
|
1113
|
+
results.append(result)
|
|
1114
|
+
|
|
1115
|
+
succeeded = sum(1 for r in results if r and r.get("success"))
|
|
1116
|
+
report = {
|
|
1117
|
+
"total": len(results),
|
|
1118
|
+
"direct_downloads": direct_hits,
|
|
1119
|
+
"succeeded": succeeded,
|
|
1120
|
+
"failed": len(results) - succeeded,
|
|
1121
|
+
"results": results,
|
|
1122
|
+
}
|
|
1123
|
+
(out_dir / "download_results.json").write_text(_json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
1124
|
+
print(_json.dumps(report, indent=2, ensure_ascii=False))
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
@app.command("config-cmd")
|
|
1128
|
+
def config_show(
|
|
1129
|
+
key: str = typer.Argument("", help="Config key to show/set"),
|
|
1130
|
+
value: str = typer.Argument("", help="Value to set"),
|
|
1131
|
+
) -> None:
|
|
1132
|
+
"""Show or set configuration values."""
|
|
1133
|
+
from ..config import load_config, save_config, update_config
|
|
1134
|
+
|
|
1135
|
+
config = load_config()
|
|
1136
|
+
|
|
1137
|
+
if not key:
|
|
1138
|
+
# Show all config
|
|
1139
|
+
for k, v in sorted(config.items()):
|
|
1140
|
+
if "key" in k.lower() or "token" in k.lower() or "secret" in k.lower() or "password" in k.lower():
|
|
1141
|
+
v = "***" if v else "(not set)"
|
|
1142
|
+
print(f" {k:30s} = {v}")
|
|
1143
|
+
return
|
|
1144
|
+
|
|
1145
|
+
if not value:
|
|
1146
|
+
v = config.get(key, "(not set)")
|
|
1147
|
+
if "key" in key.lower() or "token" in key.lower():
|
|
1148
|
+
v = "***" if v and v != "(not set)" else "(not set)"
|
|
1149
|
+
print(f" {key} = {v}")
|
|
1150
|
+
return
|
|
1151
|
+
|
|
1152
|
+
# Use update_config for proper type coercion and validation
|
|
1153
|
+
try:
|
|
1154
|
+
update_config(key, value)
|
|
1155
|
+
print(f" Set {key} = {value}")
|
|
1156
|
+
except ValueError as e:
|
|
1157
|
+
print(f" Error: {e}")
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
def main() -> None:
|
|
1161
|
+
# Console-safe printing: replace characters the target encoding cannot
|
|
1162
|
+
# represent (e.g. GBK consoles with non-ASCII author names) instead of
|
|
1163
|
+
# crashing on UnicodeEncodeError.
|
|
1164
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
1165
|
+
try:
|
|
1166
|
+
sys.stdout.reconfigure(errors="replace")
|
|
1167
|
+
except Exception:
|
|
1168
|
+
pass
|
|
1169
|
+
app()
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
if __name__ == "__main__":
|
|
1173
|
+
main()
|