litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Is a proxy program (v2rayN, Clash, ...) in the way of the institutional route?
|
|
2
|
+
|
|
3
|
+
Institutional access is granted by the campus network's IP, so LitLoop's own
|
|
4
|
+
traffic must leave directly, not through the user's proxy:
|
|
5
|
+
|
|
6
|
+
* system-proxy mode (the proxy program sets Windows' proxy setting): LitLoop
|
|
7
|
+
already ignores it — requests run with ``trust_env=False`` and the browsers
|
|
8
|
+
it starts get ``--no-proxy-server``. Nothing to do.
|
|
9
|
+
* TUN / virtual-adapter mode: the proxy program captures all traffic below
|
|
10
|
+
the applications, which no program can opt out of. The user has to add
|
|
11
|
+
"direct" rules for LitLoop's processes in the proxy program.
|
|
12
|
+
|
|
13
|
+
Only local facts are read (network adapters, process names, the Windows
|
|
14
|
+
proxy setting); nothing is sent anywhere.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
# Proxy programs, by process name (lower case, without .exe) -> what to call them.
|
|
28
|
+
PROGRAMS = {
|
|
29
|
+
"v2rayn": "v2rayN", "xray": "Xray", "sing-box": "sing-box", "v2ray": "V2Ray",
|
|
30
|
+
"clash-verge": "Clash Verge", "clash verge": "Clash Verge", "verge-mihomo": "Clash Verge",
|
|
31
|
+
"mihomo": "mihomo", "clash-meta": "Clash Meta", "clash": "Clash", "clashx": "ClashX",
|
|
32
|
+
"nekoray": "NekoRay", "hiddify": "Hiddify",
|
|
33
|
+
}
|
|
34
|
+
# Virtual adapters such programs create for TUN mode (name or description).
|
|
35
|
+
_TUN = re.compile(r"sing-?tun|singbox|wintun|mihomo|clash|meta|\btun\b|v2ray|xray|nekoray|hiddify|tap-windows", re.I)
|
|
36
|
+
|
|
37
|
+
BROWSER_PROCESS = "litloop-chrome.exe" # browser_backend.STABLE_BROWSER_NAME
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def litloop_processes() -> list[str]:
|
|
41
|
+
"""Process names that need a direct rule: the Python running LitLoop and its browser."""
|
|
42
|
+
names = [Path(sys.executable).name or "python.exe", BROWSER_PROCESS]
|
|
43
|
+
return list(dict.fromkeys(names))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def litloop_paths() -> list[str]:
|
|
47
|
+
"""Full paths of those programs, for proxy programs that match by path.
|
|
48
|
+
|
|
49
|
+
A virtual environment's python.exe only starts the base interpreter, which
|
|
50
|
+
is the process that talks to the network, so that is the one listed.
|
|
51
|
+
"""
|
|
52
|
+
paths = [getattr(sys, "_base_executable", "") or sys.executable]
|
|
53
|
+
from ..config import DATA_DIR
|
|
54
|
+
try:
|
|
55
|
+
paths += [str(p) for p in Path(DATA_DIR).glob(f"*/{BROWSER_PROCESS}")]
|
|
56
|
+
except OSError:
|
|
57
|
+
pass
|
|
58
|
+
return list(dict.fromkeys(p for p in paths if p))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _windows_facts() -> dict[str, Any]:
|
|
62
|
+
script = (
|
|
63
|
+
"$a = Get-NetAdapter -IncludeHidden -ErrorAction SilentlyContinue | Where-Object Status -eq 'Up' | "
|
|
64
|
+
"Select-Object Name, InterfaceDescription; "
|
|
65
|
+
"$p = Get-Process -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Name; "
|
|
66
|
+
"$s = Get-ItemProperty 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings' -ErrorAction SilentlyContinue; "
|
|
67
|
+
"@{adapters=@($a); processes=@($p); proxy_enable=$s.ProxyEnable; proxy_server=$s.ProxyServer} | ConvertTo-Json -Depth 3 -Compress"
|
|
68
|
+
)
|
|
69
|
+
out = subprocess.run(["powershell", "-NoProfile", "-NonInteractive", "-Command", script],
|
|
70
|
+
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=20,
|
|
71
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
72
|
+
return json.loads(out.stdout or "{}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def detect(facts: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
76
|
+
"""What is running and in which mode.
|
|
77
|
+
|
|
78
|
+
``mode``: ``tun`` (traffic is captured: direct rules needed), ``system``
|
|
79
|
+
(system proxy only: LitLoop bypasses it by itself), ``none``, or
|
|
80
|
+
``unknown`` (could not look, e.g. not Windows).
|
|
81
|
+
"""
|
|
82
|
+
if facts is None:
|
|
83
|
+
if os.name != "nt":
|
|
84
|
+
return {"mode": "unknown", "programs": [], "tun": [], "system_proxy": None,
|
|
85
|
+
"direct_processes": litloop_processes()}
|
|
86
|
+
try:
|
|
87
|
+
facts = _windows_facts()
|
|
88
|
+
except (OSError, ValueError, subprocess.SubprocessError):
|
|
89
|
+
return {"mode": "unknown", "programs": [], "tun": [], "system_proxy": None,
|
|
90
|
+
"direct_processes": litloop_processes()}
|
|
91
|
+
running = {str(p).lower() for p in facts.get("processes") or []}
|
|
92
|
+
programs = list(dict.fromkeys(label for name, label in PROGRAMS.items() if name in running))
|
|
93
|
+
tun = [a for a in facts.get("adapters") or []
|
|
94
|
+
if _TUN.search(f"{a.get('Name', '')} {a.get('InterfaceDescription', '')}")]
|
|
95
|
+
system = facts.get("proxy_server") if facts.get("proxy_enable") in (1, True, "1") else None
|
|
96
|
+
mode = "tun" if tun else "system" if (system or programs) else "none"
|
|
97
|
+
return {"mode": mode, "programs": programs, "tun": [a.get("Name") for a in tun],
|
|
98
|
+
"system_proxy": system, "direct_processes": litloop_processes()}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Remote human-in-the-loop assist for SSO/CAPTCHA verification.
|
|
2
|
+
|
|
3
|
+
When a browser session encounters SSO login or CAPTCHA that requires human
|
|
4
|
+
intervention, this module starts a lightweight HTTP server so the user can
|
|
5
|
+
monitor progress and signal completion from any device (phone, tablet, etc.).
|
|
6
|
+
|
|
7
|
+
Usage pattern:
|
|
8
|
+
assist = RemoteAssist(config, publisher="elsevier")
|
|
9
|
+
assist.start()
|
|
10
|
+
# ... browser navigates to SSO page ...
|
|
11
|
+
assist.wait_for_user(timeout=300) # blocks until user signals done
|
|
12
|
+
assist.stop()
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import html
|
|
18
|
+
import json
|
|
19
|
+
import socket
|
|
20
|
+
import string
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from ..log import get_logger
|
|
27
|
+
|
|
28
|
+
log = get_logger()
|
|
29
|
+
|
|
30
|
+
# HTML template for the remote assist page. string.Template rather than str.format:
|
|
31
|
+
# the page carries CSS and JavaScript, whose braces str.format would read as fields.
|
|
32
|
+
_PAGE_TEMPLATE = string.Template("""<!DOCTYPE html>
|
|
33
|
+
<html>
|
|
34
|
+
<head>
|
|
35
|
+
<meta charset="utf-8">
|
|
36
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
37
|
+
<title>LitLoop-PDF Remote Assist</title>
|
|
38
|
+
<style>
|
|
39
|
+
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
40
|
+
body { font-family: -apple-system, system-ui, sans-serif; background: #0d1117; color: #c9d1d9;
|
|
41
|
+
display: flex; flex-direction: column; align-items: center; min-height: 100vh; padding: 20px; }
|
|
42
|
+
.card { background: #161b22; border: 1px solid #30363d; border-radius: 12px; padding: 24px;
|
|
43
|
+
max-width: 480px; width: 100%; margin-top: 20px; }
|
|
44
|
+
h1 { font-size: 20px; color: #58a6ff; margin-bottom: 8px; }
|
|
45
|
+
.publisher { font-size: 14px; color: #8b949e; margin-bottom: 16px; }
|
|
46
|
+
.status { padding: 12px 16px; border-radius: 8px; margin-bottom: 16px; font-size: 14px; }
|
|
47
|
+
.status.waiting { background: #1c1f26; border: 1px solid #d29922; color: #d29922; }
|
|
48
|
+
.status.done { background: #1c1f26; border: 1px solid #3fb950; color: #3fb950; }
|
|
49
|
+
.status.timeout { background: #1c1f26; border: 1px solid #f85149; color: #f85149; }
|
|
50
|
+
.instructions { font-size: 13px; color: #8b949e; line-height: 1.6; margin-bottom: 16px; }
|
|
51
|
+
.instructions li { margin-bottom: 6px; }
|
|
52
|
+
.btn { display: block; width: 100%; padding: 14px; border: none; border-radius: 8px;
|
|
53
|
+
font-size: 16px; font-weight: 600; cursor: pointer; transition: all 0.2s; }
|
|
54
|
+
.btn-primary { background: #238636; color: #fff; }
|
|
55
|
+
.btn-primary:hover { background: #2ea043; }
|
|
56
|
+
.btn-primary:disabled { background: #21262d; color: #484f58; cursor: not-allowed; }
|
|
57
|
+
.url-box { background: #0d1117; border: 1px solid #30363d; border-radius: 6px;
|
|
58
|
+
padding: 8px 12px; font-size: 12px; color: #58a6ff; word-break: break-all;
|
|
59
|
+
margin-bottom: 16px; }
|
|
60
|
+
.timer { font-size: 12px; color: #484f58; text-align: center; margin-top: 12px; }
|
|
61
|
+
</style>
|
|
62
|
+
</head>
|
|
63
|
+
<body>
|
|
64
|
+
<div class="card">
|
|
65
|
+
<h1>🔐 Remote Assist</h1>
|
|
66
|
+
<div class="publisher">$publisher</div>
|
|
67
|
+
<div class="status $status_class" id="status">$status_text</div>
|
|
68
|
+
<div class="instructions">
|
|
69
|
+
<ol>
|
|
70
|
+
<li>Complete the login/verification in the browser window on the host machine</li>
|
|
71
|
+
<li>After verification is done, click the button below</li>
|
|
72
|
+
</ol>
|
|
73
|
+
</div>
|
|
74
|
+
<div class="url-box">$browser_url</div>
|
|
75
|
+
<button class="btn btn-primary" id="doneBtn" onclick="signalDone()">
|
|
76
|
+
✅ Verification Complete — Continue
|
|
77
|
+
</button>
|
|
78
|
+
<div class="timer" id="timer"></div>
|
|
79
|
+
</div>
|
|
80
|
+
<script>
|
|
81
|
+
let done = false;
|
|
82
|
+
async function signalDone() {
|
|
83
|
+
if (done) return;
|
|
84
|
+
done = true;
|
|
85
|
+
document.getElementById('doneBtn').disabled = true;
|
|
86
|
+
document.getElementById('doneBtn').textContent = '⏳ Continuing...';
|
|
87
|
+
await fetch('/api/done', {method: 'POST'});
|
|
88
|
+
document.getElementById('status').className = 'status done';
|
|
89
|
+
document.getElementById('status').textContent = '✅ Done! Resuming...';
|
|
90
|
+
}
|
|
91
|
+
// Auto-check status
|
|
92
|
+
setInterval(async () => {
|
|
93
|
+
if (done) return;
|
|
94
|
+
try {
|
|
95
|
+
const r = await fetch('/api/status');
|
|
96
|
+
const j = await r.json();
|
|
97
|
+
if (j.completed) signalDone();
|
|
98
|
+
if (j.url) document.querySelector('.url-box').textContent = j.url;
|
|
99
|
+
document.getElementById('timer').textContent = 'Elapsed: ' + j.elapsed + 's / ' + j.timeout + 's';
|
|
100
|
+
} catch(e) {}
|
|
101
|
+
}, 2000);
|
|
102
|
+
</script>
|
|
103
|
+
</body>
|
|
104
|
+
</html>""")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class _RequestHandler(BaseHTTPRequestHandler):
|
|
108
|
+
"""HTTP request handler for the remote assist server."""
|
|
109
|
+
|
|
110
|
+
# Shared state across requests (set by RemoteAssist)
|
|
111
|
+
_state: dict[str, Any] = {}
|
|
112
|
+
|
|
113
|
+
def log_message(self, format, *args):
|
|
114
|
+
"""Suppress default HTTP logging."""
|
|
115
|
+
pass
|
|
116
|
+
|
|
117
|
+
def do_GET(self):
|
|
118
|
+
if self.path == "/" or self.path == "/index.html":
|
|
119
|
+
self._serve_page()
|
|
120
|
+
elif self.path == "/api/status":
|
|
121
|
+
self._serve_status()
|
|
122
|
+
else:
|
|
123
|
+
self.send_error(404)
|
|
124
|
+
|
|
125
|
+
def do_POST(self):
|
|
126
|
+
if self.path == "/api/done":
|
|
127
|
+
self._handle_done()
|
|
128
|
+
else:
|
|
129
|
+
self.send_error(404)
|
|
130
|
+
|
|
131
|
+
def _serve_page(self):
|
|
132
|
+
state = self._state
|
|
133
|
+
status_class = "done" if state.get("completed") else "waiting"
|
|
134
|
+
status_text = "✅ Verification complete!" if state.get("completed") else "⏳ Waiting for verification..."
|
|
135
|
+
page = _PAGE_TEMPLATE.substitute(
|
|
136
|
+
publisher=html.escape(str(state.get("publisher", "Unknown"))),
|
|
137
|
+
status_class=status_class,
|
|
138
|
+
status_text=status_text,
|
|
139
|
+
browser_url=html.escape(str(state.get("browser_url", ""))),
|
|
140
|
+
)
|
|
141
|
+
self.send_response(200)
|
|
142
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
143
|
+
self.end_headers()
|
|
144
|
+
self.wfile.write(page.encode())
|
|
145
|
+
|
|
146
|
+
def _serve_status(self):
|
|
147
|
+
state = self._state
|
|
148
|
+
elapsed = int(time.time() - state.get("started_at", time.time()))
|
|
149
|
+
data = {
|
|
150
|
+
"completed": state.get("completed", False),
|
|
151
|
+
"url": state.get("browser_url", ""),
|
|
152
|
+
"elapsed": elapsed,
|
|
153
|
+
"timeout": state.get("timeout", 300),
|
|
154
|
+
}
|
|
155
|
+
self.send_response(200)
|
|
156
|
+
self.send_header("Content-Type", "application/json")
|
|
157
|
+
self.end_headers()
|
|
158
|
+
self.wfile.write(json.dumps(data).encode())
|
|
159
|
+
|
|
160
|
+
def _handle_done(self):
|
|
161
|
+
self._state["completed"] = True
|
|
162
|
+
event = self._state.get("done_event")
|
|
163
|
+
if event:
|
|
164
|
+
event.set()
|
|
165
|
+
self.send_response(200)
|
|
166
|
+
self.send_header("Content-Type", "application/json")
|
|
167
|
+
self.end_headers()
|
|
168
|
+
self.wfile.write(b'{"ok":true}')
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _find_free_port() -> int:
|
|
172
|
+
"""Find a free TCP port on localhost."""
|
|
173
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
174
|
+
s.bind(("127.0.0.1", 0))
|
|
175
|
+
return s.getsockname()[1]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class RemoteAssist:
|
|
179
|
+
"""Remote human-in-the-loop assist server.
|
|
180
|
+
|
|
181
|
+
Starts an HTTP server that the user can access from any device to:
|
|
182
|
+
- See the current verification status
|
|
183
|
+
- Signal that verification is complete
|
|
184
|
+
|
|
185
|
+
Example:
|
|
186
|
+
assist = RemoteAssist(config, publisher="elsevier")
|
|
187
|
+
assist.start()
|
|
188
|
+
# ... browser navigates to SSO ...
|
|
189
|
+
assist.update_url(page.url)
|
|
190
|
+
if assist.wait_for_user(timeout=300):
|
|
191
|
+
print("User completed verification!")
|
|
192
|
+
assist.stop()
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
def __init__(self, config: dict[str, Any], publisher: str = ""):
|
|
196
|
+
self._config = config
|
|
197
|
+
self._publisher = publisher
|
|
198
|
+
self._port = int(config.get("remote_assist_port", 0)) or _find_free_port()
|
|
199
|
+
self._server: HTTPServer | None = None
|
|
200
|
+
self._thread: threading.Thread | None = None
|
|
201
|
+
self._done_event = threading.Event()
|
|
202
|
+
self._started_at = time.time()
|
|
203
|
+
self._state: dict[str, Any] = {
|
|
204
|
+
"publisher": publisher,
|
|
205
|
+
"completed": False,
|
|
206
|
+
"browser_url": "",
|
|
207
|
+
"started_at": self._started_at,
|
|
208
|
+
"timeout": 300,
|
|
209
|
+
"done_event": self._done_event,
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
@property
|
|
213
|
+
def port(self) -> int:
|
|
214
|
+
return self._port
|
|
215
|
+
|
|
216
|
+
@property
|
|
217
|
+
def url(self) -> str:
|
|
218
|
+
return f"http://localhost:{self._port}"
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def lan_url(self) -> str:
|
|
222
|
+
"""Get the LAN-accessible URL (for phone/tablet access)."""
|
|
223
|
+
try:
|
|
224
|
+
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
225
|
+
s.connect(("8.8.8.8", 80))
|
|
226
|
+
ip = s.getsockname()[0]
|
|
227
|
+
s.close()
|
|
228
|
+
return f"http://{ip}:{self._port}"
|
|
229
|
+
except Exception:
|
|
230
|
+
return self.url
|
|
231
|
+
|
|
232
|
+
def start(self) -> str:
|
|
233
|
+
"""Start the remote assist server. Returns the access URL."""
|
|
234
|
+
_RequestHandler._state = self._state
|
|
235
|
+
self._server = HTTPServer(("0.0.0.0", self._port), _RequestHandler)
|
|
236
|
+
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
|
237
|
+
self._thread.start()
|
|
238
|
+
lan = self.lan_url
|
|
239
|
+
log.info(f" [RemoteAssist] Server started: {lan}")
|
|
240
|
+
return lan
|
|
241
|
+
|
|
242
|
+
def update_url(self, browser_url: str) -> None:
|
|
243
|
+
"""Update the displayed browser URL."""
|
|
244
|
+
self._state["browser_url"] = browser_url
|
|
245
|
+
|
|
246
|
+
def wait_for_user(self, timeout: int = 300) -> bool:
|
|
247
|
+
"""Block until user signals completion or timeout.
|
|
248
|
+
|
|
249
|
+
Returns True if user completed, False on timeout.
|
|
250
|
+
"""
|
|
251
|
+
self._state["timeout"] = timeout
|
|
252
|
+
lan = self.lan_url
|
|
253
|
+
log.info(f" [RemoteAssist] Waiting for user at {lan} (timeout={timeout}s)")
|
|
254
|
+
print(f"\n{'='*60}")
|
|
255
|
+
print(f" 🔐 Remote Assist: Complete verification from any device")
|
|
256
|
+
print(f" 📱 Open this URL: {lan}")
|
|
257
|
+
print(f" ⏱ Timeout: {timeout}s")
|
|
258
|
+
print(f"{'='*60}\n")
|
|
259
|
+
completed = self._done_event.wait(timeout=timeout)
|
|
260
|
+
if completed:
|
|
261
|
+
log.info(" [RemoteAssist] User signaled completion")
|
|
262
|
+
else:
|
|
263
|
+
log.info(" [RemoteAssist] Timed out waiting for user")
|
|
264
|
+
return completed
|
|
265
|
+
|
|
266
|
+
def stop(self) -> None:
|
|
267
|
+
"""Stop the remote assist server."""
|
|
268
|
+
if self._server:
|
|
269
|
+
self._server.shutdown()
|
|
270
|
+
self._server = None
|
|
271
|
+
if self._thread:
|
|
272
|
+
self._thread.join(timeout=5)
|
|
273
|
+
self._thread = None
|
|
274
|
+
log.info(" [RemoteAssist] Server stopped")
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""School database for multi-university WebVPN support."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
_DEFAULT_KEY = b"wrdvpnisthebest!"
|
|
10
|
+
|
|
11
|
+
_DATA_DIR = Path(__file__).parent.parent / "data"
|
|
12
|
+
_DATA_FILE = _DATA_DIR / "webvpn.json"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
_schools_cache: list["SchoolEntry"] | None = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class SchoolEntry:
|
|
20
|
+
name: str
|
|
21
|
+
province: str
|
|
22
|
+
host: str
|
|
23
|
+
key: bytes
|
|
24
|
+
iv: bytes
|
|
25
|
+
school_type: str = "webvpn" # "webvpn", "easyconnect", "atrust", or "ezproxy"
|
|
26
|
+
gateway: str = "" # EasyConnect/aTrust gateway domain
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load_db() -> dict:
|
|
30
|
+
if _DATA_FILE.exists():
|
|
31
|
+
return json.loads(_DATA_FILE.read_text(encoding="utf-8"))
|
|
32
|
+
|
|
33
|
+
return {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _parse_entry(name: str, province: str, info: dict) -> SchoolEntry | None:
|
|
37
|
+
host = info.get("host", "").strip()
|
|
38
|
+
if not host:
|
|
39
|
+
return None
|
|
40
|
+
if not host.startswith("http"):
|
|
41
|
+
host = f"https://{host}"
|
|
42
|
+
|
|
43
|
+
key_str = info.get("crypto_key", "")
|
|
44
|
+
iv_str = info.get("crypto_iv", "")
|
|
45
|
+
|
|
46
|
+
key = key_str.encode("utf-8") if key_str else _DEFAULT_KEY
|
|
47
|
+
iv = iv_str.encode("utf-8") if iv_str else key
|
|
48
|
+
|
|
49
|
+
school_type = info.get("type", "webvpn")
|
|
50
|
+
gateway = info.get("gateway", "")
|
|
51
|
+
|
|
52
|
+
return SchoolEntry(
|
|
53
|
+
name=name, province=province, host=host,
|
|
54
|
+
key=key, iv=iv, school_type=school_type, gateway=gateway,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def list_schools() -> list[SchoolEntry]:
|
|
59
|
+
global _schools_cache
|
|
60
|
+
if _schools_cache is not None:
|
|
61
|
+
return _schools_cache
|
|
62
|
+
db = _load_db()
|
|
63
|
+
result = []
|
|
64
|
+
for province, schools in db.items():
|
|
65
|
+
for name, info in schools.items():
|
|
66
|
+
entry = _parse_entry(name, province, info)
|
|
67
|
+
if entry:
|
|
68
|
+
result.append(entry)
|
|
69
|
+
_schools_cache = result
|
|
70
|
+
return result
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def search_schools(query: str) -> list[SchoolEntry]:
|
|
74
|
+
query_lower = query.lower()
|
|
75
|
+
results = []
|
|
76
|
+
for entry in list_schools():
|
|
77
|
+
if (query_lower in entry.name.lower()
|
|
78
|
+
or query_lower in entry.province.lower()
|
|
79
|
+
or query_lower in entry.host.lower()):
|
|
80
|
+
results.append(entry)
|
|
81
|
+
return results
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def get_school(name: str) -> SchoolEntry:
|
|
85
|
+
schools = list_schools()
|
|
86
|
+
|
|
87
|
+
for s in schools:
|
|
88
|
+
if s.name == name:
|
|
89
|
+
return s
|
|
90
|
+
|
|
91
|
+
matches = [s for s in schools if name in s.name]
|
|
92
|
+
if len(matches) == 1:
|
|
93
|
+
return matches[0]
|
|
94
|
+
if len(matches) > 1:
|
|
95
|
+
matches.sort(key=lambda s: len(s.name))
|
|
96
|
+
return matches[0]
|
|
97
|
+
|
|
98
|
+
for s in schools:
|
|
99
|
+
if s.name in name:
|
|
100
|
+
return s
|
|
101
|
+
|
|
102
|
+
raise ValueError(f"School not found: '{name}'. Use instsci_schools to list.")
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Institutional session self-healing.
|
|
2
|
+
|
|
3
|
+
Before a download tries the institutional phase, validate the saved sessions
|
|
4
|
+
and, when ``auto_relogin`` is enabled, refresh expired ones by opening the
|
|
5
|
+
login browser. Conservative by design: only a session that previously existed
|
|
6
|
+
(cookies on disk) and is clearly expired triggers the browser; fresh users
|
|
7
|
+
and network-unreachable states never do.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import threading
|
|
13
|
+
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ..log import get_logger
|
|
17
|
+
|
|
18
|
+
log = get_logger()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _webvpn_configured(config: dict[str, Any]) -> bool:
|
|
22
|
+
return bool(config.get("vpnsci_enabled", False)) and bool(
|
|
23
|
+
config.get("vpnsci_base_url") or config.get("vpnsci_school")
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def ensure_webvpn_session(config: dict[str, Any]) -> dict[str, Any]:
|
|
28
|
+
"""Validate the WebVPN session and refresh it when clearly expired.
|
|
29
|
+
|
|
30
|
+
Returns a status report whose ``status`` is one of ``disabled``,
|
|
31
|
+
``valid``, ``none`` (no saved cookies — never auto-login), ``unreachable``
|
|
32
|
+
(network failure — never auto-login), ``refreshed``, or ``login_failed``.
|
|
33
|
+
"""
|
|
34
|
+
if not config.get("auto_relogin", True):
|
|
35
|
+
return {"status": "disabled", "auto_relogin": False}
|
|
36
|
+
from ..sources.instsci import instsci_login, session_status
|
|
37
|
+
|
|
38
|
+
status = session_status(config)
|
|
39
|
+
if status in ("valid", "none", "unreachable"):
|
|
40
|
+
return {"status": status, "auto_relogin": True}
|
|
41
|
+
log.info(" [session-heal] WebVPN session expired, opening login browser...")
|
|
42
|
+
from ..download import progress as _pr
|
|
43
|
+
school = str(config.get("vpnsci_school") or "学校").strip()
|
|
44
|
+
_pr.set_attention("webvpn-login", f"WebVPN 登录已过期,请在弹出的浏览器窗口中重新登录{school} WebVPN",
|
|
45
|
+
phase="等待你的操作")
|
|
46
|
+
try:
|
|
47
|
+
ok = instsci_login(config)
|
|
48
|
+
finally:
|
|
49
|
+
_pr.clear_attention("webvpn-login")
|
|
50
|
+
after = session_status(config)
|
|
51
|
+
return {
|
|
52
|
+
"status": "refreshed" if ok and after == "valid" else "login_failed",
|
|
53
|
+
"auto_relogin": True,
|
|
54
|
+
"validated_after": after,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_HEAL_LOCK = threading.Lock()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def ensure_institutional_sessions(config: dict[str, Any], *, use_vpnsci: bool = True) -> dict[str, Any]:
|
|
62
|
+
"""Validate/refresh institutional sessions before a download attempt.
|
|
63
|
+
|
|
64
|
+
CARSI sessions self-heal inside ``CARSIClient.login`` (cookie freshness
|
|
65
|
+
check with automatic browser fallback), so only WebVPN needs a hook here.
|
|
66
|
+
"""
|
|
67
|
+
report: dict[str, Any] = {}
|
|
68
|
+
if use_vpnsci and _webvpn_configured(config):
|
|
69
|
+
# Several downloads at once may all find the session stale: one of them
|
|
70
|
+
# refreshes it (a login window at most once), the others then find it fresh.
|
|
71
|
+
with _HEAL_LOCK:
|
|
72
|
+
report["webvpn"] = ensure_webvpn_session(config)
|
|
73
|
+
return report
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Shared cookie persistence for institutional access sessions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
_WINDOWS = os.name == "nt"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def keep_private(path: str | Path) -> None:
|
|
17
|
+
"""A cookie file is the user's school or publisher login: readable by the
|
|
18
|
+
user alone (0600 on macOS and Linux; a Windows profile folder is the
|
|
19
|
+
user's own already). 2026-10-23 review: they were written world-readable."""
|
|
20
|
+
if _WINDOWS:
|
|
21
|
+
return
|
|
22
|
+
try:
|
|
23
|
+
os.chmod(path, 0o600)
|
|
24
|
+
except OSError:
|
|
25
|
+
logger.debug("could not restrict %s", path, exc_info=True)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CookieStore:
|
|
29
|
+
"""Persist browser cookies and load them into requests sessions."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, path: str | Path):
|
|
32
|
+
self.path = Path(path)
|
|
33
|
+
|
|
34
|
+
def load(self, *, now: float | None = None) -> list[dict[str, Any]]:
|
|
35
|
+
"""Load valid cookies from disk."""
|
|
36
|
+
if not self.path.exists():
|
|
37
|
+
return []
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
|
41
|
+
except (json.JSONDecodeError, OSError) as exc:
|
|
42
|
+
logger.warning("Failed to read cookies from %s: %s", self.path, exc)
|
|
43
|
+
return []
|
|
44
|
+
|
|
45
|
+
if not isinstance(raw, list):
|
|
46
|
+
logger.warning("Cookie file %s does not contain a cookie list", self.path)
|
|
47
|
+
return []
|
|
48
|
+
|
|
49
|
+
now = time.time() if now is None else now
|
|
50
|
+
cookies = [self._normalize_cookie(c) for c in raw if isinstance(c, dict)]
|
|
51
|
+
return [c for c in cookies if self._is_valid(c, now)]
|
|
52
|
+
|
|
53
|
+
def save(self, cookies: list[dict[str, Any]], *, now: float | None = None) -> list[dict[str, Any]]:
|
|
54
|
+
"""Save valid cookies to disk and return the persisted list."""
|
|
55
|
+
now = time.time() if now is None else now
|
|
56
|
+
valid = [
|
|
57
|
+
normalized
|
|
58
|
+
for cookie in cookies
|
|
59
|
+
if isinstance(cookie, dict)
|
|
60
|
+
for normalized in [self._normalize_cookie(cookie)]
|
|
61
|
+
if self._is_valid(normalized, now)
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
self.path.write_text(json.dumps(valid, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
66
|
+
keep_private(self.path)
|
|
67
|
+
return valid
|
|
68
|
+
|
|
69
|
+
def load_into(self, session: Any, *, now: float | None = None) -> bool:
|
|
70
|
+
"""Load valid cookies into a requests-like session."""
|
|
71
|
+
cookies = self.load(now=now)
|
|
72
|
+
self.apply_to_session(session, cookies)
|
|
73
|
+
return bool(cookies)
|
|
74
|
+
|
|
75
|
+
@staticmethod
|
|
76
|
+
def apply_to_session(session: Any, cookies: list[dict[str, Any]]) -> None:
|
|
77
|
+
"""Apply cookies to a requests-like session cookie jar."""
|
|
78
|
+
for cookie in cookies:
|
|
79
|
+
name = cookie.get("name")
|
|
80
|
+
value = cookie.get("value")
|
|
81
|
+
if name is None or value is None:
|
|
82
|
+
continue
|
|
83
|
+
session.cookies.set(
|
|
84
|
+
name,
|
|
85
|
+
value,
|
|
86
|
+
domain=cookie.get("domain", ""),
|
|
87
|
+
path=cookie.get("path", "/"),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def _normalize_cookie(cookie: dict[str, Any]) -> dict[str, Any]:
|
|
92
|
+
normalized = dict(cookie)
|
|
93
|
+
if normalized.get("expires", 0) < 0:
|
|
94
|
+
normalized["expires"] = 0
|
|
95
|
+
return normalized
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def _is_valid(cookie: dict[str, Any], now: float) -> bool:
|
|
99
|
+
if cookie.get("name") is None or cookie.get("value") is None:
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
expires = cookie.get("expires", 0)
|
|
103
|
+
if not expires or expires == 0:
|
|
104
|
+
return True
|
|
105
|
+
return expires > now
|