litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""SQLite-based domain stats storage for Sci-Hub domain rotation.
|
|
2
|
+
|
|
3
|
+
Also the single persistent home for verification-wall state (ALTCHA /
|
|
4
|
+
interactive gates) — scihub.py's pacing helpers read/write the wall_state
|
|
5
|
+
table here. Do NOT build parallel in-memory or JSON health stores: three
|
|
6
|
+
generations of exactly that caused constant rework (see docs/PLAYBOOK.md).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import sqlite3
|
|
12
|
+
import time
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
_DB_FILENAME = "domain_stats.db"
|
|
18
|
+
|
|
19
|
+
_SCHEMA_SQL = """
|
|
20
|
+
CREATE TABLE IF NOT EXISTS domain_stats (
|
|
21
|
+
domain TEXT PRIMARY KEY,
|
|
22
|
+
success INTEGER NOT NULL DEFAULT 0,
|
|
23
|
+
fail INTEGER NOT NULL DEFAULT 0,
|
|
24
|
+
last_fail REAL NOT NULL DEFAULT 0,
|
|
25
|
+
fail_streak INTEGER NOT NULL DEFAULT 0,
|
|
26
|
+
avg_latency REAL,
|
|
27
|
+
reachable INTEGER,
|
|
28
|
+
updated_at REAL NOT NULL DEFAULT 0
|
|
29
|
+
);
|
|
30
|
+
CREATE TABLE IF NOT EXISTS probe_meta (
|
|
31
|
+
key TEXT PRIMARY KEY,
|
|
32
|
+
value TEXT
|
|
33
|
+
);
|
|
34
|
+
CREATE TABLE IF NOT EXISTS wall_state (
|
|
35
|
+
domain TEXT PRIMARY KEY,
|
|
36
|
+
last_solve REAL NOT NULL DEFAULT 0,
|
|
37
|
+
walls INTEGER NOT NULL DEFAULT 0,
|
|
38
|
+
cooldown_until REAL NOT NULL DEFAULT 0
|
|
39
|
+
);
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _get_db_path(config: dict[str, Any]) -> Path:
|
|
44
|
+
cache_dir = Path(config.get("cache_dir", str(Path.home() / ".litloop" / "cache")))
|
|
45
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
return cache_dir / _DB_FILENAME
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@contextmanager
|
|
50
|
+
def _conn(config: dict[str, Any]):
|
|
51
|
+
"""Open, initialize, and close the stats DB per operation.
|
|
52
|
+
|
|
53
|
+
The database is tiny and operations are short; holding connections open
|
|
54
|
+
across calls kept domain_stats.db locked on Windows and broke callers'
|
|
55
|
+
temp-dir cleanup. Per-operation open/close makes the module leak-free by
|
|
56
|
+
construction (WAL keeps concurrent access cheap).
|
|
57
|
+
"""
|
|
58
|
+
db_path = _get_db_path(config)
|
|
59
|
+
conn = sqlite3.connect(str(db_path), check_same_thread=False, timeout=5)
|
|
60
|
+
try:
|
|
61
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
62
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
63
|
+
conn.row_factory = sqlite3.Row
|
|
64
|
+
conn.executescript(_SCHEMA_SQL)
|
|
65
|
+
conn.commit()
|
|
66
|
+
yield conn
|
|
67
|
+
finally:
|
|
68
|
+
conn.close()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def load_stats(config: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
72
|
+
with _conn(config) as conn:
|
|
73
|
+
rows = conn.execute("SELECT * FROM domain_stats").fetchall()
|
|
74
|
+
stats: dict[str, dict[str, Any]] = {}
|
|
75
|
+
for row in rows:
|
|
76
|
+
stats[row["domain"]] = {
|
|
77
|
+
"success": row["success"],
|
|
78
|
+
"fail": row["fail"],
|
|
79
|
+
"last_fail_time": row["last_fail"],
|
|
80
|
+
"fail_streak": row["fail_streak"],
|
|
81
|
+
"avg_latency_ms": row["avg_latency"],
|
|
82
|
+
"reachable": bool(row["reachable"]) if row["reachable"] is not None else None,
|
|
83
|
+
}
|
|
84
|
+
# Also load probe metadata
|
|
85
|
+
meta = conn.execute("SELECT key, value FROM probe_meta").fetchall()
|
|
86
|
+
for m in meta:
|
|
87
|
+
if m["key"] == "_last_probe":
|
|
88
|
+
try:
|
|
89
|
+
stats["_last_probe"] = int(m["value"])
|
|
90
|
+
except (ValueError, TypeError):
|
|
91
|
+
stats["_last_probe"] = 0
|
|
92
|
+
return stats
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def record_result(domain: str, ok: bool, config: dict[str, Any]) -> None:
|
|
96
|
+
with _conn(config) as conn:
|
|
97
|
+
now = time.time()
|
|
98
|
+
conn.execute("""
|
|
99
|
+
INSERT INTO domain_stats (domain, success, fail, last_fail, fail_streak, updated_at)
|
|
100
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
101
|
+
ON CONFLICT(domain) DO UPDATE SET
|
|
102
|
+
success = success + excluded.success,
|
|
103
|
+
fail = fail + excluded.fail,
|
|
104
|
+
last_fail = CASE WHEN excluded.fail > 0 THEN excluded.last_fail ELSE last_fail END,
|
|
105
|
+
fail_streak = CASE WHEN excluded.fail > 0 THEN fail_streak + 1 ELSE 0 END,
|
|
106
|
+
updated_at = excluded.updated_at
|
|
107
|
+
""", (domain, 1 if ok else 0, 0 if ok else 1, now if not ok else 0, 1 if not ok else 0, now))
|
|
108
|
+
conn.commit()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def update_probe(domain: str, reachable: bool, latency_ms: float, config: dict[str, Any]) -> None:
|
|
112
|
+
with _conn(config) as conn:
|
|
113
|
+
conn.execute("""
|
|
114
|
+
INSERT INTO domain_stats (domain, success, fail, reachable, avg_latency, updated_at)
|
|
115
|
+
VALUES (?, 0, 0, ?, ?, ?)
|
|
116
|
+
ON CONFLICT(domain) DO UPDATE SET
|
|
117
|
+
reachable = excluded.reachable,
|
|
118
|
+
avg_latency = excluded.avg_latency,
|
|
119
|
+
updated_at = excluded.updated_at
|
|
120
|
+
""", (domain, 1 if reachable else 0, latency_ms, time.time()))
|
|
121
|
+
conn.commit()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def set_probe_timestamp(config: dict[str, Any], timestamp: float | None = None) -> None:
|
|
125
|
+
ts = int(timestamp) if timestamp is not None else int(time.time())
|
|
126
|
+
with _conn(config) as conn:
|
|
127
|
+
conn.execute("""
|
|
128
|
+
INSERT INTO probe_meta (key, value) VALUES ('_last_probe', ?)
|
|
129
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
130
|
+
""", (str(ts),))
|
|
131
|
+
conn.commit()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def get_probe_timestamp(config: dict[str, Any]) -> int:
|
|
135
|
+
with _conn(config) as conn:
|
|
136
|
+
row = conn.execute("SELECT value FROM probe_meta WHERE key = '_last_probe'").fetchone()
|
|
137
|
+
if row:
|
|
138
|
+
try:
|
|
139
|
+
return int(row["value"])
|
|
140
|
+
except (ValueError, TypeError):
|
|
141
|
+
return 0
|
|
142
|
+
return 0
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def close_connection() -> None:
|
|
146
|
+
"""Backward-compat no-op: connections are closed after every operation."""
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
# Verification-wall state (ALTCHA/interactive gates) — the single persistent
|
|
152
|
+
# home for mirror health cooldowns. scihub.py's pacing helpers read/write this
|
|
153
|
+
# table; do NOT build parallel in-memory or JSON health stores (that split
|
|
154
|
+
# caused three generations of rework).
|
|
155
|
+
# ---------------------------------------------------------------------------
|
|
156
|
+
|
|
157
|
+
_WALL_DEFAULTS = {"last_solve": 0.0, "walls": 0, "cooldown_until": 0.0}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def get_wall_state(domain: str, config: dict[str, Any]) -> dict[str, float]:
|
|
161
|
+
"""Latest wall state for a domain (defaults when never seen)."""
|
|
162
|
+
with _conn(config) as conn:
|
|
163
|
+
row = conn.execute(
|
|
164
|
+
"SELECT last_solve, walls, cooldown_until FROM wall_state WHERE domain = ?", (domain,)
|
|
165
|
+
).fetchone()
|
|
166
|
+
if not row:
|
|
167
|
+
return dict(_WALL_DEFAULTS)
|
|
168
|
+
return {
|
|
169
|
+
"last_solve": float(row["last_solve"]),
|
|
170
|
+
"walls": int(row["walls"]),
|
|
171
|
+
"cooldown_until": float(row["cooldown_until"]),
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def set_wall_state(
|
|
176
|
+
domain: str,
|
|
177
|
+
config: dict[str, Any],
|
|
178
|
+
*,
|
|
179
|
+
last_solve: float,
|
|
180
|
+
walls: int,
|
|
181
|
+
cooldown_until: float,
|
|
182
|
+
) -> None:
|
|
183
|
+
with _conn(config) as conn:
|
|
184
|
+
conn.execute(
|
|
185
|
+
"""INSERT INTO wall_state (domain, last_solve, walls, cooldown_until)
|
|
186
|
+
VALUES (?, ?, ?, ?)
|
|
187
|
+
ON CONFLICT(domain) DO UPDATE SET last_solve=excluded.last_solve,
|
|
188
|
+
walls=excluded.walls, cooldown_until=excluded.cooldown_until""",
|
|
189
|
+
(domain, float(last_solve), int(walls), float(cooldown_until)),
|
|
190
|
+
)
|
|
191
|
+
# Opportunistic pruning: rows untouched for a day are dead weight (the
|
|
192
|
+
# mirror recovered or was abandoned long ago). NOTE: cooldown_until=0
|
|
193
|
+
# is a VALID healthy state — pruning on it would delete just-reset
|
|
194
|
+
# rows.
|
|
195
|
+
conn.execute("DELETE FROM wall_state WHERE last_solve < ?", (time.time() - 86400,))
|
|
196
|
+
conn.commit()
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def reset_wall_state(config: dict[str, Any]) -> None:
|
|
200
|
+
"""Forget all wall cooldowns (used by tests and manual overrides)."""
|
|
201
|
+
with _conn(config) as conn:
|
|
202
|
+
conn.execute("DELETE FROM wall_state")
|
|
203
|
+
conn.commit()
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Before a batch, measure which Sci-Hub mirror and route (proxy or direct) is fastest.
|
|
2
|
+
|
|
3
|
+
``scihub_route`` decides which routes may be used: ``direct``, ``proxy``
|
|
4
|
+
(host_proxy_url only, keeping the school's exit away from Sci-Hub) or ``auto``
|
|
5
|
+
(both, the faster wins). For ``auto`` and ``proxy`` the batch first downloads
|
|
6
|
+
one known paper through each mirror on each allowed route at once and keeps the
|
|
7
|
+
fastest; the batch then asks that mirror first, on that route. Mirrors keep
|
|
8
|
+
their order behind it, so a paper the fastest one lacks is still found.
|
|
9
|
+
|
|
10
|
+
Why: on 2026-10-17 a 1 MB PDF from sci.bban.top took 18.8 s through the proxy
|
|
11
|
+
and 6.0 s direct; most of a Sci-Hub round was spent transferring PDFs.
|
|
12
|
+
The choice is kept for an hour (``scihub_route.json``) so single downloads use
|
|
13
|
+
it too; failures of the chosen mirror drop it, and the next batch measures again.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import tempfile
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any
|
|
25
|
+
from urllib.parse import urlparse
|
|
26
|
+
|
|
27
|
+
from ..log import get_logger
|
|
28
|
+
|
|
29
|
+
log = get_logger()
|
|
30
|
+
|
|
31
|
+
_TTL = 3600.0
|
|
32
|
+
_FAILS_TO_DROP = 3
|
|
33
|
+
_LOCK = threading.Lock()
|
|
34
|
+
_STATE: dict[str, Any] = {} # {"mirror", "route", "kbps", "at"}
|
|
35
|
+
_FAILS = [0]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def mode(config: dict[str, Any]) -> str:
|
|
39
|
+
"""direct | proxy | auto (scihub_via_proxy: true is the older "proxy")."""
|
|
40
|
+
m = str(config.get("scihub_route") or "").strip().lower()
|
|
41
|
+
if m in ("direct", "proxy", "auto"):
|
|
42
|
+
return m
|
|
43
|
+
return "proxy" if config.get("scihub_via_proxy") else "direct"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _file() -> Path:
|
|
47
|
+
from ..config import DATA_DIR
|
|
48
|
+
return DATA_DIR / "scihub_route.json"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def current() -> dict[str, Any] | None:
|
|
52
|
+
"""The fastest (mirror, route) measured within the hour, or None."""
|
|
53
|
+
with _LOCK:
|
|
54
|
+
if not _STATE:
|
|
55
|
+
try:
|
|
56
|
+
_STATE.update(json.loads(_file().read_text(encoding="utf-8")))
|
|
57
|
+
except (OSError, ValueError):
|
|
58
|
+
pass
|
|
59
|
+
if _STATE and time.time() - float(_STATE.get("at", 0)) < _TTL:
|
|
60
|
+
return dict(_STATE)
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def route(config: dict[str, Any]) -> str:
|
|
65
|
+
"""proxy | direct for Sci-Hub/LibGen requests right now."""
|
|
66
|
+
m = mode(config)
|
|
67
|
+
if m != "auto":
|
|
68
|
+
return m
|
|
69
|
+
cur = current()
|
|
70
|
+
return cur["route"] if cur else "proxy" # not measured yet: the private route
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def preferred_mirror() -> str | None:
|
|
74
|
+
cur = current()
|
|
75
|
+
return cur["mirror"] if cur else None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def note_result(domain: str, ok: bool, absent: bool = False) -> None:
|
|
79
|
+
"""Called after each mirror attempt: the chosen mirror failing in a row is dropped."""
|
|
80
|
+
cur = current()
|
|
81
|
+
if not cur or domain != cur.get("mirror"):
|
|
82
|
+
return
|
|
83
|
+
if ok or absent:
|
|
84
|
+
_FAILS[0] = 0
|
|
85
|
+
return
|
|
86
|
+
_FAILS[0] += 1
|
|
87
|
+
if _FAILS[0] >= _FAILS_TO_DROP:
|
|
88
|
+
log.info(f" [scihub-speed] {domain} failed {_FAILS[0]} times in a row; measuring again next batch")
|
|
89
|
+
forget()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def forget() -> None:
|
|
93
|
+
with _LOCK:
|
|
94
|
+
_STATE.clear()
|
|
95
|
+
_FAILS[0] = 0
|
|
96
|
+
try:
|
|
97
|
+
_file().unlink(missing_ok=True)
|
|
98
|
+
except OSError:
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _routes(config: dict[str, Any]) -> list[str]:
|
|
103
|
+
from ..network import _proxy_reachable
|
|
104
|
+
m = mode(config)
|
|
105
|
+
url = str(config.get("host_proxy_url") or "").strip()
|
|
106
|
+
proxy_ok = bool(url) and _proxy_reachable(url)
|
|
107
|
+
if m == "direct":
|
|
108
|
+
return ["direct"]
|
|
109
|
+
if m == "proxy":
|
|
110
|
+
return ["proxy"] if proxy_ok else []
|
|
111
|
+
return (["proxy"] if proxy_ok else []) + ["direct"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _one(doi: str, mirror: str, rt: str, config: dict[str, Any], tmp: Path) -> tuple[str, str, float, int]:
|
|
115
|
+
"""(mirror, route, seconds, bytes) for one test download; seconds is inf on failure."""
|
|
116
|
+
from .scihub import try_scihub_domain
|
|
117
|
+
cfg = {**config, "scihub_route": rt, "scihub_use_browser": False}
|
|
118
|
+
out = tmp / f"{urlparse(mirror).hostname}_{rt}.pdf"
|
|
119
|
+
t = time.time()
|
|
120
|
+
try:
|
|
121
|
+
r = try_scihub_domain(doi, mirror, out, cfg, False, [])
|
|
122
|
+
except Exception:
|
|
123
|
+
r = None
|
|
124
|
+
dt = time.time() - t
|
|
125
|
+
if r and r.get("success") and out.exists():
|
|
126
|
+
return mirror, rt, dt, out.stat().st_size
|
|
127
|
+
return mirror, rt, float("inf"), 0
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _cleanup_later(futs: list[Any], tmp: str) -> None:
|
|
131
|
+
import shutil
|
|
132
|
+
for f in futs:
|
|
133
|
+
try:
|
|
134
|
+
f.result(timeout=120)
|
|
135
|
+
except Exception:
|
|
136
|
+
pass
|
|
137
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def measure(config: dict[str, Any], *, force: bool = False) -> dict[str, Any] | None:
|
|
141
|
+
"""Pick the fastest mirror and route for this batch (cached for the hour)."""
|
|
142
|
+
if mode(config) == "direct" and not force:
|
|
143
|
+
return None
|
|
144
|
+
if not force and current():
|
|
145
|
+
return current()
|
|
146
|
+
routes = _routes(config)
|
|
147
|
+
mirrors = [str(m).rstrip("/") for m in (config.get("scihub_domains") or [])][: int(config.get("scihub_speedtest_mirrors", 5) or 5)]
|
|
148
|
+
doi = str(config.get("scihub_speedtest_doi") or "10.1016/j.colsurfb.2017.09.051")
|
|
149
|
+
if not routes or not mirrors:
|
|
150
|
+
return None
|
|
151
|
+
budget = float(config.get("scihub_speedtest_seconds", 40) or 40)
|
|
152
|
+
log.info(f" [scihub-speed] measuring {len(mirrors)} mirrors x {routes} with {doi}")
|
|
153
|
+
results: list[tuple[str, str, float, int]] = []
|
|
154
|
+
# All start together, so the first to finish is the fastest: stop there
|
|
155
|
+
# (waiting out the budget for a dead mirror cost 40 s on 2026-10-17).
|
|
156
|
+
# Losers still writing keep their files open: leave the folder to them.
|
|
157
|
+
tmp = tempfile.mkdtemp(prefix="litloop-speed-")
|
|
158
|
+
ex = ThreadPoolExecutor(max_workers=len(mirrors) * len(routes))
|
|
159
|
+
futs = [ex.submit(_one, doi, m, r, config, Path(tmp)) for m in mirrors for r in routes]
|
|
160
|
+
try:
|
|
161
|
+
for f in as_completed(futs, timeout=budget):
|
|
162
|
+
results.append(f.result())
|
|
163
|
+
if results[-1][2] != float("inf"):
|
|
164
|
+
break
|
|
165
|
+
except Exception:
|
|
166
|
+
pass # past the budget: nothing finished
|
|
167
|
+
ex.shutdown(wait=False, cancel_futures=True)
|
|
168
|
+
threading.Thread(target=_cleanup_later, args=(futs, tmp), daemon=True).start()
|
|
169
|
+
ok = [r for r in results if r[2] != float("inf")]
|
|
170
|
+
for m, rt, dt, size in sorted(results, key=lambda r: r[2]):
|
|
171
|
+
log.info(f" [scihub-speed] {m} via {rt}: " + (f"{dt:.1f}s, {size // 1024} KB" if dt != float("inf") else "failed"))
|
|
172
|
+
if not ok:
|
|
173
|
+
return None
|
|
174
|
+
m, rt, dt, size = min(ok, key=lambda r: r[2])
|
|
175
|
+
state = {"mirror": m, "route": rt, "kbps": round(size / 1024 / max(dt, 0.01)), "seconds": round(dt, 1),
|
|
176
|
+
"at": time.time()}
|
|
177
|
+
with _LOCK:
|
|
178
|
+
_STATE.clear()
|
|
179
|
+
_STATE.update(state)
|
|
180
|
+
_FAILS[0] = 0
|
|
181
|
+
try:
|
|
182
|
+
from .. import store
|
|
183
|
+
store.write_json(_file(), state)
|
|
184
|
+
except Exception:
|
|
185
|
+
pass
|
|
186
|
+
log.info(f" [scihub-speed] using {m} via {rt} ({state['kbps']} KB/s)")
|
|
187
|
+
return state
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Adaptive source scoring with exponential moving average (EMA).
|
|
2
|
+
|
|
3
|
+
Tracks per-source success rate and latency. Uses EMA so:
|
|
4
|
+
- Recent results have higher weight
|
|
5
|
+
- Temporary network blips don't permanently ruin a source's score
|
|
6
|
+
- Scores naturally recover when a source starts working again
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ..config import DATA_DIR
|
|
17
|
+
|
|
18
|
+
_SCORES_FILE = DATA_DIR / "source_scores.json"
|
|
19
|
+
|
|
20
|
+
# EMA decay factor: higher = more weight on recent data (0.05-0.2 typical)
|
|
21
|
+
_ALPHA = 0.1
|
|
22
|
+
|
|
23
|
+
# Initial score for unknown sources (neutral)
|
|
24
|
+
_DEFAULT_SCORE = 0.5
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _load_scores() -> dict[str, dict[str, Any]]:
|
|
28
|
+
if _SCORES_FILE.exists():
|
|
29
|
+
try:
|
|
30
|
+
with _SCORES_FILE.open("r", encoding="utf-8") as f:
|
|
31
|
+
return json.load(f)
|
|
32
|
+
except Exception:
|
|
33
|
+
pass
|
|
34
|
+
return {}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _save_scores(scores: dict[str, dict[str, Any]]) -> None:
|
|
38
|
+
_SCORES_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
with _SCORES_FILE.open("w", encoding="utf-8") as f:
|
|
40
|
+
json.dump(scores, f, indent=2, ensure_ascii=False)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def record_result(source: str, success: bool, latency_ms: float = 0, error_type: str = "") -> None:
|
|
44
|
+
"""Record a download attempt result for adaptive scoring."""
|
|
45
|
+
scores = _load_scores()
|
|
46
|
+
entry = scores.get(source, {
|
|
47
|
+
"success_ema": _DEFAULT_SCORE,
|
|
48
|
+
"latency_ema": 5000.0,
|
|
49
|
+
"attempts": 0,
|
|
50
|
+
"last_error": "",
|
|
51
|
+
"last_update": 0,
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
# Update success EMA
|
|
55
|
+
success_val = 1.0 if success else 0.0
|
|
56
|
+
entry["success_ema"] = _ALPHA * success_val + (1 - _ALPHA) * entry["success_ema"]
|
|
57
|
+
|
|
58
|
+
# Update latency EMA (only on success)
|
|
59
|
+
if success and latency_ms > 0:
|
|
60
|
+
entry["latency_ema"] = _ALPHA * latency_ms + (1 - _ALPHA) * entry["latency_ema"]
|
|
61
|
+
|
|
62
|
+
entry["attempts"] = entry.get("attempts", 0) + 1
|
|
63
|
+
entry["last_error"] = error_type if not success else ""
|
|
64
|
+
entry["last_update"] = int(time.time())
|
|
65
|
+
|
|
66
|
+
scores[source] = entry
|
|
67
|
+
_save_scores(scores)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def get_score(source: str) -> float:
|
|
71
|
+
"""Get adaptive score for a source (0.0-1.0, higher = better)."""
|
|
72
|
+
scores = _load_scores()
|
|
73
|
+
entry = scores.get(source)
|
|
74
|
+
if not entry:
|
|
75
|
+
return _DEFAULT_SCORE
|
|
76
|
+
return entry.get("success_ema", _DEFAULT_SCORE)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def get_latency(source: str) -> float:
|
|
80
|
+
"""Get EMA latency for a source in ms."""
|
|
81
|
+
scores = _load_scores()
|
|
82
|
+
entry = scores.get(source)
|
|
83
|
+
if not entry:
|
|
84
|
+
return 5000.0
|
|
85
|
+
return entry.get("latency_ema", 5000.0)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def sort_sources(sources: list[tuple[Any, str]]) -> list[tuple[Any, str]]:
|
|
89
|
+
"""Sort sources by adaptive score (best first), with latency as tiebreaker."""
|
|
90
|
+
def _key(item: tuple[Any, str]) -> tuple[float, float]:
|
|
91
|
+
_, label = item
|
|
92
|
+
score = get_score(label)
|
|
93
|
+
latency = get_latency(label)
|
|
94
|
+
# Higher score first, lower latency first
|
|
95
|
+
return (-score, latency)
|
|
96
|
+
return sorted(sources, key=_key)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def classify_error(resp_status: int = 0, exception: Exception | None = None, html: str = "") -> str:
|
|
100
|
+
"""Classify download error into a category."""
|
|
101
|
+
if resp_status == 404:
|
|
102
|
+
return "not_found"
|
|
103
|
+
if resp_status == 403:
|
|
104
|
+
return "forbidden"
|
|
105
|
+
if resp_status == 429:
|
|
106
|
+
return "rate_limited"
|
|
107
|
+
if resp_status >= 500:
|
|
108
|
+
return "server_error"
|
|
109
|
+
if exception and "timeout" in str(exception).lower():
|
|
110
|
+
return "timeout"
|
|
111
|
+
if exception and "ssl" in str(exception).lower():
|
|
112
|
+
return "ssl_error"
|
|
113
|
+
if html and ("captcha" in html.lower() or "challenge" in html.lower()):
|
|
114
|
+
return "captcha"
|
|
115
|
+
return "unknown"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def get_user_advice(error_type: str, source: str) -> str:
|
|
119
|
+
"""Return user-friendly advice with actionable steps."""
|
|
120
|
+
advice = {
|
|
121
|
+
"not_found": "论文在此源不存在(404),跳过",
|
|
122
|
+
"forbidden": "访问被拒绝(403)→ 建议:1) 配置代理 config_set network_proxy 2) 启用 WebVPN",
|
|
123
|
+
"rate_limited": "请求过于频繁(429)→ 建议:稍后重试,或配置 openalex_api_key 提升配额",
|
|
124
|
+
"timeout": "连接超时 → 建议:1) 检查网络连通性 2) 配置代理绕过封锁 config_set network_proxy 3) 启用 Tor(tor_start)",
|
|
125
|
+
"captcha": "触发 Cloudflare 防护 → 建议:1) 安装 CloakBrowser (pip install cloakbrowser) 2) 配置代理",
|
|
126
|
+
"ssl_error": "SSL 连接错误 → 建议:1) 检查代理是否正确 2) 尝试不同代理协议(socks5/http)3) 更新证书",
|
|
127
|
+
"server_error": "服务器错误(5xx)→ 暂时不可用,稍后重试",
|
|
128
|
+
"dns_blocked": "DNS 解析失败 → 建议:1) 配置代理 config_set network_proxy 2) 更换 DNS(8.8.8.8)3) 启用 Tor",
|
|
129
|
+
"network_blocked": "网络完全不通 → 建议:1) 检查代理配置 config_set network_proxy 2) 使用 WebVPN 机构代理 3) 换用海外网络",
|
|
130
|
+
"paywall": "论文需要机构订阅 → 运行 litloop_browser_login 或 litloop_instsci_login 登录机构账号",
|
|
131
|
+
"cloudflare_blocked": "Cloudflare 反爬封锁 → 安装 CloakBrowser (pip install cloakbrowser) 或配置代理",
|
|
132
|
+
}
|
|
133
|
+
return advice.get(error_type, "未知错误 → 建议:运行 network_diagnose 检查网络状态")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def get_all_scores() -> dict[str, dict[str, Any]]:
|
|
137
|
+
"""Return all source scores for diagnostics."""
|
|
138
|
+
return _load_scores()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def diagnose_network(config: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
142
|
+
"""Run network diagnostics and return actionable report."""
|
|
143
|
+
import os
|
|
144
|
+
import socket
|
|
145
|
+
|
|
146
|
+
report: dict[str, Any] = {
|
|
147
|
+
"proxy": {"configured": False, "source": "none", "url": ""},
|
|
148
|
+
"tests": [],
|
|
149
|
+
"recommendations": [],
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
# Check proxy configuration
|
|
153
|
+
env_proxy = os.environ.get("LITLOOP_PROXY") or os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or ""
|
|
154
|
+
cfg_proxy = (config or {}).get("network_proxy", "")
|
|
155
|
+
active_proxy = env_proxy or cfg_proxy
|
|
156
|
+
|
|
157
|
+
if active_proxy:
|
|
158
|
+
report["proxy"] = {"configured": True, "source": "env" if env_proxy else "config", "url": active_proxy}
|
|
159
|
+
else:
|
|
160
|
+
# Check if system has proxy env vars that litloop ignores
|
|
161
|
+
sys_proxy = os.environ.get("HTTP_PROXY") or os.environ.get("HTTPS_PROXY") or ""
|
|
162
|
+
if sys_proxy:
|
|
163
|
+
report["recommendations"].append(
|
|
164
|
+
f"检测到系统代理 {sys_proxy},但 litloop 未使用。"
|
|
165
|
+
f"运行: litloop config_set network_proxy \"{sys_proxy}\""
|
|
166
|
+
)
|
|
167
|
+
else:
|
|
168
|
+
report["recommendations"].append(
|
|
169
|
+
"未配置代理。如需访问被封锁的网站,运行: litloop config_set network_proxy \"socks5://127.0.0.1:1080\""
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
# Test DNS resolution
|
|
173
|
+
test_domains = [
|
|
174
|
+
("sci-hub.mksa.top", "Sci-Hub"),
|
|
175
|
+
("api.openalex.org", "OpenAlex"),
|
|
176
|
+
("doi.org", "DOI"),
|
|
177
|
+
]
|
|
178
|
+
for domain, label in test_domains:
|
|
179
|
+
try:
|
|
180
|
+
ip = socket.gethostbyname(domain)
|
|
181
|
+
report["tests"].append({"target": label, "domain": domain, "dns": "ok", "ip": ip})
|
|
182
|
+
except socket.gaierror:
|
|
183
|
+
report["tests"].append({"target": label, "domain": domain, "dns": "failed", "ip": ""})
|
|
184
|
+
report["recommendations"].append(f"DNS 解析 {domain} 失败 → 可能被封锁,建议配置代理")
|
|
185
|
+
|
|
186
|
+
# Test TCP connectivity
|
|
187
|
+
tcp_tests = [
|
|
188
|
+
("sci-hub.mksa.top", 443, "Sci-Hub HTTPS"),
|
|
189
|
+
("api.openalex.org", 443, "OpenAlex HTTPS"),
|
|
190
|
+
]
|
|
191
|
+
for host, port, label in tcp_tests:
|
|
192
|
+
try:
|
|
193
|
+
sock = socket.create_connection((host, port), timeout=5)
|
|
194
|
+
sock.close()
|
|
195
|
+
report["tests"].append({"target": label, "tcp": "ok"})
|
|
196
|
+
except Exception as e:
|
|
197
|
+
report["tests"].append({"target": label, "tcp": "failed", "error": str(e)[:50]})
|
|
198
|
+
if "timed out" in str(e).lower():
|
|
199
|
+
report["recommendations"].append(f"{label} 连接超时 → 网络被封锁,强烈建议配置代理")
|
|
200
|
+
|
|
201
|
+
# Test proxy connectivity
|
|
202
|
+
if active_proxy:
|
|
203
|
+
try:
|
|
204
|
+
import requests
|
|
205
|
+
resp = requests.get("https://sci-hub.mksa.top", timeout=10,
|
|
206
|
+
proxies={"https": active_proxy, "http": active_proxy},
|
|
207
|
+
headers={"User-Agent": "Mozilla/5.0"})
|
|
208
|
+
report["tests"].append({"target": "Sci-Hub via proxy", "status": resp.status_code})
|
|
209
|
+
if resp.status_code == 200:
|
|
210
|
+
report["recommendations"].append("代理访问 Sci-Hub 正常 ✓")
|
|
211
|
+
except Exception as e:
|
|
212
|
+
report["tests"].append({"target": "Sci-Hub via proxy", "error": str(e)[:50]})
|
|
213
|
+
report["recommendations"].append(f"代理访问 Sci-Hub 失败 → 检查代理是否正确: {active_proxy}")
|
|
214
|
+
|
|
215
|
+
# Check Tor status
|
|
216
|
+
try:
|
|
217
|
+
from ..tor.proxy import check_tor_circuit
|
|
218
|
+
tor_ok = check_tor_circuit(config)
|
|
219
|
+
report["tests"].append({"target": "Tor SOCKS5", "status": "running" if tor_ok else "not running"})
|
|
220
|
+
if not tor_ok:
|
|
221
|
+
report["recommendations"].append("Tor 未运行 → 如需匿名访问,运行: litloop tor_start")
|
|
222
|
+
except Exception:
|
|
223
|
+
report["tests"].append({"target": "Tor SOCKS5", "status": "unknown"})
|
|
224
|
+
|
|
225
|
+
# Check CloakBrowser
|
|
226
|
+
try:
|
|
227
|
+
from ..browser.engine import is_available as browser_avail
|
|
228
|
+
if browser_avail(config):
|
|
229
|
+
report["tests"].append({"target": "CloakBrowser", "status": "available"})
|
|
230
|
+
else:
|
|
231
|
+
report["tests"].append({"target": "CloakBrowser", "status": "not installed"})
|
|
232
|
+
report["recommendations"].append(
|
|
233
|
+
"CloakBrowser 不可用 → 如遇 Cloudflare 封锁,运行: pip install cloakbrowser"
|
|
234
|
+
)
|
|
235
|
+
except Exception:
|
|
236
|
+
report["tests"].append({"target": "CloakBrowser", "status": "not installed"})
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# Summary
|
|
240
|
+
failed_tests = [t for t in report["tests"] if t.get("dns") == "failed" or t.get("tcp") == "failed"]
|
|
241
|
+
if not failed_tests and not active_proxy:
|
|
242
|
+
report["summary"] = "网络正常,直连可用"
|
|
243
|
+
elif not failed_tests and active_proxy:
|
|
244
|
+
report["summary"] = "网络正常,代理工作正常"
|
|
245
|
+
elif failed_tests and active_proxy:
|
|
246
|
+
report["summary"] = "部分连接失败,代理可能需要调整"
|
|
247
|
+
else:
|
|
248
|
+
report["summary"] = "网络受限,强烈建议配置代理"
|
|
249
|
+
|
|
250
|
+
return report
|