litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/optional.py ADDED
@@ -0,0 +1,310 @@
1
+ """Optional parts of LitLoop that the installer leaves out.
2
+
3
+ The layout model (`pymupdf-layout`, `onnxruntime`, `networkx`, about 65 MB) only
4
+ makes PDF-to-markdown tidier: without it `download/markdown.py` falls back to
5
+ plain reading order. The installer ships without it; the settings page installs
6
+ it on demand with uv into `<data dir>/packages`, and this module puts that
7
+ directory on `sys.path` so the rest of LitLoop can import it as if it had been
8
+ installed normally.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import importlib.util
14
+ import os
15
+ import shutil
16
+ import subprocess
17
+ import sys
18
+ import threading
19
+ import time
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ #: Where on-demand components go. One directory for all of them: removing one
24
+ #: file is enough to uninstall, and a component never shadows a bundled module.
25
+ COMPONENTS: dict[str, dict[str, Any]] = {
26
+ "layout": {
27
+ "label": "版面分析",
28
+ "packages": ["pymupdf-layout", "onnxruntime", "networkx"],
29
+ "module": "onnxruntime",
30
+ "size_mb": 65,
31
+ },
32
+ }
33
+
34
+ _INSTALL_LOCK = threading.Lock()
35
+ _STATUS_LOCK = threading.Lock()
36
+ _STATUS: dict[str, dict[str, Any]] = {}
37
+
38
+ _STAGES = ("准备环境", "正在下载依赖", "正在安装", "正在校验")
39
+ _STAGE_ORDER = {name: idx for idx, name in enumerate(_STAGES)}
40
+ _NETWORK_ERROR_HINTS = (
41
+ "timed out",
42
+ "timeout",
43
+ "connection",
44
+ "ssl",
45
+ "certificate",
46
+ "dns",
47
+ "name or service not known",
48
+ "temporary failure in name resolution",
49
+ "failed to fetch",
50
+ "network",
51
+ "proxy",
52
+ "403",
53
+ "404",
54
+ "502",
55
+ "503",
56
+ "index",
57
+ )
58
+
59
+
60
+ def packages_dir() -> Path:
61
+ """`<data dir>/packages`, next to the config file (LITLOOP_DATA_DIR moves both)."""
62
+ root = Path(os.environ.get("LITLOOP_DATA_DIR") or (Path.home() / ".litloop"))
63
+ return root / "packages"
64
+
65
+
66
+ def ensure_on_path() -> None:
67
+ """Make installed components importable. Called once, when LitLoop starts."""
68
+ folder = packages_dir()
69
+ entry = str(folder)
70
+ try:
71
+ if folder.is_dir() and entry not in sys.path:
72
+ sys.path.insert(0, entry)
73
+ except OSError:
74
+ pass
75
+
76
+
77
+ def installed(key: str) -> bool:
78
+ """Whether the component can be imported right now."""
79
+ spec = COMPONENTS.get(key) or {}
80
+ module = str(spec.get("module") or "")
81
+ if not module:
82
+ return False
83
+ ensure_on_path()
84
+ try:
85
+ return importlib.util.find_spec(module) is not None
86
+ except (ImportError, ValueError):
87
+ return False
88
+
89
+
90
+ def uv_command() -> str | None:
91
+ """uv, if it can be found: shipped next to Python, on PATH, or configured."""
92
+ for candidate in (os.environ.get("LITLOOP_UV", ""),
93
+ str(Path(sys.executable).with_name("uv.exe")),
94
+ str(Path(sys.executable).with_name("uv")),
95
+ shutil.which("uv") or ""):
96
+ if candidate and Path(candidate).is_file():
97
+ return candidate
98
+ return None
99
+
100
+
101
+ def _set_status(
102
+ key: str,
103
+ *,
104
+ state: str,
105
+ stage: str = "",
106
+ elapsed: float = 0.0,
107
+ line: str = "",
108
+ ok: bool | None = None,
109
+ log: str = "",
110
+ started_at: float | None = None,
111
+ ) -> None:
112
+ """Record the latest install progress for *key* under ``_STATUS_LOCK``."""
113
+ with _STATUS_LOCK:
114
+ _STATUS[key] = {
115
+ "key": key,
116
+ "state": state,
117
+ "stage": stage,
118
+ "elapsed": round(float(elapsed), 1),
119
+ "line": line[:160],
120
+ "ok": ok,
121
+ "log": log,
122
+ "started_at": started_at,
123
+ }
124
+
125
+
126
+ def install_status(key: str) -> dict[str, Any]:
127
+ """Return the current install status for an optional component."""
128
+ if key not in COMPONENTS:
129
+ raise KeyError(f"no such component: {key}")
130
+ with _STATUS_LOCK:
131
+ entry = dict(_STATUS.get(key) or {})
132
+ if not entry:
133
+ return {
134
+ "key": key,
135
+ "installed": installed(key),
136
+ "state": "idle",
137
+ "stage": "",
138
+ "elapsed": 0.0,
139
+ "line": "",
140
+ "ok": None,
141
+ "log": "",
142
+ }
143
+ elapsed = float(entry.get("elapsed") or 0.0)
144
+ started_at = entry.get("started_at")
145
+ if entry.get("state") == "running" and isinstance(started_at, (int, float)):
146
+ elapsed = round(max(0.0, time.monotonic() - started_at), 1)
147
+ return {
148
+ "key": key,
149
+ "installed": installed(key),
150
+ "state": str(entry.get("state") or "idle"),
151
+ "stage": str(entry.get("stage") or ""),
152
+ "elapsed": elapsed,
153
+ "line": str(entry.get("line") or ""),
154
+ "ok": entry.get("ok"),
155
+ "log": str(entry.get("log") or ""),
156
+ }
157
+
158
+
159
+ def _stage_from_line(line: str, current: str) -> str:
160
+ """Advance the install stage monotonically from a single line of uv output."""
161
+ low = line.lower()
162
+ detected = current
163
+ if any(tok in low for tok in ("installing", "unpacking", "installed", "built ")):
164
+ detected = "正在安装"
165
+ elif any(tok in low for tok in ("downloading", "fetching", "resolved ", "prepared ")):
166
+ detected = "正在下载依赖"
167
+ if _STAGE_ORDER.get(detected, 0) >= _STAGE_ORDER.get(current, 0):
168
+ return detected
169
+ return current
170
+
171
+
172
+ def _format_failure_log(returncode: int, tail: str) -> str:
173
+ """Build a clear Chinese failure reason with trailing uv output."""
174
+ low = tail.lower()
175
+ if any(hint in low for hint in _NETWORK_ERROR_HINTS):
176
+ prefix = f"网络或镜像连接失败(退出码 {returncode}),请在设置中检查网络或更换镜像源后重试"
177
+ else:
178
+ prefix = f"安装失败(uv 退出码 {returncode})"
179
+ return f"{prefix}\n{tail}" if tail else prefix
180
+
181
+
182
+ def install(key: str, *, index_url: str = "", timeout: float = 1800.0) -> dict[str, Any]:
183
+ """Download a component into `packages_dir()`. Blocking; run it in a thread.
184
+
185
+ ``index_url`` overrides the package index (a domestic mirror), since the
186
+ official PyPI is slow from some networks. Returns ``{ok, log}``: the log is
187
+ the tail of uv's output, which the settings page shows when it fails.
188
+ """
189
+ spec = COMPONENTS.get(key)
190
+ if not spec:
191
+ raise KeyError(f"no such component: {key}")
192
+ uv = uv_command()
193
+ if not uv:
194
+ msg = "找不到 uv:安装包应当自带,或在 PATH 里放一个 uv"
195
+ _set_status(key, state="failed", stage="", elapsed=0.0, line=msg, ok=False, log=msg)
196
+ return {"ok": False, "log": msg}
197
+ target = packages_dir()
198
+ target.mkdir(parents=True, exist_ok=True)
199
+ cmd = [uv, "pip", "install", "--python", sys.executable, "--target", str(target),
200
+ *spec["packages"]]
201
+ env = {**os.environ, "UV_LINK_MODE": "copy"}
202
+ if index_url:
203
+ env["UV_INDEX_URL"] = index_url
204
+ if not _INSTALL_LOCK.acquire(timeout=5):
205
+ return {"ok": False, "log": "另一个组件正在安装,稍后再试"}
206
+
207
+ started = time.monotonic()
208
+ stage = "准备环境"
209
+ lines: list[str] = []
210
+ last_line = ""
211
+ timed_out = False
212
+ _set_status(key, state="running", stage=stage, elapsed=0.0, line="", ok=None, log="", started_at=started)
213
+
214
+ try:
215
+ proc = subprocess.Popen(
216
+ cmd,
217
+ stdout=subprocess.PIPE,
218
+ stderr=subprocess.STDOUT,
219
+ text=True,
220
+ encoding="utf-8",
221
+ errors="replace",
222
+ bufsize=1,
223
+ env=env,
224
+ )
225
+ except OSError as exc:
226
+ _INSTALL_LOCK.release()
227
+ elapsed = round(time.monotonic() - started, 1)
228
+ msg = f"无法运行 uv:{exc}"
229
+ _set_status(key, state="failed", stage=stage, elapsed=elapsed, line=msg, ok=False, log=msg)
230
+ return {"ok": False, "log": msg}
231
+
232
+ def _on_timeout() -> None:
233
+ nonlocal timed_out
234
+ timed_out = True
235
+ try:
236
+ proc.kill()
237
+ except OSError:
238
+ pass
239
+
240
+ timer = threading.Timer(timeout, _on_timeout)
241
+ timer.daemon = True
242
+ timer.start()
243
+ try:
244
+ if proc.stdout is not None:
245
+ for raw_line in proc.stdout:
246
+ text_line = raw_line.rstrip("\r\n")
247
+ stripped = text_line.strip()
248
+ if stripped:
249
+ lines.append(stripped)
250
+ last_line = stripped[:160]
251
+ stage = _stage_from_line(stripped, stage)
252
+ elapsed = round(time.monotonic() - started, 1)
253
+ _set_status(
254
+ key,
255
+ state="running",
256
+ stage=stage,
257
+ elapsed=elapsed,
258
+ line=last_line,
259
+ ok=None,
260
+ log="\n".join(lines[-20:]),
261
+ started_at=started,
262
+ )
263
+ returncode = proc.wait()
264
+ finally:
265
+ timer.cancel()
266
+ if proc.stdout is not None:
267
+ try:
268
+ proc.stdout.close()
269
+ except OSError:
270
+ pass
271
+ _INSTALL_LOCK.release()
272
+
273
+ tail = "\n".join(lines[-20:]).strip()
274
+ elapsed = round(time.monotonic() - started, 1)
275
+
276
+ if timed_out:
277
+ msg = f"安装超过了 {int(timeout)} 秒,已放弃"
278
+ full_log = f"{msg}\n{tail}" if tail else msg
279
+ _set_status(key, state="failed", stage=stage, elapsed=elapsed, line=msg, ok=False, log=full_log)
280
+ return {"ok": False, "log": full_log}
281
+
282
+ if returncode != 0:
283
+ msg = _format_failure_log(returncode, tail)
284
+ _set_status(key, state="failed", stage=stage, elapsed=elapsed, line=last_line or msg, ok=False, log=msg)
285
+ return {"ok": False, "log": msg}
286
+
287
+ stage = "正在校验"
288
+ _set_status(
289
+ key,
290
+ state="running",
291
+ stage=stage,
292
+ elapsed=elapsed,
293
+ line=last_line,
294
+ ok=None,
295
+ log=tail,
296
+ started_at=started,
297
+ )
298
+ # The new directory may be new to the import system.
299
+ importlib.invalidate_caches()
300
+ ensure_on_path()
301
+ ok = installed(key)
302
+ elapsed = round(time.monotonic() - started, 1)
303
+ if ok:
304
+ log = tail or "已安装"
305
+ _set_status(key, state="done", stage=stage, elapsed=elapsed, line=last_line or "已安装", ok=True, log=log)
306
+ return {"ok": True, "log": log}
307
+
308
+ msg = f"安装命令已完成,但仍无法导入组件。\n{tail}" if tail else "安装命令已完成,但仍无法导入组件"
309
+ _set_status(key, state="failed", stage=stage, elapsed=elapsed, line=last_line or msg, ok=False, log=msg)
310
+ return {"ok": False, "log": msg}
@@ -0,0 +1 @@
1
+ """Publisher access rules, profiles, PDF routers, and download strategies."""
@@ -0,0 +1,232 @@
1
+ """Reusable publisher access catalog and live verification helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ import requests
10
+
11
+ from .pdf_router import build_pdf_candidates, discover_pdf_candidates_from_html
12
+ from .profiles import get_publisher_profile, list_publisher_profiles
13
+
14
+ CATALOG_PATH = Path(__file__).parent.parent / "data" / "publisher_access_catalog.json"
15
+ INSTITUTIONAL_IDENTITY_POLICY_PATH = Path(__file__).parent.parent / "data" / "institutional_identity_policy.json"
16
+
17
+ DEFAULT_HEADERS = {
18
+ "User-Agent": (
19
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
20
+ "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
21
+ ),
22
+ "Accept": "text/html,application/xhtml+xml,application/pdf;q=0.9,*/*;q=0.8",
23
+ }
24
+
25
+
26
+ def load_publisher_access_catalog(path: str | Path | None = None) -> dict[str, Any]:
27
+ """Load the machine-readable publisher access catalog."""
28
+ catalog_path = Path(path) if path else CATALOG_PATH
29
+ return json.loads(catalog_path.read_text(encoding="utf-8"))
30
+
31
+
32
+ def load_institutional_identity_policy(path: str | Path | None = None) -> dict[str, Any]:
33
+ """Load reusable policy for choosing institutional identity routes."""
34
+ policy_path = Path(path) if path else INSTITUTIONAL_IDENTITY_POLICY_PATH
35
+ return json.loads(policy_path.read_text(encoding="utf-8"))
36
+
37
+
38
+ def verify_publisher_access(
39
+ publisher: str,
40
+ *,
41
+ session: requests.Session | None = None,
42
+ probe_pdf: bool = True,
43
+ max_candidates: int = 4,
44
+ timeout: int = 20,
45
+ ) -> dict[str, Any]:
46
+ """Resolve a sample DOI and classify official PDF candidate behavior."""
47
+ catalog = load_publisher_access_catalog()
48
+ profile = get_publisher_profile(publisher)
49
+ key = _profile_key(publisher)
50
+ entry = catalog["publishers"][key]
51
+ doi = entry["verification"]["sample_doi"]
52
+ http = session or requests.Session()
53
+
54
+ landing = _get(
55
+ http,
56
+ f"https://doi.org/{doi}",
57
+ timeout=timeout,
58
+ headers=DEFAULT_HEADERS,
59
+ allow_redirects=True,
60
+ )
61
+ landing_url = getattr(landing, "url", "")
62
+ history_urls = [getattr(item, "url", "") for item in getattr(landing, "history", [])]
63
+ chain = [url for url in [*history_urls, landing_url] if url]
64
+ landing_status = int(getattr(landing, "status_code", 0) or 0)
65
+ landing_text = _response_text(landing)
66
+
67
+ discovered = discover_pdf_candidates_from_html(landing_text, landing_url) if landing_text else []
68
+ candidates = build_pdf_candidates(
69
+ profile,
70
+ doi,
71
+ source_url=landing_url,
72
+ discovered_urls=discovered,
73
+ )
74
+
75
+ candidate_probes: list[dict[str, Any]] = []
76
+ if probe_pdf:
77
+ for url in candidates[:max(0, max_candidates)]:
78
+ candidate_probes.append(_probe_candidate(http, url, timeout=timeout))
79
+
80
+ expected_domains = [domain.lower() for domain in entry["verification"].get("expected_domains", [])]
81
+ chain_haystack = " ".join(chain).lower()
82
+ resolved_to_expected_domain = any(domain in chain_haystack for domain in expected_domains)
83
+ observed_access = _observed_access(landing, candidate_probes)
84
+ return {
85
+ "profile_key": key,
86
+ "display_name": entry["display_name"],
87
+ "sample_doi": doi,
88
+ "landing_status": landing_status,
89
+ "landing_url": landing_url,
90
+ "redirect_chain": chain,
91
+ "resolved_to_expected_domain": resolved_to_expected_domain,
92
+ "pdf_candidates": candidates,
93
+ "candidate_probes": candidate_probes,
94
+ "observed_access": observed_access,
95
+ "identity": entry["identity"],
96
+ "persistence": entry["persistence"],
97
+ "link_characteristics": entry["link_characteristics"],
98
+ }
99
+
100
+
101
+ def verify_publishers(
102
+ publishers: list[str] | None = None,
103
+ *,
104
+ probe_pdf: bool = True,
105
+ max_candidates: int = 4,
106
+ timeout: int = 20,
107
+ ) -> list[dict[str, Any]]:
108
+ keys = publishers or list_publisher_profiles()
109
+ session = requests.Session()
110
+ return [
111
+ verify_publisher_access(
112
+ key,
113
+ session=session,
114
+ probe_pdf=probe_pdf,
115
+ max_candidates=max_candidates,
116
+ timeout=timeout,
117
+ )
118
+ for key in keys
119
+ ]
120
+
121
+
122
+ def _profile_key(publisher: str) -> str:
123
+ profile = get_publisher_profile(publisher)
124
+ for key in list_publisher_profiles():
125
+ if get_publisher_profile(key) is profile:
126
+ return key
127
+ raise ValueError(f"Unknown publisher profile: {publisher}")
128
+
129
+
130
+ def _get(session: requests.Session, url: str, **kwargs):
131
+ return session.get(url, **kwargs)
132
+
133
+
134
+ def _probe_candidate(session: requests.Session, url: str, *, timeout: int) -> dict[str, Any]:
135
+ headers = {
136
+ **DEFAULT_HEADERS,
137
+ "Accept": "application/pdf,text/html;q=0.8,*/*;q=0.5",
138
+ "Range": "bytes=0-4095",
139
+ }
140
+ try:
141
+ response = _get(
142
+ session,
143
+ url,
144
+ headers=headers,
145
+ timeout=timeout,
146
+ allow_redirects=True,
147
+ stream=True,
148
+ )
149
+ result = {
150
+ "url": url,
151
+ "final_url": getattr(response, "url", url),
152
+ "status": int(getattr(response, "status_code", 0) or 0),
153
+ "content_type": _header(response, "content-type"),
154
+ "classification": _classify_response(response),
155
+ }
156
+ close = getattr(response, "close", None)
157
+ if close:
158
+ close()
159
+ return result
160
+ except requests.RequestException as exc:
161
+ return {
162
+ "url": url,
163
+ "final_url": "",
164
+ "status": 0,
165
+ "content_type": "",
166
+ "classification": "network_error",
167
+ "error": str(exc),
168
+ }
169
+
170
+
171
+ def _classify_response(response) -> str:
172
+ status = int(getattr(response, "status_code", 0) or 0)
173
+ content_type = _header(response, "content-type").lower()
174
+ final_url = str(getattr(response, "url", "")).lower()
175
+ text = _response_text(response, limit=4000).lower()
176
+ login_markers = (
177
+ "login", "signin", "sign in", "openathens",
178
+ "institution", "shibboleth", "access through",
179
+ )
180
+ challenge_markers = ("captcha", "robot", "challenge", "perfdrive", "cloudflare", "validate.perfdrive")
181
+ if 200 <= status < 300 and "pdf" in content_type:
182
+ return "pdf_accessible"
183
+ if status in {401, 402, 403}:
184
+ if any(marker in final_url or marker in text for marker in challenge_markers):
185
+ return "challenge_or_bot_check"
186
+ return "identity_required"
187
+ if status == 404:
188
+ return "not_found"
189
+ if status >= 500:
190
+ return "server_error"
191
+ if any(marker in final_url or marker in text for marker in challenge_markers):
192
+ return "challenge_or_bot_check"
193
+ if any(marker in final_url or marker in text for marker in login_markers):
194
+ return "identity_required"
195
+ if 200 <= status < 300 and "html" in content_type:
196
+ return "html_or_reader"
197
+ if 300 <= status < 400:
198
+ return "redirect"
199
+ return "unknown"
200
+
201
+
202
+ def _observed_access(landing, candidate_probes: list[dict[str, Any]]) -> str:
203
+ classifications = [probe["classification"] for probe in candidate_probes]
204
+ if "pdf_accessible" in classifications:
205
+ return "pdf_accessible"
206
+ if "identity_required" in classifications:
207
+ return "identity_required"
208
+ if "challenge_or_bot_check" in classifications or _classify_response(landing) == "challenge_or_bot_check":
209
+ return "challenge_or_bot_check"
210
+ if not candidate_probes:
211
+ return _classify_response(landing)
212
+ if all(item == "not_found" for item in classifications):
213
+ return "not_found"
214
+ return classifications[0] if classifications else "unknown"
215
+
216
+
217
+ def _response_text(response, *, limit: int = 200_000) -> str:
218
+ content_type = _header(response, "content-type").lower()
219
+ if content_type and "html" not in content_type and "text" not in content_type and "xml" not in content_type:
220
+ return ""
221
+ try:
222
+ return str(getattr(response, "text", "") or "")[:limit]
223
+ except Exception:
224
+ return ""
225
+
226
+
227
+ def _header(response, name: str) -> str:
228
+ headers = getattr(response, "headers", {}) or {}
229
+ try:
230
+ return str(headers.get(name) or headers.get(name.title()) or "")
231
+ except AttributeError:
232
+ return ""