litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,569 @@
1
+ """CARSI (Shibboleth/SAML) federated authentication for publisher access.
2
+
3
+ Provides institutional login through CARSI federation, supporting
4
+ publishers like Elsevier, Springer Nature, Wiley, ACS, etc.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import os
11
+ import re
12
+ import threading
13
+ import time
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Any
17
+ from urllib.parse import urlparse
18
+
19
+ import requests
20
+
21
+ from ..browser.pdf import capture_pdf_via_cdp, find_page_pdf_link
22
+ from ..config import DATA_DIR
23
+ from ..log import get_logger
24
+ from ..publishers.strategies import (
25
+ _IDP_MAP,
26
+ _AUTH_KEYWORDS,
27
+ _AUTH_TITLES,
28
+ _INSTITUTION_SEARCH_SELECTORS,
29
+ _SSO_LINK_FINDER_JS,
30
+ _INSTITUTION_CLICK_JS,
31
+ )
32
+
33
+ log = get_logger()
34
+
35
+ _PUBLISHER_CONFIGS_FILE = DATA_DIR / "publisher_carsi.json"
36
+ _PKG_DATA_DIR = Path(__file__).resolve().parent.parent / "data"
37
+ _PKG_PUBLISHER_CONFIGS_FILE = _PKG_DATA_DIR / "publisher_carsi.json"
38
+
39
+
40
+ @dataclass
41
+ class PublisherCARSIConfig:
42
+ name: str
43
+ domains: list[str]
44
+ login_url: str
45
+ search_selector: str
46
+ result_selector: str
47
+ success_url_pattern: str
48
+ pdf_pattern: str
49
+
50
+
51
+ def _load_publisher_configs() -> dict[str, PublisherCARSIConfig]:
52
+ # Try package data first, then user data dir
53
+ config_file = _PKG_PUBLISHER_CONFIGS_FILE if _PKG_PUBLISHER_CONFIGS_FILE.exists() else _PUBLISHER_CONFIGS_FILE
54
+ if not config_file.exists():
55
+ return {}
56
+ data = json.loads(config_file.read_text(encoding="utf-8"))
57
+ configs = {}
58
+ for key, val in data.items():
59
+ configs[key] = PublisherCARSIConfig(**val)
60
+ return configs
61
+
62
+
63
+ def detect_publisher(url: str) -> str | None:
64
+ """Detect publisher key from a URL."""
65
+ hostname = urlparse(url).hostname or ""
66
+ configs = _load_publisher_configs()
67
+ for key, cfg in configs.items():
68
+ for domain in cfg.domains:
69
+ if domain in hostname:
70
+ return key
71
+ return None
72
+
73
+
74
+ class CARSIClient:
75
+ """Manages CARSI/Shibboleth federated authentication with academic publishers."""
76
+
77
+ _login_lock = threading.Lock()
78
+
79
+ def __init__(self, config: dict[str, Any]):
80
+ self.config = config
81
+ self._sessions: dict[str, requests.Session] = {}
82
+ self._publisher_configs = _load_publisher_configs()
83
+ self._cookie_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache"))) / "carsi_cookies"
84
+ self._cookie_dir.mkdir(parents=True, exist_ok=True)
85
+
86
+ def _cookie_path(self, publisher: str) -> Path:
87
+ return self._cookie_dir / f"{publisher}.json"
88
+
89
+ def _get_session(self, publisher: str) -> requests.Session:
90
+ if publisher not in self._sessions:
91
+ sess = requests.Session()
92
+ sess.trust_env = False
93
+ sess.headers.update({
94
+ "User-Agent": (
95
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
96
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
97
+ "Chrome/120.0.0.0 Safari/537.36"
98
+ )
99
+ })
100
+ self._sessions[publisher] = sess
101
+ return self._sessions[publisher]
102
+
103
+ def login(self, publisher: str, force: bool = False) -> bool:
104
+ """Ensure we have a valid CARSI session for the given publisher."""
105
+ with self._login_lock:
106
+ if not force and self._try_load_cookies(publisher):
107
+ log.info(f" [CARSI] Loaded saved cookies for {publisher}")
108
+ return True
109
+ log.info(f" [CARSI] No valid session for {publisher}. Opening browser...")
110
+ return self._browser_login(publisher)
111
+
112
+ def fetch(self, url: str, **kwargs) -> requests.Response | None:
113
+ """Fetch a URL using CARSI-authenticated session."""
114
+ publisher = detect_publisher(url)
115
+ if not publisher:
116
+ return None
117
+
118
+ if not self.login(publisher):
119
+ return None
120
+
121
+ sess = self._get_session(publisher)
122
+ kwargs.setdefault("timeout", 30)
123
+ kwargs.setdefault("allow_redirects", True)
124
+ try:
125
+ return sess.get(url, **kwargs)
126
+ except requests.RequestException as e:
127
+ log.warning(f" [CARSI] Fetch failed: {e}")
128
+ return None
129
+
130
+ def download_via_browser(self, doi: str, article_url: str, output_path: Path) -> dict[str, Any] | None:
131
+ """Download PDF via CloakBrowser with CARSI auth."""
132
+ return self._download_via_cloakbrowser(doi, article_url, output_path)
133
+
134
+ def _download_via_cloakbrowser(self, doi: str, article_url: str, output_path: Path) -> dict[str, Any] | None:
135
+ """Download PDF via CloakBrowser with CARSI auth. Single session: login + download."""
136
+ publisher = detect_publisher(article_url)
137
+ if not publisher:
138
+ return None
139
+ cfg = self._publisher_configs.get(publisher)
140
+ if not cfg:
141
+ return None
142
+
143
+ try:
144
+ from ..browser.backend import launch # noqa: F401
145
+ from ..browser.backend import is_available as _browser_backend_available
146
+ if not _browser_backend_available():
147
+ log.info(" [CARSI-Browser] no browser backend available")
148
+ return None
149
+ except ImportError:
150
+ log.info(" [CARSI-Browser] no browser backend available")
151
+ return None
152
+
153
+ idp_name = self.config.get("carsi_idp_name", "")
154
+ if not idp_name:
155
+ log.info(" [CARSI-Browser] No carsi_idp_name configured")
156
+ return None
157
+
158
+ idp_en = _IDP_MAP.get(idp_name, idp_name)
159
+
160
+ from ..download.pdf import is_pdf_file, success as _success
161
+
162
+ # Serialize browser opens across threads — only one browser at a time
163
+ with self._login_lock:
164
+ log.info(f" [CARSI-Browser] Opening browser for {publisher}...")
165
+ try:
166
+ from ..publishers.strategies import _visible_browser, _save_all_cookie_formats
167
+
168
+ with _visible_browser(self.config, publisher, viewport=None) as (context, page):
169
+ def _try_save_captured() -> dict[str, Any] | None:
170
+ """If a PDF was captured, save and validate it."""
171
+ if captured_pdf:
172
+ output_path.parent.mkdir(parents=True, exist_ok=True)
173
+ output_path.write_bytes(captured_pdf[-1])
174
+ if is_pdf_file(output_path):
175
+ return _success(doi, output_path, "CARSI-Browser")
176
+ return None
177
+
178
+ def _try_fetch_pdf(url: str) -> dict[str, Any] | None:
179
+ """Open a PDF URL in the page and save the PDF the browser receives."""
180
+ body = capture_pdf_via_cdp(context, page, url)
181
+ if body:
182
+ output_path.parent.mkdir(parents=True, exist_ok=True)
183
+ output_path.write_bytes(body)
184
+ if is_pdf_file(output_path):
185
+ return _success(doi, output_path, "CARSI-Browser")
186
+ return None
187
+
188
+ # Restore saved cookies if any (supplements persistent profile)
189
+ cookie_file = self._cookie_path(publisher)
190
+ if cookie_file.exists():
191
+ try:
192
+ saved = json.loads(cookie_file.read_text(encoding="utf-8"))
193
+ pw_cookies = []
194
+ for c in saved:
195
+ pw_c = {"name": c["name"], "value": c["value"], "domain": c.get("domain", ""), "path": c.get("path", "/")}
196
+ if pw_c["domain"]:
197
+ pw_cookies.append(pw_c)
198
+ if pw_cookies:
199
+ context.add_cookies(pw_cookies)
200
+ log.info(f" [CARSI-Browser] Restored {len(pw_cookies)} cookies from file")
201
+ except Exception:
202
+ pass
203
+
204
+ # Capture PDF from network
205
+ captured_pdf = []
206
+ def on_response(response):
207
+ try:
208
+ ct = response.headers.get("content-type", "")
209
+ url = response.url
210
+ is_pdf_ct = "pdf" in ct.lower() or "octet-stream" in ct.lower()
211
+ is_pdf_url = url.lower().endswith(".pdf") or "/pdfdirect/" in url or "/doi/pdf/" in url
212
+ if not (is_pdf_ct or is_pdf_url):
213
+ return
214
+ if response.status >= 400:
215
+ return
216
+ body = response.body()
217
+ if len(body) > 5000 and body[:4] == b"%PDF-":
218
+ captured_pdf.append(body)
219
+ log.info(f" [CARSI-Browser] PDF captured: {len(body)} bytes")
220
+ except Exception:
221
+ pass
222
+ page.on("response", on_response)
223
+
224
+ # Step 1: Navigate to article page first (gets Cloudflare clearance)
225
+ log.info(f" [CARSI-Browser] Loading article: {article_url[:60]}")
226
+ try:
227
+ page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
228
+ time.sleep(5)
229
+ except Exception:
230
+ pass
231
+
232
+ title = page.title()
233
+ url = page.url
234
+ log.info(f" [CARSI-Browser] Page: '{title[:40]}' {url[:60]}")
235
+ if url.startswith("chrome-error://"):
236
+ # The article did not load (network): that says nothing
237
+ # about access, so no login is asked for. One more try.
238
+ try:
239
+ page.goto(article_url, wait_until="domcontentloaded", timeout=60000)
240
+ time.sleep(3)
241
+ except Exception:
242
+ pass
243
+ if page.url.startswith("chrome-error://"):
244
+ log.info(" [CARSI-Browser] the article page did not load (network); not asking for a login")
245
+ return None
246
+
247
+ # Wait for Cloudflare challenge to resolve (visible stealth browser can pass it)
248
+ from ..network import is_cloudflare_challenge
249
+ from ..sources import note_user_waiting
250
+ for _cf_wait in range(60): # up to 5 min: the user may have to tick it
251
+ if is_cloudflare_challenge(page.title() or ""):
252
+ if _cf_wait % 6 == 0:
253
+ log.info(f" [CARSI-Browser] Cloudflare challenge detected, waiting for it or the user... ({_cf_wait * 5}s)")
254
+ note_user_waiting("verify")
255
+ try:
256
+ from ..publishers.strategies._core import _is_challenge_dead_end
257
+ if _is_challenge_dead_end(page.content()):
258
+ log.info(" [CARSI-Browser] the check cannot run in this browser; giving up")
259
+ break
260
+ except Exception:
261
+ pass
262
+ time.sleep(5)
263
+ else:
264
+ break
265
+ else:
266
+ log.info(" [CARSI-Browser] Cloudflare challenge did not resolve")
267
+
268
+ # Step 1b: Check if restored cookies already grant access
269
+ needs_login = False
270
+ try:
271
+ needs_login = page.evaluate("""
272
+ () => {
273
+ if (!document.body) return true;
274
+ const body = (document.body.innerText || '').toLowerCase();
275
+ const hasPaywall = body.includes('purchase') || body.includes('subscribe')
276
+ || body.includes('access through your institution')
277
+ || body.includes('sign in to access')
278
+ || body.includes('buy this article');
279
+ const hasPdf = !!document.querySelector('a[href*="pdf"], a[href*="download"], iframe[src*="pdf"]');
280
+ return hasPaywall && !hasPdf;
281
+ }
282
+ """)
283
+ except Exception as _e:
284
+ log.info(f" [CARSI-Browser] paywall check error (likely Cloudflare): {_e}")
285
+ needs_login = True
286
+
287
+ # Decide whether access is already granted from the page itself.
288
+ # (A fetch() of /pdfft cannot prove it: ScienceDirect's PDF link needs
289
+ # page-issued md5/pid params and redirects cross-origin, so such a
290
+ # probe fails even with valid entitlement.)
291
+ cookies_valid = False
292
+ if not needs_login:
293
+ try:
294
+ cookies_valid = bool(page.evaluate("""
295
+ () => {
296
+ if (!document.body) return false;
297
+ const body = (document.body.innerText || '').toLowerCase();
298
+ if (body.includes('get access') || body.includes('purchase pdf')
299
+ || body.includes('access through your institution')) return false;
300
+ for (const a of document.querySelectorAll('a')) {
301
+ const href = (a.getAttribute('href') || '').toLowerCase();
302
+ const text = (a.innerText || '').toLowerCase();
303
+ if (href.includes('pdfft') || href.includes('/doi/pdf')
304
+ || href.includes('/pdfdirect/') || text.includes('view pdf')
305
+ || text.includes('download pdf')) return true;
306
+ }
307
+ return false;
308
+ }
309
+ """))
310
+ except Exception as _e:
311
+ log.info(f" [CARSI-Browser] access check error: {_e}")
312
+ if cookies_valid:
313
+ log.info(" [CARSI-Browser] Full-text access detected, skipping login")
314
+ else:
315
+ log.info(" [CARSI-Browser] No full-text access on page, login needed")
316
+
317
+ if not cookies_valid:
318
+ # Step 2: Navigate to "Institutional login" link on article page
319
+ sso_href = page.evaluate(_SSO_LINK_FINDER_JS)
320
+ if sso_href:
321
+ log.info(f" [CARSI-Browser] Navigating to SSO: {sso_href[:80]}")
322
+ try:
323
+ page.goto(sso_href, wait_until="domcontentloaded", timeout=30000)
324
+ except Exception:
325
+ pass
326
+ else:
327
+ log.info(" [CARSI-Browser] No SSO link found, trying direct login URL...")
328
+ try:
329
+ page.goto(cfg.login_url, wait_until="domcontentloaded", timeout=30000)
330
+ except Exception:
331
+ pass
332
+
333
+ time.sleep(8)
334
+
335
+ # Step 3: Search for institution in the WAYF page
336
+ search_input = page.query_selector('#searchInstitution')
337
+ if not search_input:
338
+ for sel in _INSTITUTION_SEARCH_SELECTORS[1:]: # skip #searchInstitution (already tried)
339
+ search_input = page.query_selector(sel)
340
+ if search_input:
341
+ break
342
+
343
+ if search_input:
344
+ search_input.fill(idp_en)
345
+ time.sleep(3)
346
+ log.info(f" [CARSI-Browser] Searched for '{idp_en}'")
347
+
348
+ # Click matching institution
349
+ clicked = page.evaluate(_INSTITUTION_CLICK_JS, idp_en)
350
+ if clicked:
351
+ log.info(f" [CARSI-Browser] Selected: {clicked}")
352
+ time.sleep(5)
353
+ else:
354
+ search_input.press("Enter")
355
+ time.sleep(3)
356
+ else:
357
+ log.info(" [CARSI-Browser] No institution search box found")
358
+
359
+ # Step 4: Wait for CAS login
360
+ _ak = _AUTH_KEYWORDS
361
+ _at = _AUTH_TITLES
362
+
363
+ url = page.url
364
+ title = page.title()
365
+ if any(x in url.lower() for x in _ak) or any(x in title for x in _at):
366
+ log.info(" [CARSI-Browser] CAS login required. Please log in...")
367
+ from ..sources import note_user_waiting
368
+ for i in range(100):
369
+ note_user_waiting("login") # keep the race from closing this window mid-login
370
+ time.sleep(3)
371
+ try:
372
+ title = page.title()
373
+ url = page.url
374
+ except Exception:
375
+ return None
376
+ is_auth = any(x in title for x in _at)
377
+ is_auth_url = any(x in url.lower() for x in _ak)
378
+ if not is_auth and not is_auth_url:
379
+ # Login success - save cookies in all formats + bridge to CloakBrowser
380
+ try:
381
+ cookies = context.cookies()
382
+ _save_all_cookie_formats(cookies, publisher, self.config)
383
+ except Exception:
384
+ pass
385
+ break
386
+ else:
387
+ log.info(" [CARSI-Browser] Login timed out")
388
+ return None
389
+ else:
390
+ log.info(" [CARSI-Browser] Already authenticated")
391
+
392
+ # Step 5: Navigate to article (with CARSI auth now)
393
+ time.sleep(2)
394
+ log.info(f" [CARSI-Browser] Navigating to article: {article_url[:60]}")
395
+ try:
396
+ page.goto(article_url, wait_until="domcontentloaded", timeout=30000)
397
+ time.sleep(5)
398
+ except Exception:
399
+ pass
400
+
401
+ # Check for PDF via network capture
402
+ saved = _try_save_captured()
403
+ if saved:
404
+ return saved
405
+
406
+ # Step 5b: Fetch the page's own PDF link (carries page-issued tokens,
407
+ # e.g. ScienceDirect's /pdfft?md5=...&pid=...)
408
+ page_pdf_href = find_page_pdf_link(page, doi=doi)
409
+ if page_pdf_href:
410
+ log.info(f" [CARSI-Browser] Page PDF link: {page_pdf_href[:80]}")
411
+ saved = _try_fetch_pdf(page_pdf_href)
412
+ if saved:
413
+ return saved
414
+
415
+ # Step 6: Try direct PDF URL
416
+ pii_match = re.search(r"pii/([A-Z0-9]+)", page.url)
417
+ pii_value = pii_match.group(1) if pii_match else ""
418
+ art_match = re.search(r"/document/(\d+)", page.url)
419
+ article_id = art_match.group(1) if art_match else ""
420
+ doi_suffix = doi.split("/", 1)[1] if "/" in doi else ""
421
+ pdf_pattern = (
422
+ cfg.pdf_pattern.replace("{doi}", doi)
423
+ .replace("{pii}", pii_value)
424
+ .replace("{article_id}", article_id)
425
+ .replace("{doi_suffix}", doi_suffix)
426
+ )
427
+ if pdf_pattern and not pdf_pattern.startswith("http"):
428
+ pdf_url = f"https://{cfg.domains[0]}{pdf_pattern}"
429
+ else:
430
+ pdf_url = pdf_pattern
431
+
432
+ # A leftover "{...}" means a placeholder we could not fill
433
+ # (e.g. {article_id} on a non-IEEE page) — the URL would be
434
+ # sent with literal braces and fail downstream (#54).
435
+ if pdf_url and "{" not in pdf_url:
436
+ log.info(f" [CARSI-Browser] Trying PDF: {pdf_url[:80]}")
437
+ captured_pdf.clear()
438
+ try:
439
+ page.goto(pdf_url, wait_until="commit", timeout=30000)
440
+ time.sleep(5)
441
+ except Exception:
442
+ pass
443
+ saved = _try_save_captured() or _try_fetch_pdf(page.url)
444
+ if saved:
445
+ return saved
446
+
447
+ # Step 7: Find PDF link in HTML
448
+ from ..download.pdf import extract_pdf_url_from_html
449
+ html = page.content()
450
+ found_pdf = extract_pdf_url_from_html(html, page.url)
451
+ if found_pdf:
452
+ log.info(f" [CARSI-Browser] Found PDF link: {found_pdf[:80]}")
453
+ captured_pdf.clear()
454
+ try:
455
+ page.goto(found_pdf, wait_until="commit", timeout=30000)
456
+ time.sleep(5)
457
+ except Exception:
458
+ pass
459
+ saved = _try_save_captured() or _try_fetch_pdf(page.url)
460
+ if saved:
461
+ return saved
462
+
463
+ # Step 8: Click PDF button
464
+ click_result = page.evaluate("""
465
+ () => {
466
+ const links = document.querySelectorAll('a');
467
+ for (const a of links) {
468
+ const href = (a.getAttribute('href') || '').toLowerCase();
469
+ const text = (a.innerText || '').toLowerCase();
470
+ if ((href.includes('pdf') || href.includes('download')) && !href.includes('supplement')) {
471
+ if (text.includes('pdf') || text.includes('download')) {
472
+ a.click();
473
+ return a.href;
474
+ }
475
+ }
476
+ }
477
+ return null;
478
+ }
479
+ """)
480
+ if click_result:
481
+ log.info(f" [CARSI-Browser] Clicked: {str(click_result)[:80]}")
482
+ time.sleep(8)
483
+ saved = _try_save_captured() or _try_fetch_pdf(page.url)
484
+ if saved:
485
+ return saved
486
+
487
+ log.info(f" [CARSI-Browser] No PDF found. Title: {page.title()[:40]} URL: {page.url[:60]}")
488
+ return None
489
+
490
+ except Exception as e:
491
+ log.info(f" [CARSI-Browser] Error: {e}")
492
+ return None
493
+
494
+ def _try_load_cookies(self, publisher: str) -> bool:
495
+ cookie_file = self._cookie_path(publisher)
496
+ if not cookie_file.exists():
497
+ return False
498
+ try:
499
+ cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
500
+ except (json.JSONDecodeError, OSError):
501
+ return False
502
+
503
+ sess = self._get_session(publisher)
504
+ for cookie in cookies:
505
+ sess.cookies.set(
506
+ cookie["name"],
507
+ cookie["value"],
508
+ domain=cookie.get("domain", ""),
509
+ path=cookie.get("path", "/"),
510
+ )
511
+ return self._validate_session(publisher)
512
+
513
+ def _validate_session(self, publisher: str) -> bool:
514
+ cfg = self._publisher_configs.get(publisher)
515
+ if not cfg:
516
+ return False
517
+ sess = self._get_session(publisher)
518
+
519
+ # Check cookie file freshness — accept if < 24h old
520
+ cookie_file = self._cookie_path(publisher)
521
+ try:
522
+ age_hours = (time.time() - os.path.getmtime(cookie_file)) / 3600
523
+ if age_hours > 24:
524
+ log.info(f" [CARSI] Cookies for {publisher} expired ({age_hours:.1f}h old)")
525
+ return False
526
+ except OSError:
527
+ return False
528
+
529
+ # Validate by hitting a publisher page that requires auth
530
+ # Use the main domain, not login_url (which always contains "login")
531
+ try:
532
+ test_url = f"https://{cfg.domains[0]}/"
533
+ resp = sess.get(test_url, timeout=15, allow_redirects=True)
534
+ # If we get redirected to a SSO/CAS/WAYF page, session is invalid
535
+ url_lower = resp.url.lower()
536
+ sso_keywords = ("wayf", "shibboleth", "saml", "idp.bayern", "passport")
537
+ if any(k in url_lower for k in sso_keywords):
538
+ return False
539
+ # Cloudflare-fronted publishers (ScienceDirect, Wiley, ...) challenge
540
+ # every non-browser request, so this probe cannot tell a live session
541
+ # from a dead one. Trust the freshness check above; the browser flow
542
+ # verifies access on the article page itself.
543
+ server = resp.headers.get("server", "").lower()
544
+ if resp.headers.get("cf-mitigated") == "challenge" or (
545
+ resp.status_code in (403, 503) and "cloudflare" in server):
546
+ log.info(f" [CARSI] {publisher}: Cloudflare blocks the HTTP probe; using fresh cookies")
547
+ return True
548
+ return resp.status_code == 200
549
+ except requests.RequestException:
550
+ return False
551
+
552
+ def _browser_login(self, publisher: str) -> bool:
553
+ """Login via CARSI using CloakBrowser."""
554
+ cfg = self._publisher_configs.get(publisher)
555
+ if not cfg:
556
+ log.error(f" [CARSI] Unknown publisher: {publisher}")
557
+ return False
558
+
559
+ try:
560
+ from ..browser.login import carsi_login
561
+ return carsi_login(publisher, self.config, login_url=cfg.login_url, domains=cfg.domains)
562
+ except Exception as exc:
563
+ log.error(f" [CARSI] CloakBrowser login failed: {exc}")
564
+ return False
565
+
566
+ def close(self):
567
+ for sess in self._sessions.values():
568
+ sess.close()
569
+ self._sessions.clear()
@@ -0,0 +1,63 @@
1
+ """Config adapter wrapping the flat config dict for institutional workflows."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+
8
+ class ConfigAdapter:
9
+ """Dict-compatible wrapper around litloop config.
10
+
11
+ Provides ``.load()`` as the canonical entry point and exposes
12
+ ``._config`` for direct dict access when needed (e.g. setting
13
+ ``output_dir``). Implements enough of the ``dict`` protocol so
14
+ that downstream consumers (``PaperFetcher``, ``EZProxyAuth``,
15
+ ``WebVPNAuth``, ``CARSIClient``, ...) can treat it as a plain
16
+ config dict.
17
+ """
18
+
19
+ def __init__(self) -> None:
20
+ from ..config import load_config
21
+
22
+ self._config: dict[str, Any] = load_config()
23
+
24
+ @classmethod
25
+ def load(cls) -> ConfigAdapter:
26
+ """Return a fully initialised adapter (same as ``ConfigAdapter()``)."""
27
+ return cls()
28
+
29
+ @property
30
+ def chrome_profile_dir(self) -> str:
31
+ """Path to the Chrome/Chromium profile directory."""
32
+ return self._config.get("chrome_profile_dir", "")
33
+
34
+ # dict protocol – makes the adapter transparent to downstream code
35
+ def get(self, key: str, default: Any = None) -> Any:
36
+ return self._config.get(key, default)
37
+
38
+ def __getitem__(self, key: str) -> Any:
39
+ return self._config[key]
40
+
41
+ def __setitem__(self, key: str, value: Any) -> None:
42
+ self._config[key] = value
43
+
44
+ def __contains__(self, key: str) -> bool: # type: ignore[override]
45
+ return key in self._config
46
+
47
+ def __iter__(self): # type: ignore[override]
48
+ return iter(self._config)
49
+
50
+ def keys(self):
51
+ return self._config.keys()
52
+
53
+ def items(self):
54
+ return self._config.items()
55
+
56
+ def values(self):
57
+ return self._config.values()
58
+
59
+ def __len__(self) -> int:
60
+ return len(self._config)
61
+
62
+ def __repr__(self) -> str:
63
+ return f"ConfigAdapter({self._config!r})"