litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/browser/pdf.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Browser-side helpers shared by the institutional download flows (CARSI, WebVPN).
|
|
2
|
+
|
|
3
|
+
Publisher PDFs opened in a visible browser often end up in Chrome's inline PDF
|
|
4
|
+
viewer (e.g. ScienceDirect: pdfft -> pdf.sciencedirectassets.com). The response
|
|
5
|
+
listener cannot read that body, an out-of-browser request is challenged by
|
|
6
|
+
Cloudflare, and a page fetch() cannot read a cross-origin redirect. So the browser
|
|
7
|
+
itself must open the URL, and the body is read while the response is paused via
|
|
8
|
+
the CDP Fetch domain.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import base64
|
|
14
|
+
import time
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from ..log import get_logger
|
|
18
|
+
|
|
19
|
+
log = get_logger()
|
|
20
|
+
|
|
21
|
+
_LOADING_TITLES = ("loading", "请稍候", "please wait", "just a moment", "redirecting")
|
|
22
|
+
_FINAL_BLOCK_MARKERS = ("浏览器不支持", "browser is not supported", "unsupported browser",
|
|
23
|
+
"your browser is not supported")
|
|
24
|
+
|
|
25
|
+
PAGE_PDF_LINK_JS = """
|
|
26
|
+
(doi) => {
|
|
27
|
+
// "Recommended articles" lists may carry other papers' PDF links: prefer a
|
|
28
|
+
// link naming this DOI, else fall back to the first one in page order
|
|
29
|
+
// (the article's own button comes first; some URLs, e.g. ACS
|
|
30
|
+
// /article-pdf/.../am5c16640.pdf, do not contain the DOI).
|
|
31
|
+
const want = (doi || '').toLowerCase();
|
|
32
|
+
const suffix = want.includes('/') ? want.split('/').slice(1).join('/') : '';
|
|
33
|
+
let first = null;
|
|
34
|
+
for (const a of document.querySelectorAll('a[href]')) {
|
|
35
|
+
const href = a.getAttribute('href') || '';
|
|
36
|
+
const low = href.toLowerCase();
|
|
37
|
+
if (low.includes('pdfft') || low.includes('/doi/pdf') || low.includes('/pdfdirect/')
|
|
38
|
+
|| low.includes('/content/pdf/') || low.includes('/stamp/stamp.jsp')
|
|
39
|
+
// Silverchair platforms (RSC since 2026, OUP): /article-pdf/doi/...pdf
|
|
40
|
+
|| low.includes('/article-pdf/') || low.includes('/content/articlepdf/')
|
|
41
|
+
// SciOpen: /local/article_pdf/10.26599/JAC....pdf
|
|
42
|
+
|| low.includes('/article_pdf/')) {
|
|
43
|
+
let abs;
|
|
44
|
+
try { abs = new URL(href, location.href).href; } catch (e) { continue; }
|
|
45
|
+
if (!want) return abs;
|
|
46
|
+
first = first || abs;
|
|
47
|
+
let plain = abs.toLowerCase();
|
|
48
|
+
try { plain = decodeURIComponent(plain); } catch (e) {}
|
|
49
|
+
if (plain.includes(want) || (suffix && plain.includes(suffix))) return abs;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (first) return first;
|
|
53
|
+
// The button itself: "View PDF" / "Download PDF" whose link carries none
|
|
54
|
+
// of the patterns above (2026-10-17: ScienceDirect reached outside the
|
|
55
|
+
// WebVPN showed "View PDF" yet no link was found).
|
|
56
|
+
for (const a of document.querySelectorAll('a[href]')) {
|
|
57
|
+
const text = (a.innerText || a.getAttribute('aria-label') || '').trim().toLowerCase();
|
|
58
|
+
if (/^(view|download|get) pdf\\b|^pdf$|查看 ?pdf|下载 ?pdf/.test(text)) {
|
|
59
|
+
try { return new URL(a.getAttribute('href'), location.href).href; } catch (e) {}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_GONE_MARKERS = ("driver is gone", "has been closed", "browser has disconnected", "target closed",
|
|
68
|
+
"connection closed")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def browser_gone(exc: BaseException) -> bool:
|
|
72
|
+
"""The page's browser is no longer there (killed by a skip or a cancel, or closed).
|
|
73
|
+
|
|
74
|
+
The waiting helpers below used to swallow this like any other error and keep
|
|
75
|
+
waiting out their whole budget (2026-10-19: a skipped paper took 101 s to
|
|
76
|
+
stop, three helpers of ~35 s each on a browser already killed)."""
|
|
77
|
+
text = str(exc).lower()
|
|
78
|
+
return any(m in text for m in _GONE_MARKERS)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def wait_page_settled(page: Any, max_wait_s: float = 30.0) -> tuple[str, str]:
|
|
82
|
+
"""Wait until the page leaves interstitial/redirect states; return (title, url).
|
|
83
|
+
|
|
84
|
+
Interstitials such as WebVPN's "Loading https://..." page or a Cloudflare
|
|
85
|
+
"请稍候" page carry no auth keywords, so a single early look at the title
|
|
86
|
+
misreads where the browser is really heading.
|
|
87
|
+
"""
|
|
88
|
+
title, url = "", ""
|
|
89
|
+
waited = 0.0
|
|
90
|
+
while waited < max_wait_s:
|
|
91
|
+
try:
|
|
92
|
+
page.wait_for_load_state("domcontentloaded", timeout=5000)
|
|
93
|
+
except Exception as exc:
|
|
94
|
+
if browser_gone(exc):
|
|
95
|
+
return "", ""
|
|
96
|
+
try:
|
|
97
|
+
title = page.title() or ""
|
|
98
|
+
url = page.url or ""
|
|
99
|
+
except Exception as exc:
|
|
100
|
+
if browser_gone(exc):
|
|
101
|
+
return "", ""
|
|
102
|
+
title, url = "", ""
|
|
103
|
+
low = title.strip().lower()
|
|
104
|
+
if low and not any(low.startswith(t) or t in low for t in _LOADING_TITLES):
|
|
105
|
+
return title, url
|
|
106
|
+
# Cloudflare's "unsupported browser" verdict keeps the "请稍候" title
|
|
107
|
+
# forever (e.g. a challenge rewritten by a WebVPN); waiting is futile.
|
|
108
|
+
try:
|
|
109
|
+
body = (page.evaluate("document.body ? document.body.innerText.slice(0, 2000) : ''") or "").lower()
|
|
110
|
+
if any(m in body for m in _FINAL_BLOCK_MARKERS):
|
|
111
|
+
return title, url
|
|
112
|
+
except Exception:
|
|
113
|
+
pass
|
|
114
|
+
# Not page.wait_for_timeout: on a page stuck loading (2026-10-17,
|
|
115
|
+
# auth.elsevier.com reached through WebVPN) it never returned.
|
|
116
|
+
time.sleep(1.0)
|
|
117
|
+
waited += 1.0
|
|
118
|
+
return title, url
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
_TRANSIT_HOSTS = ("id.elsevier.com", "auth.elsevier.com", "linkinghub.elsevier.com",
|
|
122
|
+
"idp.", "sso.", "wayf.", "shibboleth")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def wait_url_stable(page: Any, max_wait_s: float = 45.0) -> str:
|
|
126
|
+
"""Wait until the page stops redirecting; return the final URL.
|
|
127
|
+
|
|
128
|
+
Publisher pages bounce through auth/redirect hosts (e.g. ScienceDirect ->
|
|
129
|
+
id.elsevier.com OAuth -> back) after DOMContentLoaded. Reading the DOM during
|
|
130
|
+
those hops fails with "Execution context was destroyed".
|
|
131
|
+
"""
|
|
132
|
+
prev = ""
|
|
133
|
+
waited = 0.0
|
|
134
|
+
while waited < max_wait_s:
|
|
135
|
+
try:
|
|
136
|
+
url = page.url or ""
|
|
137
|
+
state = page.evaluate("document.readyState")
|
|
138
|
+
except Exception as exc:
|
|
139
|
+
if browser_gone(exc):
|
|
140
|
+
return prev
|
|
141
|
+
url, state = "", ""
|
|
142
|
+
host = url.split("/")[2] if url.count("/") >= 2 else ""
|
|
143
|
+
in_transit = any(t in host for t in _TRANSIT_HOSTS)
|
|
144
|
+
# "interactive" (DOM parsed) is enough: pages with ads/trackers (e.g.
|
|
145
|
+
# Wiley) may never reach "complete", and JS-rendered PDF links are
|
|
146
|
+
# waited for separately by find_page_pdf_link.
|
|
147
|
+
if url and url == prev and state in ("interactive", "complete") and not in_transit:
|
|
148
|
+
return url
|
|
149
|
+
prev = url
|
|
150
|
+
time.sleep(1.0) # not page.wait_for_timeout: it hangs on a page stuck loading
|
|
151
|
+
waited += 1.0
|
|
152
|
+
return prev
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def find_page_pdf_link(page: Any, max_wait_s: float = 15.0, doi: str = "") -> str | None:
|
|
156
|
+
"""Return the article page's own PDF link (carries page-issued tokens), if any.
|
|
157
|
+
|
|
158
|
+
Publisher pages (e.g. ScienceDirect) render the "View PDF" link with JS after
|
|
159
|
+
DOMContentLoaded, so poll for it instead of looking once.
|
|
160
|
+
"""
|
|
161
|
+
# Wall-clock bound: an evaluate() on a page that is still loading can itself
|
|
162
|
+
# take seconds, so counting iterations under-waits on slow sites.
|
|
163
|
+
deadline = time.monotonic() + max_wait_s
|
|
164
|
+
while time.monotonic() < deadline:
|
|
165
|
+
try:
|
|
166
|
+
href = page.evaluate(PAGE_PDF_LINK_JS, doi or "")
|
|
167
|
+
if href:
|
|
168
|
+
return href
|
|
169
|
+
except Exception as exc:
|
|
170
|
+
if browser_gone(exc):
|
|
171
|
+
return None
|
|
172
|
+
time.sleep(0.5) # not page.wait_for_timeout: it hangs on a page stuck loading
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _is_bounce_to_html(ev: dict[str, Any], pdf_url: str) -> bool:
|
|
177
|
+
"""A 2xx HTML document at a URL without "pdf" in it, reached while loading a PDF URL.
|
|
178
|
+
|
|
179
|
+
ScienceDirect's /pdfft first answers with an HTML check page but stays on a
|
|
180
|
+
pdfft URL, so it is not mistaken for a bounce.
|
|
181
|
+
"""
|
|
182
|
+
status = ev.get("responseStatusCode") or 0
|
|
183
|
+
headers = {h["name"].lower(): h["value"] for h in ev.get("responseHeaders") or []}
|
|
184
|
+
req_url = str((ev.get("request") or {}).get("url", ""))
|
|
185
|
+
return (200 <= status < 300 and "text/html" in headers.get("content-type", "").lower()
|
|
186
|
+
and "pdf" not in req_url.lower() and req_url != pdf_url)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def capture_pdf_via_cdp(context: Any, page: Any, url: str, timeout_s: float = 90.0) -> bytes | None:
|
|
190
|
+
"""Navigate ``page`` to ``url`` and return the PDF body the browser receives."""
|
|
191
|
+
if not url or not url.startswith("http"):
|
|
192
|
+
return None
|
|
193
|
+
try:
|
|
194
|
+
cdp = context.new_cdp_session(page)
|
|
195
|
+
except Exception as exc:
|
|
196
|
+
log.info(f" [browser-pdf] CDP session unavailable: {exc}")
|
|
197
|
+
return None
|
|
198
|
+
|
|
199
|
+
bodies: list[bytes] = []
|
|
200
|
+
# The sync API cannot call back into the browser from an event handler, so the
|
|
201
|
+
# handler only queues paused requests; the loop below (pumping events via
|
|
202
|
+
# wait_for_timeout) reads and releases them.
|
|
203
|
+
paused: list[dict[str, Any]] = []
|
|
204
|
+
|
|
205
|
+
def _handle(ev: dict[str, Any]) -> None:
|
|
206
|
+
rid = ev["requestId"]
|
|
207
|
+
try:
|
|
208
|
+
status = ev.get("responseStatusCode") or 0
|
|
209
|
+
headers = {h["name"].lower(): h["value"] for h in ev.get("responseHeaders") or []}
|
|
210
|
+
ctype = headers.get("content-type", "").lower()
|
|
211
|
+
if 200 <= status < 300 and ("pdf" in ctype or "octet-stream" in ctype):
|
|
212
|
+
got = cdp.send("Fetch.getResponseBody", {"requestId": rid})
|
|
213
|
+
raw = got.get("body", "")
|
|
214
|
+
bodies.append(base64.b64decode(raw) if got.get("base64Encoded") else raw.encode("latin-1"))
|
|
215
|
+
except Exception as exc:
|
|
216
|
+
log.info(f" [browser-pdf] CDP body read error: {exc}")
|
|
217
|
+
finally:
|
|
218
|
+
try:
|
|
219
|
+
cdp.send("Fetch.continueRequest", {"requestId": rid})
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
try:
|
|
224
|
+
cdp.on("Fetch.requestPaused", lambda ev: paused.append(ev))
|
|
225
|
+
cdp.send("Fetch.enable", {"patterns": [
|
|
226
|
+
{"urlPattern": "*", "resourceType": "Document", "requestStage": "Response"}
|
|
227
|
+
]})
|
|
228
|
+
try:
|
|
229
|
+
page.evaluate("(u) => { window.location.href = u; }", url)
|
|
230
|
+
except Exception:
|
|
231
|
+
pass # context may be torn down by the navigation itself
|
|
232
|
+
# Large PDFs behind ScienceDirect's crasolve check can take ~1 min; bound
|
|
233
|
+
# the wait by wall-clock time rather than by iterations.
|
|
234
|
+
deadline = time.monotonic() + timeout_s
|
|
235
|
+
bounced_at = 0.0
|
|
236
|
+
while time.monotonic() < deadline:
|
|
237
|
+
page.wait_for_timeout(500)
|
|
238
|
+
while paused:
|
|
239
|
+
ev = paused.pop(0)
|
|
240
|
+
if _is_bounce_to_html(ev, url):
|
|
241
|
+
bounced_at = bounced_at or time.monotonic()
|
|
242
|
+
_handle(ev)
|
|
243
|
+
if bodies:
|
|
244
|
+
break
|
|
245
|
+
# The PDF URL redirected to an ordinary page (no entitlement: Nature
|
|
246
|
+
# sends .pdf back to the article). Waiting the full timeout for a
|
|
247
|
+
# PDF that will never come cost 90 s per attempt.
|
|
248
|
+
if bounced_at and time.monotonic() - bounced_at > 3:
|
|
249
|
+
log.info(f" [browser-pdf] PDF URL redirected to an HTML page, not entitled: {page.url[:60]}")
|
|
250
|
+
break
|
|
251
|
+
finally:
|
|
252
|
+
try:
|
|
253
|
+
cdp.send("Fetch.disable")
|
|
254
|
+
cdp.detach()
|
|
255
|
+
except Exception:
|
|
256
|
+
pass
|
|
257
|
+
|
|
258
|
+
for body in bodies:
|
|
259
|
+
if len(body) > 5000 and body[:5] == b"%PDF-":
|
|
260
|
+
log.info(f" [browser-pdf] PDF captured via CDP: {len(body)} bytes")
|
|
261
|
+
return body
|
|
262
|
+
log.info(f" [browser-pdf] no PDF body captured for {url[:60]} (now at {page.url[:60]})")
|
|
263
|
+
return None
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Browser profile session diagnostics without exposing cookie values."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import sqlite3
|
|
8
|
+
import tempfile
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
DATA_DIR = Path(os.environ.get("LITLOOP_DATA_DIR", str(Path.home() / ".litloop")))
|
|
14
|
+
|
|
15
|
+
DEFAULT_SESSION_DOMAINS = (
|
|
16
|
+
"tsinghua.edu.cn",
|
|
17
|
+
"id.tsinghua.edu.cn",
|
|
18
|
+
"idp.tsinghua.edu.cn",
|
|
19
|
+
"openathens.net",
|
|
20
|
+
"wayfinder.openathens.net",
|
|
21
|
+
"connect.openathens.net",
|
|
22
|
+
"login.openathens.net",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
CHROME_EPOCH_OFFSET_SECONDS = 11_644_473_600
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def chrome_time_to_iso(value: int | None) -> str:
|
|
29
|
+
"""Convert Chromium cookie expires_utc to ISO text."""
|
|
30
|
+
if not value:
|
|
31
|
+
return "session"
|
|
32
|
+
try:
|
|
33
|
+
seconds = (int(value) / 1_000_000) - CHROME_EPOCH_OFFSET_SECONDS
|
|
34
|
+
return datetime.fromtimestamp(seconds, tz=timezone.utc).isoformat(timespec="seconds")
|
|
35
|
+
except Exception:
|
|
36
|
+
return ""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def candidate_profile_dirs(config: dict | None = None, *, workspace: Path | None = None) -> list[Path]:
|
|
40
|
+
"""Return known local browser profile candidates in priority order."""
|
|
41
|
+
config = config or {}
|
|
42
|
+
candidates = [
|
|
43
|
+
Path(config["chrome_profile_dir"]) if config.get("chrome_profile_dir") else None,
|
|
44
|
+
DATA_DIR / "browser_profiles" / "webvpn",
|
|
45
|
+
DATA_DIR / "chrome-profile",
|
|
46
|
+
]
|
|
47
|
+
if workspace is not None:
|
|
48
|
+
candidates.append(workspace / ".chrome-sciencedirect")
|
|
49
|
+
|
|
50
|
+
seen: set[str] = set()
|
|
51
|
+
out: list[Path] = []
|
|
52
|
+
for candidate in candidates:
|
|
53
|
+
if candidate is None:
|
|
54
|
+
continue
|
|
55
|
+
resolved = str(candidate)
|
|
56
|
+
if resolved in seen:
|
|
57
|
+
continue
|
|
58
|
+
seen.add(resolved)
|
|
59
|
+
out.append(candidate)
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def inspect_browser_profile(profile_dir: str | Path, domains: list[str] | tuple[str, ...]) -> dict[str, Any]:
|
|
64
|
+
"""Inspect cookie host presence for a Chromium profile.
|
|
65
|
+
|
|
66
|
+
Cookie values are never read or returned. The report only includes host names
|
|
67
|
+
and counts, enough to diagnose whether a profile likely contains a session.
|
|
68
|
+
"""
|
|
69
|
+
profile = Path(profile_dir)
|
|
70
|
+
cookies_db = profile / "Default" / "Network" / "Cookies"
|
|
71
|
+
report: dict[str, Any] = {
|
|
72
|
+
"profile_dir": str(profile),
|
|
73
|
+
"exists": profile.exists(),
|
|
74
|
+
"cookies_db": str(cookies_db),
|
|
75
|
+
"cookies_db_exists": cookies_db.exists(),
|
|
76
|
+
"domains": {},
|
|
77
|
+
"error": "",
|
|
78
|
+
}
|
|
79
|
+
if not cookies_db.exists():
|
|
80
|
+
return report
|
|
81
|
+
|
|
82
|
+
try:
|
|
83
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
84
|
+
copy_path = Path(tmp) / "Cookies"
|
|
85
|
+
shutil.copy2(cookies_db, copy_path)
|
|
86
|
+
conn = sqlite3.connect(copy_path)
|
|
87
|
+
try:
|
|
88
|
+
columns = {
|
|
89
|
+
str(row[1])
|
|
90
|
+
for row in conn.execute("pragma table_info(cookies)").fetchall()
|
|
91
|
+
}
|
|
92
|
+
has_expires = "expires_utc" in columns
|
|
93
|
+
for domain in domains:
|
|
94
|
+
if has_expires:
|
|
95
|
+
rows = conn.execute(
|
|
96
|
+
"""
|
|
97
|
+
select
|
|
98
|
+
host_key,
|
|
99
|
+
count(*),
|
|
100
|
+
sum(case when expires_utc = 0 then 1 else 0 end),
|
|
101
|
+
sum(case when expires_utc != 0 then 1 else 0 end),
|
|
102
|
+
max(expires_utc),
|
|
103
|
+
sum(case
|
|
104
|
+
when expires_utc != 0 and expires_utc < ((strftime('%s','now') + ?) * 1000000)
|
|
105
|
+
then 1 else 0 end)
|
|
106
|
+
from cookies
|
|
107
|
+
where host_key like ?
|
|
108
|
+
group by host_key
|
|
109
|
+
order by host_key
|
|
110
|
+
""",
|
|
111
|
+
(CHROME_EPOCH_OFFSET_SECONDS, f"%{domain}"),
|
|
112
|
+
).fetchall()
|
|
113
|
+
hosts = [
|
|
114
|
+
{
|
|
115
|
+
"host": row[0],
|
|
116
|
+
"cookie_count": int(row[1] or 0),
|
|
117
|
+
"session_cookie_count": int(row[2] or 0),
|
|
118
|
+
"persistent_cookie_count": int(row[3] or 0),
|
|
119
|
+
"latest_expires_utc": int(row[4] or 0),
|
|
120
|
+
"latest_expires_at": chrome_time_to_iso(int(row[4] or 0)),
|
|
121
|
+
"expired_cookie_count": int(row[5] or 0),
|
|
122
|
+
}
|
|
123
|
+
for row in rows
|
|
124
|
+
]
|
|
125
|
+
else:
|
|
126
|
+
rows = conn.execute(
|
|
127
|
+
"""
|
|
128
|
+
select host_key, count(*)
|
|
129
|
+
from cookies
|
|
130
|
+
where host_key like ?
|
|
131
|
+
group by host_key
|
|
132
|
+
order by host_key
|
|
133
|
+
""",
|
|
134
|
+
(f"%{domain}",),
|
|
135
|
+
).fetchall()
|
|
136
|
+
hosts = [
|
|
137
|
+
{
|
|
138
|
+
"host": row[0],
|
|
139
|
+
"cookie_count": int(row[1]),
|
|
140
|
+
"session_cookie_count": 0,
|
|
141
|
+
"persistent_cookie_count": 0,
|
|
142
|
+
"latest_expires_utc": 0,
|
|
143
|
+
"latest_expires_at": "",
|
|
144
|
+
"expired_cookie_count": 0,
|
|
145
|
+
}
|
|
146
|
+
for row in rows
|
|
147
|
+
]
|
|
148
|
+
report["domains"][domain] = {
|
|
149
|
+
"cookie_count": sum(int(host["cookie_count"]) for host in hosts),
|
|
150
|
+
"session_cookie_count": sum(int(host["session_cookie_count"]) for host in hosts),
|
|
151
|
+
"persistent_cookie_count": sum(int(host["persistent_cookie_count"]) for host in hosts),
|
|
152
|
+
"expired_cookie_count": sum(int(host["expired_cookie_count"]) for host in hosts),
|
|
153
|
+
"latest_expires_at": max(
|
|
154
|
+
(
|
|
155
|
+
str(host["latest_expires_at"])
|
|
156
|
+
for host in hosts
|
|
157
|
+
if host["latest_expires_at"] not in ("", "session")
|
|
158
|
+
),
|
|
159
|
+
default="",
|
|
160
|
+
),
|
|
161
|
+
"hosts": hosts,
|
|
162
|
+
}
|
|
163
|
+
finally:
|
|
164
|
+
conn.close()
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
report["error"] = f"{type(exc).__name__}: {exc}"
|
|
167
|
+
return report
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Shared browser runtime discovery for LitLoop.
|
|
2
|
+
|
|
3
|
+
The doctor in this module is intentionally diagnostic only: it does not install
|
|
4
|
+
packages, download browser binaries, launch browsers, or write browser profiles.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib.util
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import shlex
|
|
13
|
+
import shutil
|
|
14
|
+
import urllib.request
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Callable, Dict, Iterable, List, Optional
|
|
17
|
+
|
|
18
|
+
def _default_profile_dir() -> str:
|
|
19
|
+
"""Where persistent browser profiles actually live (auth.py convention)."""
|
|
20
|
+
explicit = os.environ.get("LITLOOP_CHROME_PROFILE_DIR", "").strip()
|
|
21
|
+
if explicit:
|
|
22
|
+
return explicit
|
|
23
|
+
from ..config import DATA_DIR
|
|
24
|
+
return str(Path(DATA_DIR) / "browser_profiles")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _default_cache_dir() -> str:
|
|
28
|
+
from ..config import DATA_DIR
|
|
29
|
+
return str(Path(DATA_DIR) / "cache" / "browser")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _default_system_browser_paths() -> List[str]:
|
|
33
|
+
return [
|
|
34
|
+
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
|
|
35
|
+
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
|
|
36
|
+
r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
|
|
37
|
+
r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _default_current_module_paths() -> List[str]:
|
|
42
|
+
base = Path(__file__).resolve().parent
|
|
43
|
+
return [
|
|
44
|
+
str(base / "cookies.py"),
|
|
45
|
+
str(base / "engine.py"),
|
|
46
|
+
str(base / "login.py"),
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _split_command(value: str) -> List[str]:
|
|
51
|
+
value = value.strip()
|
|
52
|
+
if not value:
|
|
53
|
+
return []
|
|
54
|
+
if value.startswith("["):
|
|
55
|
+
parsed = json.loads(value)
|
|
56
|
+
return [str(item) for item in parsed]
|
|
57
|
+
return shlex.split(value, posix=os.name != "nt")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _http_available(url: str) -> bool:
|
|
61
|
+
if not url:
|
|
62
|
+
return False
|
|
63
|
+
try:
|
|
64
|
+
request = urllib.request.Request(url.rstrip("/") + "/health", headers={"Accept": "application/json"})
|
|
65
|
+
with urllib.request.urlopen(request, timeout=0.25) as response:
|
|
66
|
+
return response.status < 500
|
|
67
|
+
except Exception:
|
|
68
|
+
return False
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class BrowserRuntimeResolver:
|
|
72
|
+
def __init__(
|
|
73
|
+
self,
|
|
74
|
+
*,
|
|
75
|
+
env: Optional[Dict[str, str]] = None,
|
|
76
|
+
path_lookup: Optional[Callable[[str], Optional[str]]] = None,
|
|
77
|
+
import_exists: Optional[Callable[[str], bool]] = None,
|
|
78
|
+
path_exists: Optional[Callable[[str], bool]] = None,
|
|
79
|
+
http_available: Optional[Callable[[str], bool]] = None,
|
|
80
|
+
current_module_paths: Optional[Iterable[str]] = None,
|
|
81
|
+
default_system_browser_paths: Optional[Iterable[str]] = None,
|
|
82
|
+
) -> None:
|
|
83
|
+
self.env = dict(os.environ if env is None else env)
|
|
84
|
+
self.path_lookup = path_lookup or shutil.which
|
|
85
|
+
self.import_exists = import_exists or (lambda name: importlib.util.find_spec(name) is not None)
|
|
86
|
+
self.path_exists = path_exists or (lambda path: Path(path).exists())
|
|
87
|
+
self.http_available = http_available or _http_available
|
|
88
|
+
self.current_module_paths = list(
|
|
89
|
+
_default_current_module_paths() if current_module_paths is None else current_module_paths
|
|
90
|
+
)
|
|
91
|
+
self.system_browser_paths = list(
|
|
92
|
+
_default_system_browser_paths() if default_system_browser_paths is None else default_system_browser_paths
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def doctor(self) -> Dict[str, Any]:
|
|
96
|
+
candidates = self._candidates()
|
|
97
|
+
selected = next((item for item in candidates if item.get("available")), None)
|
|
98
|
+
if selected:
|
|
99
|
+
result = dict(selected)
|
|
100
|
+
result["install_needed"] = False
|
|
101
|
+
result["install_hint"] = ""
|
|
102
|
+
else:
|
|
103
|
+
result = {
|
|
104
|
+
"selected": "",
|
|
105
|
+
"source": "",
|
|
106
|
+
"available": False,
|
|
107
|
+
"install_needed": True,
|
|
108
|
+
"install_hint": "No reusable browser runtime found. Suggested explicit install: pip install cloakbrowser, or configure LITLOOP_BROWSER_COMMAND.",
|
|
109
|
+
}
|
|
110
|
+
result["profile_dir"] = _default_profile_dir()
|
|
111
|
+
result["cache_dir"] = _default_cache_dir()
|
|
112
|
+
result["candidates"] = candidates
|
|
113
|
+
return result
|
|
114
|
+
|
|
115
|
+
def _candidates(self) -> List[Dict[str, Any]]:
|
|
116
|
+
candidates: List[Dict[str, Any]] = []
|
|
117
|
+
configured = self.env.get("LITLOOP_BROWSER_COMMAND", "").strip()
|
|
118
|
+
if configured:
|
|
119
|
+
candidates.append(
|
|
120
|
+
{
|
|
121
|
+
"selected": "configured_command",
|
|
122
|
+
"source": "LITLOOP_BROWSER_COMMAND",
|
|
123
|
+
"available": True,
|
|
124
|
+
"command": _split_command(configured),
|
|
125
|
+
}
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
command_path = self.path_lookup("litloop-browser")
|
|
129
|
+
candidates.append(
|
|
130
|
+
{
|
|
131
|
+
"selected": "litloop_browser_command",
|
|
132
|
+
"source": "PATH",
|
|
133
|
+
"available": bool(command_path),
|
|
134
|
+
"path": command_path or "",
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
candidates.append(
|
|
138
|
+
{
|
|
139
|
+
"selected": "litloop_browser_package",
|
|
140
|
+
"source": "python_package",
|
|
141
|
+
"available": bool(self.import_exists("litloop_browser")),
|
|
142
|
+
"package": "litloop_browser",
|
|
143
|
+
}
|
|
144
|
+
)
|
|
145
|
+
candidates.append(self._current_litloop())
|
|
146
|
+
for package in ("cloakbrowser", "playwright"):
|
|
147
|
+
candidates.append(
|
|
148
|
+
{
|
|
149
|
+
"selected": f"{package}_package",
|
|
150
|
+
"source": "python_package",
|
|
151
|
+
"available": bool(self.import_exists(package)),
|
|
152
|
+
"package": package,
|
|
153
|
+
}
|
|
154
|
+
)
|
|
155
|
+
system_browser = self._system_browser()
|
|
156
|
+
candidates.append(
|
|
157
|
+
{
|
|
158
|
+
"selected": "system_browser",
|
|
159
|
+
"source": "system_browser",
|
|
160
|
+
"available": bool(system_browser),
|
|
161
|
+
"browser_binary": system_browser or "",
|
|
162
|
+
}
|
|
163
|
+
)
|
|
164
|
+
return candidates
|
|
165
|
+
|
|
166
|
+
def _current_litloop(self) -> Dict[str, Any]:
|
|
167
|
+
modules = [path for path in self.current_module_paths if self.path_exists(path)]
|
|
168
|
+
return {
|
|
169
|
+
"selected": "litloop_browser",
|
|
170
|
+
"source": "current_litloop",
|
|
171
|
+
"available": bool(modules),
|
|
172
|
+
"modules": [path.replace("\\", "/") for path in modules],
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
def _system_browser(self) -> str:
|
|
176
|
+
for path in self.system_browser_paths:
|
|
177
|
+
if self.path_exists(path):
|
|
178
|
+
return path.replace("\\", "/")
|
|
179
|
+
for name in ("chrome", "chrome.exe", "msedge", "msedge.exe"):
|
|
180
|
+
found = self.path_lookup(name)
|
|
181
|
+
if found:
|
|
182
|
+
return found.replace("\\", "/")
|
|
183
|
+
return ""
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def doctor() -> Dict[str, Any]:
|
|
187
|
+
result = BrowserRuntimeResolver().doctor()
|
|
188
|
+
# Surface cloakbrowser version freshness. cloakbrowser iterates on
|
|
189
|
+
# anti-detection patches fast; surfacing staleness here lets users
|
|
190
|
+
# diagnose silent bot-detection failures via litloop browser-doctor.
|
|
191
|
+
try:
|
|
192
|
+
from ..cli import _cloakbrowser_version_status, CLOAKBROWSER_RECOMMENDED_MIN
|
|
193
|
+
cb_status, cb_detail = _cloakbrowser_version_status()
|
|
194
|
+
result["cloakbrowser_version"] = {
|
|
195
|
+
"status": cb_status,
|
|
196
|
+
"detail": cb_detail,
|
|
197
|
+
"recommended_min": CLOAKBROWSER_RECOMMENDED_MIN,
|
|
198
|
+
}
|
|
199
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
200
|
+
result["cloakbrowser_version"] = {"status": "unknown", "detail": str(exc)}
|
|
201
|
+
return result
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
205
|
+
print(json.dumps(doctor(), ensure_ascii=False, indent=2))
|
|
206
|
+
return 0
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
raise SystemExit(main())
|