litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""CamoFox cloud-API client — NOT the local camoufox browser backend.
|
|
2
|
+
|
|
3
|
+
``camofox_*`` config keys (``camofox_enabled``, ``camofox_url``,
|
|
4
|
+
``camofox_api_key``, ``camofox_access_key``) belong to THIS module: a
|
|
5
|
+
remote stealth-browser service for Cloudflare bypass. The local
|
|
6
|
+
anti-detect browser backend is ``camoufox`` (one "u") — see
|
|
7
|
+
``browser_backend.py`` and the ``browser_backend`` config key. The two
|
|
8
|
+
names are unfortunately close; they share no code.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import urllib3
|
|
18
|
+
|
|
19
|
+
from ..log import get_logger
|
|
20
|
+
|
|
21
|
+
log = get_logger()
|
|
22
|
+
|
|
23
|
+
_USER_ID = "litloop"
|
|
24
|
+
_SESSION_KEY = "download"
|
|
25
|
+
|
|
26
|
+
_DEFAULT_TIMEOUT = 30.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _build_headers(config: dict[str, Any]) -> dict[str, str]:
|
|
30
|
+
headers: dict[str, str] = {"Accept": "application/json"}
|
|
31
|
+
access_key = config.get("camofox_access_key", "")
|
|
32
|
+
api_key = config.get("camofox_api_key", "")
|
|
33
|
+
if access_key:
|
|
34
|
+
headers["Authorization"] = f"Bearer {access_key}"
|
|
35
|
+
elif api_key:
|
|
36
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
37
|
+
return headers
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _api_request(
|
|
41
|
+
config: dict[str, Any],
|
|
42
|
+
method: str,
|
|
43
|
+
path: str,
|
|
44
|
+
*,
|
|
45
|
+
body: dict[str, Any] | None = None,
|
|
46
|
+
params: dict[str, str] | None = None,
|
|
47
|
+
timeout: float = _DEFAULT_TIMEOUT,
|
|
48
|
+
) -> dict[str, Any]:
|
|
49
|
+
url = config.get("camofox_url", "http://localhost:9377").rstrip("/") + path
|
|
50
|
+
if params:
|
|
51
|
+
from urllib.parse import urlencode
|
|
52
|
+
url += "?" + urlencode(params)
|
|
53
|
+
|
|
54
|
+
headers = _build_headers(config)
|
|
55
|
+
body_bytes = None
|
|
56
|
+
if body is not None:
|
|
57
|
+
headers["Content-Type"] = "application/json"
|
|
58
|
+
body_bytes = json.dumps(body).encode("utf-8")
|
|
59
|
+
|
|
60
|
+
retry = urllib3.Retry(total=2, backoff_factor=1.0, status_forcelist=[503])
|
|
61
|
+
pool = urllib3.PoolManager(retries=retry)
|
|
62
|
+
resp = pool.request(
|
|
63
|
+
method, url,
|
|
64
|
+
headers=headers, body=body_bytes,
|
|
65
|
+
timeout=urllib3.Timeout(connect=timeout, read=timeout),
|
|
66
|
+
)
|
|
67
|
+
payload = json.loads(resp.data.decode("utf-8"))
|
|
68
|
+
if resp.status >= 400:
|
|
69
|
+
error_msg = payload.get("error", "Unknown error") if isinstance(payload, dict) else str(payload)
|
|
70
|
+
raise RuntimeError(f"Camofox {resp.status}: {error_msg}")
|
|
71
|
+
return payload if isinstance(payload, dict) else {"result": payload}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def is_available(config: dict[str, Any]) -> bool:
|
|
75
|
+
"""Check if camofox-browser is reachable."""
|
|
76
|
+
url = config.get("camofox_url", "")
|
|
77
|
+
if not url:
|
|
78
|
+
return False
|
|
79
|
+
try:
|
|
80
|
+
_api_request(config, "GET", "/health", timeout=5.0)
|
|
81
|
+
return True
|
|
82
|
+
except Exception:
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def solve_url(
|
|
87
|
+
url: str,
|
|
88
|
+
config: dict[str, Any],
|
|
89
|
+
*,
|
|
90
|
+
max_timeout: int = 60000,
|
|
91
|
+
) -> dict[str, Any] | None:
|
|
92
|
+
"""Fetch URL via camofox-browser. Returns dict with status/solution keys.
|
|
93
|
+
|
|
94
|
+
Returns None on failure.
|
|
95
|
+
"""
|
|
96
|
+
tab_id: str | None = None
|
|
97
|
+
timeout_sec = max_timeout / 1000.0
|
|
98
|
+
try:
|
|
99
|
+
# Create tab and navigate
|
|
100
|
+
result = _api_request(
|
|
101
|
+
config, "POST", "/tabs",
|
|
102
|
+
body={"userId": _USER_ID, "sessionKey": _SESSION_KEY, "url": url},
|
|
103
|
+
timeout=timeout_sec,
|
|
104
|
+
)
|
|
105
|
+
tab_id = result.get("tabId")
|
|
106
|
+
if not tab_id:
|
|
107
|
+
log.info("camofox: no tabId returned")
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
# Get snapshot for final URL
|
|
111
|
+
snapshot = _api_request(
|
|
112
|
+
config, "GET", f"/tabs/{tab_id}/snapshot",
|
|
113
|
+
params={"userId": _USER_ID},
|
|
114
|
+
timeout=timeout_sec,
|
|
115
|
+
)
|
|
116
|
+
final_url = snapshot.get("url", url)
|
|
117
|
+
|
|
118
|
+
# Extract HTML via JS evaluation
|
|
119
|
+
html = ""
|
|
120
|
+
try:
|
|
121
|
+
eval_result = _api_request(
|
|
122
|
+
config, "POST", f"/tabs/{tab_id}/evaluate",
|
|
123
|
+
body={"userId": _USER_ID, "expression": "document.documentElement.outerHTML"},
|
|
124
|
+
timeout=15.0,
|
|
125
|
+
)
|
|
126
|
+
html = eval_result.get("result", "")
|
|
127
|
+
except Exception as e:
|
|
128
|
+
log.info(f"camofox: HTML eval failed: {e}")
|
|
129
|
+
|
|
130
|
+
if not html:
|
|
131
|
+
snapshot_text = snapshot.get("snapshot", "")
|
|
132
|
+
html = f"<html><body><pre>{snapshot_text}</pre></body></html>"
|
|
133
|
+
|
|
134
|
+
# Extract cookies via JS
|
|
135
|
+
cookies: list[dict[str, str]] = []
|
|
136
|
+
try:
|
|
137
|
+
cookie_result = _api_request(
|
|
138
|
+
config, "POST", f"/tabs/{tab_id}/evaluate",
|
|
139
|
+
body={
|
|
140
|
+
"userId": _USER_ID,
|
|
141
|
+
"expression": (
|
|
142
|
+
"document.cookie.split(';').map(c => c.trim()).filter(Boolean).map(c => { "
|
|
143
|
+
"const i = c.indexOf('='); "
|
|
144
|
+
"return {name: c.substring(0, i), value: c.substring(i+1)}; "
|
|
145
|
+
"})"
|
|
146
|
+
),
|
|
147
|
+
},
|
|
148
|
+
timeout=10.0,
|
|
149
|
+
)
|
|
150
|
+
raw = cookie_result.get("result", [])
|
|
151
|
+
if isinstance(raw, list):
|
|
152
|
+
for c in raw:
|
|
153
|
+
if isinstance(c, dict) and c.get("name"):
|
|
154
|
+
cookies.append({"name": c["name"], "value": c.get("value", "")})
|
|
155
|
+
except Exception as e:
|
|
156
|
+
log.info(f"camofox: cookie extraction failed: {e}")
|
|
157
|
+
|
|
158
|
+
log.info(f"camofox: ok, final_url={final_url}, html_len={len(html)}")
|
|
159
|
+
return {
|
|
160
|
+
"status": "ok",
|
|
161
|
+
"solution": {
|
|
162
|
+
"url": final_url,
|
|
163
|
+
"status": 200,
|
|
164
|
+
"response": html,
|
|
165
|
+
"cookies": cookies,
|
|
166
|
+
},
|
|
167
|
+
}
|
|
168
|
+
except Exception as e:
|
|
169
|
+
log.info(f"camofox: error - {e}")
|
|
170
|
+
return None
|
|
171
|
+
finally:
|
|
172
|
+
if tab_id:
|
|
173
|
+
try:
|
|
174
|
+
_api_request(
|
|
175
|
+
config, "DELETE", f"/tabs/{tab_id}",
|
|
176
|
+
params={"userId": _USER_ID},
|
|
177
|
+
timeout=5.0,
|
|
178
|
+
)
|
|
179
|
+
except Exception:
|
|
180
|
+
pass
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def get_html(
|
|
184
|
+
url: str,
|
|
185
|
+
config: dict[str, Any],
|
|
186
|
+
*,
|
|
187
|
+
max_timeout: int = 60000,
|
|
188
|
+
) -> str | None:
|
|
189
|
+
"""Solve and return page HTML."""
|
|
190
|
+
result = solve_url(url, config, max_timeout=max_timeout)
|
|
191
|
+
if not result:
|
|
192
|
+
return None
|
|
193
|
+
return result.get("solution", {}).get("response")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def import_cookies(cookie_file: str | Path, config: dict[str, Any], *, domain_suffix: str | None = None) -> int:
|
|
197
|
+
"""Import Netscape-format cookies into camofox-browser. Returns count imported."""
|
|
198
|
+
cookie_path = Path(cookie_file)
|
|
199
|
+
if not cookie_path.exists():
|
|
200
|
+
raise FileNotFoundError(f"Cookie file not found: {cookie_path}")
|
|
201
|
+
|
|
202
|
+
text = cookie_path.read_text(encoding="utf-8")
|
|
203
|
+
cookies = _parse_netscape_cookies(text)
|
|
204
|
+
if domain_suffix:
|
|
205
|
+
cookies = [c for c in cookies if c.get("domain", "").endswith(domain_suffix)]
|
|
206
|
+
if not cookies:
|
|
207
|
+
return 0
|
|
208
|
+
|
|
209
|
+
_api_request(
|
|
210
|
+
config, "POST", f"/sessions/{_USER_ID}/cookies",
|
|
211
|
+
body={"cookies": cookies},
|
|
212
|
+
timeout=15.0,
|
|
213
|
+
)
|
|
214
|
+
log.info(f"camofox: imported {len(cookies)} cookies from {cookie_path.name}")
|
|
215
|
+
return len(cookies)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _parse_netscape_cookies(text: str) -> list[dict[str, Any]]:
|
|
219
|
+
"""Parse Netscape-format cookie file into structured cookie objects."""
|
|
220
|
+
cookies: list[dict[str, Any]] = []
|
|
221
|
+
for line in text.splitlines():
|
|
222
|
+
line = line.strip()
|
|
223
|
+
if not line or (line.startswith("#") and not line.startswith("#HttpOnly_")):
|
|
224
|
+
continue
|
|
225
|
+
http_only = False
|
|
226
|
+
if line.startswith("#HttpOnly_"):
|
|
227
|
+
http_only = True
|
|
228
|
+
line = line.removeprefix("#HttpOnly_")
|
|
229
|
+
parts = line.split("\t")
|
|
230
|
+
if len(parts) < 7:
|
|
231
|
+
continue
|
|
232
|
+
domain = parts[0]
|
|
233
|
+
cookie_path = parts[2]
|
|
234
|
+
secure = parts[3].upper() == "TRUE"
|
|
235
|
+
try:
|
|
236
|
+
expires = int(parts[4])
|
|
237
|
+
except (ValueError, IndexError):
|
|
238
|
+
expires = 0
|
|
239
|
+
name = parts[5]
|
|
240
|
+
value = "\t".join(parts[6:])
|
|
241
|
+
cookie: dict[str, Any] = {
|
|
242
|
+
"name": name, "value": value,
|
|
243
|
+
"domain": domain, "path": cookie_path,
|
|
244
|
+
"secure": secure, "expires": expires,
|
|
245
|
+
}
|
|
246
|
+
if http_only:
|
|
247
|
+
cookie["httpOnly"] = True
|
|
248
|
+
cookies.append(cookie)
|
|
249
|
+
return cookies
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def evaluate_js(
|
|
253
|
+
tab_id: str,
|
|
254
|
+
expression: str,
|
|
255
|
+
config: dict[str, Any],
|
|
256
|
+
*,
|
|
257
|
+
timeout: float = 15.0,
|
|
258
|
+
) -> Any:
|
|
259
|
+
"""Evaluate JavaScript expression in a camofox tab. Returns the JS result."""
|
|
260
|
+
result = _api_request(
|
|
261
|
+
config, "POST", f"/tabs/{tab_id}/evaluate",
|
|
262
|
+
body={"userId": _USER_ID, "expression": expression},
|
|
263
|
+
timeout=timeout,
|
|
264
|
+
)
|
|
265
|
+
return result.get("result")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def create_tab(url: str, config: dict[str, Any], *, timeout: float = 30.0) -> str | None:
|
|
269
|
+
"""Create a new camofox tab and navigate to URL. Returns tab_id or None."""
|
|
270
|
+
try:
|
|
271
|
+
result = _api_request(
|
|
272
|
+
config, "POST", "/tabs",
|
|
273
|
+
body={"userId": _USER_ID, "sessionKey": _SESSION_KEY, "url": url},
|
|
274
|
+
timeout=timeout,
|
|
275
|
+
)
|
|
276
|
+
return result.get("tabId")
|
|
277
|
+
except Exception as e:
|
|
278
|
+
log.info(f"camofox: create_tab failed - {e}")
|
|
279
|
+
return None
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def close_tab(tab_id: str, config: dict[str, Any]) -> None:
|
|
283
|
+
"""Close a camofox tab."""
|
|
284
|
+
try:
|
|
285
|
+
_api_request(
|
|
286
|
+
config, "DELETE", f"/tabs/{tab_id}",
|
|
287
|
+
params={"userId": _USER_ID},
|
|
288
|
+
timeout=5.0,
|
|
289
|
+
)
|
|
290
|
+
except Exception:
|
|
291
|
+
pass
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def navigate_tab(tab_id: str, url: str, config: dict[str, Any], *, timeout: float = 30.0) -> bool:
|
|
295
|
+
"""Navigate an existing tab to a new URL."""
|
|
296
|
+
try:
|
|
297
|
+
_api_request(
|
|
298
|
+
config, "POST", f"/tabs/{tab_id}/navigate",
|
|
299
|
+
body={"userId": _USER_ID, "url": url},
|
|
300
|
+
timeout=timeout,
|
|
301
|
+
)
|
|
302
|
+
return True
|
|
303
|
+
except Exception as e:
|
|
304
|
+
log.info(f"camofox: navigate failed - {e}")
|
|
305
|
+
return False
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _is_pdf_url(url: str) -> bool:
|
|
309
|
+
"""Check if a URL looks like a direct PDF link."""
|
|
310
|
+
lower = url.lower()
|
|
311
|
+
return (
|
|
312
|
+
lower.endswith(".pdf")
|
|
313
|
+
or "/pdf/" in lower
|
|
314
|
+
or "content/pdf" in lower
|
|
315
|
+
or "pdfdirect" in lower
|
|
316
|
+
or "/doi/pdf/" in lower
|
|
317
|
+
or "type=printable" in lower
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def fetch_url(
|
|
322
|
+
tab_id: str,
|
|
323
|
+
url: str,
|
|
324
|
+
config: dict[str, Any],
|
|
325
|
+
*,
|
|
326
|
+
timeout: float = 30.0,
|
|
327
|
+
) -> dict[str, Any] | None:
|
|
328
|
+
"""Navigate tab to URL and capture any PDF responses from the network layer."""
|
|
329
|
+
import base64
|
|
330
|
+
|
|
331
|
+
try:
|
|
332
|
+
result = _api_request(
|
|
333
|
+
config, "POST", f"/tabs/{tab_id}/fetch-url",
|
|
334
|
+
body={"userId": _USER_ID, "url": url, "timeout": int(timeout * 1000)},
|
|
335
|
+
timeout=timeout + 15,
|
|
336
|
+
)
|
|
337
|
+
except Exception as e:
|
|
338
|
+
log.info(f"camofox: fetch_url failed - {e}")
|
|
339
|
+
return None
|
|
340
|
+
|
|
341
|
+
responses = result.get("responses", [])
|
|
342
|
+
for resp in responses:
|
|
343
|
+
data = resp.get("dataBase64", "")
|
|
344
|
+
if data:
|
|
345
|
+
pdf_bytes = base64.b64decode(data)
|
|
346
|
+
if pdf_bytes[:5] == b"%PDF-" and len(pdf_bytes) > 5000:
|
|
347
|
+
return {"status": "ok", "bytes": len(pdf_bytes), "data": pdf_bytes}
|
|
348
|
+
return None
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def get_captured_responses(
|
|
352
|
+
tab_id: str,
|
|
353
|
+
config: dict[str, Any],
|
|
354
|
+
*,
|
|
355
|
+
consume: bool = True,
|
|
356
|
+
) -> list[dict[str, Any]]:
|
|
357
|
+
"""Get captured PDF responses for a tab. Optionally consume (clear) them."""
|
|
358
|
+
try:
|
|
359
|
+
result = _api_request(
|
|
360
|
+
config, "GET", f"/tabs/{tab_id}/captured-responses",
|
|
361
|
+
params={"userId": _USER_ID, "consume": str(consume).lower()},
|
|
362
|
+
timeout=10.0,
|
|
363
|
+
)
|
|
364
|
+
return result.get("responses", [])
|
|
365
|
+
except Exception as e:
|
|
366
|
+
log.info(f"camofox: get_captured_responses failed: {e}")
|
|
367
|
+
return []
|
|
368
|
+
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Keep a private copy of the installed Google Chrome on the system's version.
|
|
2
|
+
|
|
3
|
+
LitLoop can drive a copy of the user's Chrome (``browser_executable`` pointing
|
|
4
|
+
at e.g. ``~/.litloop/chrome-real/chrome.exe``) so its process has its own name
|
|
5
|
+
for proxy rules and does not collide with the user's browser. Chrome updates
|
|
6
|
+
itself; the copy does not, and an outdated build is easier for publishers to
|
|
7
|
+
flag as automation. Before each launch the copy is compared with the installed
|
|
8
|
+
Chrome and refreshed when the system has moved on.
|
|
9
|
+
|
|
10
|
+
Windows only. Never blocks a launch: any failure is logged and the existing
|
|
11
|
+
copy is used as it is.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from ..log import get_logger
|
|
23
|
+
|
|
24
|
+
logger = get_logger()
|
|
25
|
+
|
|
26
|
+
_VERSION_DIR = re.compile(r"^\d+\.\d+\.\d+\.\d+$")
|
|
27
|
+
# What a Chrome copy needs; installer leftovers (SetupMetrics, initial_preferences) are skipped.
|
|
28
|
+
_TOP_FILES = ("chrome.exe", "chrome_proxy.exe", "chrome.VisualElementsManifest.xml")
|
|
29
|
+
_RUNNING_NAME = "litloop-chrome.exe"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def system_chrome_dirs() -> list[Path]:
|
|
33
|
+
"""Install locations of Google Chrome on Windows, system-wide first."""
|
|
34
|
+
roots = [os.environ.get("ProgramFiles", r"C:\Program Files"),
|
|
35
|
+
os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"),
|
|
36
|
+
os.environ.get("LOCALAPPDATA", "")]
|
|
37
|
+
return [Path(r) / "Google" / "Chrome" / "Application" for r in roots if r]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def chrome_version(app_dir: Path) -> tuple[int, ...] | None:
|
|
41
|
+
"""Highest version folder (e.g. 155.0.8059.41) that holds a chrome.dll."""
|
|
42
|
+
try:
|
|
43
|
+
versions = [tuple(int(x) for x in p.name.split("."))
|
|
44
|
+
for p in app_dir.iterdir()
|
|
45
|
+
if p.is_dir() and _VERSION_DIR.match(p.name) and (p / "chrome.dll").exists()]
|
|
46
|
+
except OSError:
|
|
47
|
+
return None
|
|
48
|
+
return max(versions) if versions else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _system_chrome() -> tuple[Path, tuple[int, ...]] | None:
|
|
52
|
+
for d in system_chrome_dirs():
|
|
53
|
+
if (d / "chrome.exe").exists():
|
|
54
|
+
v = chrome_version(d)
|
|
55
|
+
if v:
|
|
56
|
+
return d, v
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def is_chrome_copy(exe: Path) -> bool:
|
|
61
|
+
"""A chrome.exe outside the install locations, laid out like a Chrome install."""
|
|
62
|
+
exe = Path(exe)
|
|
63
|
+
if exe.name.lower() != "chrome.exe":
|
|
64
|
+
return False
|
|
65
|
+
here = str(exe.parent.resolve()).lower()
|
|
66
|
+
if any(here == str(d.resolve()).lower() for d in system_chrome_dirs() if d.exists()):
|
|
67
|
+
return False
|
|
68
|
+
return chrome_version(exe.parent) is not None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _copy_running() -> bool:
|
|
72
|
+
try:
|
|
73
|
+
out = subprocess.run(["tasklist", "/FI", f"IMAGENAME eq {_RUNNING_NAME}", "/NH"],
|
|
74
|
+
capture_output=True, text=True, timeout=10, errors="replace").stdout or ""
|
|
75
|
+
except Exception:
|
|
76
|
+
return False
|
|
77
|
+
return _RUNNING_NAME.lower() in out.lower()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def refresh_chrome_copy(exe: str | Path, *, system: tuple[Path, tuple[int, ...]] | None = None,
|
|
81
|
+
running: bool | None = None) -> str:
|
|
82
|
+
"""Bring the copy at ``exe`` up to the installed Chrome's version.
|
|
83
|
+
|
|
84
|
+
Returns what happened: "current", "updated", "skipped-running",
|
|
85
|
+
"not-a-copy", "no-system-chrome" or "failed". The swap is atomic from the
|
|
86
|
+
caller's view: the new build is assembled beside the old one and only then
|
|
87
|
+
renamed into place, so an interrupted copy never leaves a broken browser.
|
|
88
|
+
"""
|
|
89
|
+
exe = Path(exe)
|
|
90
|
+
if os.name != "nt" and system is None:
|
|
91
|
+
return "not-a-copy"
|
|
92
|
+
if not is_chrome_copy(exe):
|
|
93
|
+
return "not-a-copy"
|
|
94
|
+
system = system or _system_chrome()
|
|
95
|
+
if not system:
|
|
96
|
+
return "no-system-chrome"
|
|
97
|
+
src_dir, sys_version = system
|
|
98
|
+
copy_dir = exe.parent
|
|
99
|
+
copy_version = chrome_version(copy_dir)
|
|
100
|
+
if copy_version and copy_version >= sys_version:
|
|
101
|
+
return "current"
|
|
102
|
+
if running if running is not None else _copy_running():
|
|
103
|
+
logger.info("chrome copy: system Chrome is %s but the copy (%s) is running; will update next time",
|
|
104
|
+
_fmt(sys_version), _fmt(copy_version))
|
|
105
|
+
return "skipped-running"
|
|
106
|
+
|
|
107
|
+
staging = copy_dir.with_name(copy_dir.name + ".new")
|
|
108
|
+
retired = copy_dir.with_name(copy_dir.name + ".old")
|
|
109
|
+
logger.info("chrome copy: updating %s from %s to %s", copy_dir, _fmt(copy_version), _fmt(sys_version))
|
|
110
|
+
try:
|
|
111
|
+
for leftover in (staging, retired):
|
|
112
|
+
if leftover.exists():
|
|
113
|
+
shutil.rmtree(leftover)
|
|
114
|
+
staging.mkdir(parents=True)
|
|
115
|
+
version_name = _fmt(sys_version)
|
|
116
|
+
shutil.copytree(src_dir / version_name, staging / version_name)
|
|
117
|
+
for name in _TOP_FILES:
|
|
118
|
+
if (src_dir / name).exists():
|
|
119
|
+
shutil.copy2(src_dir / name, staging / name)
|
|
120
|
+
copy_dir.rename(retired)
|
|
121
|
+
try:
|
|
122
|
+
staging.rename(copy_dir)
|
|
123
|
+
except OSError:
|
|
124
|
+
retired.rename(copy_dir) # put the old copy back
|
|
125
|
+
raise
|
|
126
|
+
shutil.rmtree(retired, ignore_errors=True)
|
|
127
|
+
except Exception as exc:
|
|
128
|
+
logger.warning("chrome copy: update failed (%s); keeping the existing copy", exc)
|
|
129
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
130
|
+
return "failed"
|
|
131
|
+
logger.info("chrome copy: now on %s", _fmt(sys_version))
|
|
132
|
+
return "updated"
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _fmt(version: tuple[int, ...] | None) -> str:
|
|
136
|
+
return ".".join(str(x) for x in version) if version else "unknown"
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Compatibility helpers for CloakBrowser runtime quirks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import platform
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
_CLOAKBROWSER_CACHE_ENV = "CLOAKBROWSER_CACHE_DIR"
|
|
11
|
+
_LITLOOP_CACHE_ENV = "LITLOOP_CLOAKBROWSER_CACHE_DIR"
|
|
12
|
+
_BUILTIN_CACHE_DIR = Path(__file__).resolve().parent.parent / "_browsers" / "cloakbrowser"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def configure_builtin_cloakbrowser(
|
|
16
|
+
cache_dir: str | os.PathLike[str] | None = None,
|
|
17
|
+
*,
|
|
18
|
+
create_dir: bool = True,
|
|
19
|
+
) -> Path:
|
|
20
|
+
"""Point CloakBrowser at LitLoop-PDF's project-managed browser cache."""
|
|
21
|
+
existing = os.environ.get(_CLOAKBROWSER_CACHE_ENV)
|
|
22
|
+
if existing:
|
|
23
|
+
return Path(existing)
|
|
24
|
+
|
|
25
|
+
target = Path(cache_dir or os.environ.get(_LITLOOP_CACHE_ENV, "") or _BUILTIN_CACHE_DIR)
|
|
26
|
+
target = target.expanduser().resolve()
|
|
27
|
+
if create_dir:
|
|
28
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
os.environ[_CLOAKBROWSER_CACHE_ENV] = str(target)
|
|
30
|
+
return target
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def prepare_cloakbrowser_runtime(config_module: Any | None = None) -> Path:
|
|
34
|
+
"""Configure LitLoop-PDF's CloakBrowser runtime before importing launch APIs."""
|
|
35
|
+
cache_dir = configure_builtin_cloakbrowser()
|
|
36
|
+
ensure_cloakbrowser_platform_compatible(config_module)
|
|
37
|
+
return cache_dir
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def ensure_cloakbrowser_platform_compatible(config_module: Any | None = None) -> bool:
|
|
41
|
+
"""Patch CloakBrowser platform detection when Windows reports no machine.
|
|
42
|
+
|
|
43
|
+
Some Windows Python environments return an empty string from
|
|
44
|
+
``platform.machine()``. CloakBrowser supports Windows x64, but its lookup
|
|
45
|
+
table cannot match that empty architecture value. We add the narrow missing
|
|
46
|
+
lookup entry at runtime instead of modifying the third-party package.
|
|
47
|
+
"""
|
|
48
|
+
if platform.system() != "Windows" or platform.machine():
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
config = config_module
|
|
53
|
+
if config is None:
|
|
54
|
+
from cloakbrowser import config as config # type: ignore[no-redef]
|
|
55
|
+
except Exception:
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
supported = getattr(config, "SUPPORTED_PLATFORMS", None)
|
|
59
|
+
if not isinstance(supported, dict):
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
if ("Windows", "") in supported:
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
is_64bit_windows = bool(os.environ.get("ProgramFiles(x86)")) or bool(
|
|
66
|
+
os.environ.get("PROCESSOR_ARCHITEW6432")
|
|
67
|
+
)
|
|
68
|
+
if not is_64bit_windows:
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
supported[("Windows", "")] = supported.get(("Windows", "AMD64"), "windows-x64")
|
|
72
|
+
return True
|