litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/browser/pdf.py ADDED
@@ -0,0 +1,263 @@
1
+ """Browser-side helpers shared by the institutional download flows (CARSI, WebVPN).
2
+
3
+ Publisher PDFs opened in a visible browser often end up in Chrome's inline PDF
4
+ viewer (e.g. ScienceDirect: pdfft -> pdf.sciencedirectassets.com). The response
5
+ listener cannot read that body, an out-of-browser request is challenged by
6
+ Cloudflare, and a page fetch() cannot read a cross-origin redirect. So the browser
7
+ itself must open the URL, and the body is read while the response is paused via
8
+ the CDP Fetch domain.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import base64
14
+ import time
15
+ from typing import Any
16
+
17
+ from ..log import get_logger
18
+
19
+ log = get_logger()
20
+
21
+ _LOADING_TITLES = ("loading", "请稍候", "please wait", "just a moment", "redirecting")
22
+ _FINAL_BLOCK_MARKERS = ("浏览器不支持", "browser is not supported", "unsupported browser",
23
+ "your browser is not supported")
24
+
25
+ PAGE_PDF_LINK_JS = """
26
+ (doi) => {
27
+ // "Recommended articles" lists may carry other papers' PDF links: prefer a
28
+ // link naming this DOI, else fall back to the first one in page order
29
+ // (the article's own button comes first; some URLs, e.g. ACS
30
+ // /article-pdf/.../am5c16640.pdf, do not contain the DOI).
31
+ const want = (doi || '').toLowerCase();
32
+ const suffix = want.includes('/') ? want.split('/').slice(1).join('/') : '';
33
+ let first = null;
34
+ for (const a of document.querySelectorAll('a[href]')) {
35
+ const href = a.getAttribute('href') || '';
36
+ const low = href.toLowerCase();
37
+ if (low.includes('pdfft') || low.includes('/doi/pdf') || low.includes('/pdfdirect/')
38
+ || low.includes('/content/pdf/') || low.includes('/stamp/stamp.jsp')
39
+ // Silverchair platforms (RSC since 2026, OUP): /article-pdf/doi/...pdf
40
+ || low.includes('/article-pdf/') || low.includes('/content/articlepdf/')
41
+ // SciOpen: /local/article_pdf/10.26599/JAC....pdf
42
+ || low.includes('/article_pdf/')) {
43
+ let abs;
44
+ try { abs = new URL(href, location.href).href; } catch (e) { continue; }
45
+ if (!want) return abs;
46
+ first = first || abs;
47
+ let plain = abs.toLowerCase();
48
+ try { plain = decodeURIComponent(plain); } catch (e) {}
49
+ if (plain.includes(want) || (suffix && plain.includes(suffix))) return abs;
50
+ }
51
+ }
52
+ if (first) return first;
53
+ // The button itself: "View PDF" / "Download PDF" whose link carries none
54
+ // of the patterns above (2026-10-17: ScienceDirect reached outside the
55
+ // WebVPN showed "View PDF" yet no link was found).
56
+ for (const a of document.querySelectorAll('a[href]')) {
57
+ const text = (a.innerText || a.getAttribute('aria-label') || '').trim().toLowerCase();
58
+ if (/^(view|download|get) pdf\\b|^pdf$|查看 ?pdf|下载 ?pdf/.test(text)) {
59
+ try { return new URL(a.getAttribute('href'), location.href).href; } catch (e) {}
60
+ }
61
+ }
62
+ return null;
63
+ }
64
+ """
65
+
66
+
67
+ _GONE_MARKERS = ("driver is gone", "has been closed", "browser has disconnected", "target closed",
68
+ "connection closed")
69
+
70
+
71
+ def browser_gone(exc: BaseException) -> bool:
72
+ """The page's browser is no longer there (killed by a skip or a cancel, or closed).
73
+
74
+ The waiting helpers below used to swallow this like any other error and keep
75
+ waiting out their whole budget (2026-10-19: a skipped paper took 101 s to
76
+ stop, three helpers of ~35 s each on a browser already killed)."""
77
+ text = str(exc).lower()
78
+ return any(m in text for m in _GONE_MARKERS)
79
+
80
+
81
+ def wait_page_settled(page: Any, max_wait_s: float = 30.0) -> tuple[str, str]:
82
+ """Wait until the page leaves interstitial/redirect states; return (title, url).
83
+
84
+ Interstitials such as WebVPN's "Loading https://..." page or a Cloudflare
85
+ "请稍候" page carry no auth keywords, so a single early look at the title
86
+ misreads where the browser is really heading.
87
+ """
88
+ title, url = "", ""
89
+ waited = 0.0
90
+ while waited < max_wait_s:
91
+ try:
92
+ page.wait_for_load_state("domcontentloaded", timeout=5000)
93
+ except Exception as exc:
94
+ if browser_gone(exc):
95
+ return "", ""
96
+ try:
97
+ title = page.title() or ""
98
+ url = page.url or ""
99
+ except Exception as exc:
100
+ if browser_gone(exc):
101
+ return "", ""
102
+ title, url = "", ""
103
+ low = title.strip().lower()
104
+ if low and not any(low.startswith(t) or t in low for t in _LOADING_TITLES):
105
+ return title, url
106
+ # Cloudflare's "unsupported browser" verdict keeps the "请稍候" title
107
+ # forever (e.g. a challenge rewritten by a WebVPN); waiting is futile.
108
+ try:
109
+ body = (page.evaluate("document.body ? document.body.innerText.slice(0, 2000) : ''") or "").lower()
110
+ if any(m in body for m in _FINAL_BLOCK_MARKERS):
111
+ return title, url
112
+ except Exception:
113
+ pass
114
+ # Not page.wait_for_timeout: on a page stuck loading (2026-10-17,
115
+ # auth.elsevier.com reached through WebVPN) it never returned.
116
+ time.sleep(1.0)
117
+ waited += 1.0
118
+ return title, url
119
+
120
+
121
+ _TRANSIT_HOSTS = ("id.elsevier.com", "auth.elsevier.com", "linkinghub.elsevier.com",
122
+ "idp.", "sso.", "wayf.", "shibboleth")
123
+
124
+
125
+ def wait_url_stable(page: Any, max_wait_s: float = 45.0) -> str:
126
+ """Wait until the page stops redirecting; return the final URL.
127
+
128
+ Publisher pages bounce through auth/redirect hosts (e.g. ScienceDirect ->
129
+ id.elsevier.com OAuth -> back) after DOMContentLoaded. Reading the DOM during
130
+ those hops fails with "Execution context was destroyed".
131
+ """
132
+ prev = ""
133
+ waited = 0.0
134
+ while waited < max_wait_s:
135
+ try:
136
+ url = page.url or ""
137
+ state = page.evaluate("document.readyState")
138
+ except Exception as exc:
139
+ if browser_gone(exc):
140
+ return prev
141
+ url, state = "", ""
142
+ host = url.split("/")[2] if url.count("/") >= 2 else ""
143
+ in_transit = any(t in host for t in _TRANSIT_HOSTS)
144
+ # "interactive" (DOM parsed) is enough: pages with ads/trackers (e.g.
145
+ # Wiley) may never reach "complete", and JS-rendered PDF links are
146
+ # waited for separately by find_page_pdf_link.
147
+ if url and url == prev and state in ("interactive", "complete") and not in_transit:
148
+ return url
149
+ prev = url
150
+ time.sleep(1.0) # not page.wait_for_timeout: it hangs on a page stuck loading
151
+ waited += 1.0
152
+ return prev
153
+
154
+
155
+ def find_page_pdf_link(page: Any, max_wait_s: float = 15.0, doi: str = "") -> str | None:
156
+ """Return the article page's own PDF link (carries page-issued tokens), if any.
157
+
158
+ Publisher pages (e.g. ScienceDirect) render the "View PDF" link with JS after
159
+ DOMContentLoaded, so poll for it instead of looking once.
160
+ """
161
+ # Wall-clock bound: an evaluate() on a page that is still loading can itself
162
+ # take seconds, so counting iterations under-waits on slow sites.
163
+ deadline = time.monotonic() + max_wait_s
164
+ while time.monotonic() < deadline:
165
+ try:
166
+ href = page.evaluate(PAGE_PDF_LINK_JS, doi or "")
167
+ if href:
168
+ return href
169
+ except Exception as exc:
170
+ if browser_gone(exc):
171
+ return None
172
+ time.sleep(0.5) # not page.wait_for_timeout: it hangs on a page stuck loading
173
+ return None
174
+
175
+
176
+ def _is_bounce_to_html(ev: dict[str, Any], pdf_url: str) -> bool:
177
+ """A 2xx HTML document at a URL without "pdf" in it, reached while loading a PDF URL.
178
+
179
+ ScienceDirect's /pdfft first answers with an HTML check page but stays on a
180
+ pdfft URL, so it is not mistaken for a bounce.
181
+ """
182
+ status = ev.get("responseStatusCode") or 0
183
+ headers = {h["name"].lower(): h["value"] for h in ev.get("responseHeaders") or []}
184
+ req_url = str((ev.get("request") or {}).get("url", ""))
185
+ return (200 <= status < 300 and "text/html" in headers.get("content-type", "").lower()
186
+ and "pdf" not in req_url.lower() and req_url != pdf_url)
187
+
188
+
189
+ def capture_pdf_via_cdp(context: Any, page: Any, url: str, timeout_s: float = 90.0) -> bytes | None:
190
+ """Navigate ``page`` to ``url`` and return the PDF body the browser receives."""
191
+ if not url or not url.startswith("http"):
192
+ return None
193
+ try:
194
+ cdp = context.new_cdp_session(page)
195
+ except Exception as exc:
196
+ log.info(f" [browser-pdf] CDP session unavailable: {exc}")
197
+ return None
198
+
199
+ bodies: list[bytes] = []
200
+ # The sync API cannot call back into the browser from an event handler, so the
201
+ # handler only queues paused requests; the loop below (pumping events via
202
+ # wait_for_timeout) reads and releases them.
203
+ paused: list[dict[str, Any]] = []
204
+
205
+ def _handle(ev: dict[str, Any]) -> None:
206
+ rid = ev["requestId"]
207
+ try:
208
+ status = ev.get("responseStatusCode") or 0
209
+ headers = {h["name"].lower(): h["value"] for h in ev.get("responseHeaders") or []}
210
+ ctype = headers.get("content-type", "").lower()
211
+ if 200 <= status < 300 and ("pdf" in ctype or "octet-stream" in ctype):
212
+ got = cdp.send("Fetch.getResponseBody", {"requestId": rid})
213
+ raw = got.get("body", "")
214
+ bodies.append(base64.b64decode(raw) if got.get("base64Encoded") else raw.encode("latin-1"))
215
+ except Exception as exc:
216
+ log.info(f" [browser-pdf] CDP body read error: {exc}")
217
+ finally:
218
+ try:
219
+ cdp.send("Fetch.continueRequest", {"requestId": rid})
220
+ except Exception:
221
+ pass
222
+
223
+ try:
224
+ cdp.on("Fetch.requestPaused", lambda ev: paused.append(ev))
225
+ cdp.send("Fetch.enable", {"patterns": [
226
+ {"urlPattern": "*", "resourceType": "Document", "requestStage": "Response"}
227
+ ]})
228
+ try:
229
+ page.evaluate("(u) => { window.location.href = u; }", url)
230
+ except Exception:
231
+ pass # context may be torn down by the navigation itself
232
+ # Large PDFs behind ScienceDirect's crasolve check can take ~1 min; bound
233
+ # the wait by wall-clock time rather than by iterations.
234
+ deadline = time.monotonic() + timeout_s
235
+ bounced_at = 0.0
236
+ while time.monotonic() < deadline:
237
+ page.wait_for_timeout(500)
238
+ while paused:
239
+ ev = paused.pop(0)
240
+ if _is_bounce_to_html(ev, url):
241
+ bounced_at = bounced_at or time.monotonic()
242
+ _handle(ev)
243
+ if bodies:
244
+ break
245
+ # The PDF URL redirected to an ordinary page (no entitlement: Nature
246
+ # sends .pdf back to the article). Waiting the full timeout for a
247
+ # PDF that will never come cost 90 s per attempt.
248
+ if bounced_at and time.monotonic() - bounced_at > 3:
249
+ log.info(f" [browser-pdf] PDF URL redirected to an HTML page, not entitled: {page.url[:60]}")
250
+ break
251
+ finally:
252
+ try:
253
+ cdp.send("Fetch.disable")
254
+ cdp.detach()
255
+ except Exception:
256
+ pass
257
+
258
+ for body in bodies:
259
+ if len(body) > 5000 and body[:5] == b"%PDF-":
260
+ log.info(f" [browser-pdf] PDF captured via CDP: {len(body)} bytes")
261
+ return body
262
+ log.info(f" [browser-pdf] no PDF body captured for {url[:60]} (now at {page.url[:60]})")
263
+ return None
@@ -0,0 +1,167 @@
1
+ """Browser profile session diagnostics without exposing cookie values."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import shutil
7
+ import sqlite3
8
+ import tempfile
9
+ from datetime import datetime, timezone
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ DATA_DIR = Path(os.environ.get("LITLOOP_DATA_DIR", str(Path.home() / ".litloop")))
14
+
15
+ DEFAULT_SESSION_DOMAINS = (
16
+ "tsinghua.edu.cn",
17
+ "id.tsinghua.edu.cn",
18
+ "idp.tsinghua.edu.cn",
19
+ "openathens.net",
20
+ "wayfinder.openathens.net",
21
+ "connect.openathens.net",
22
+ "login.openathens.net",
23
+ )
24
+
25
+ CHROME_EPOCH_OFFSET_SECONDS = 11_644_473_600
26
+
27
+
28
+ def chrome_time_to_iso(value: int | None) -> str:
29
+ """Convert Chromium cookie expires_utc to ISO text."""
30
+ if not value:
31
+ return "session"
32
+ try:
33
+ seconds = (int(value) / 1_000_000) - CHROME_EPOCH_OFFSET_SECONDS
34
+ return datetime.fromtimestamp(seconds, tz=timezone.utc).isoformat(timespec="seconds")
35
+ except Exception:
36
+ return ""
37
+
38
+
39
+ def candidate_profile_dirs(config: dict | None = None, *, workspace: Path | None = None) -> list[Path]:
40
+ """Return known local browser profile candidates in priority order."""
41
+ config = config or {}
42
+ candidates = [
43
+ Path(config["chrome_profile_dir"]) if config.get("chrome_profile_dir") else None,
44
+ DATA_DIR / "browser_profiles" / "webvpn",
45
+ DATA_DIR / "chrome-profile",
46
+ ]
47
+ if workspace is not None:
48
+ candidates.append(workspace / ".chrome-sciencedirect")
49
+
50
+ seen: set[str] = set()
51
+ out: list[Path] = []
52
+ for candidate in candidates:
53
+ if candidate is None:
54
+ continue
55
+ resolved = str(candidate)
56
+ if resolved in seen:
57
+ continue
58
+ seen.add(resolved)
59
+ out.append(candidate)
60
+ return out
61
+
62
+
63
+ def inspect_browser_profile(profile_dir: str | Path, domains: list[str] | tuple[str, ...]) -> dict[str, Any]:
64
+ """Inspect cookie host presence for a Chromium profile.
65
+
66
+ Cookie values are never read or returned. The report only includes host names
67
+ and counts, enough to diagnose whether a profile likely contains a session.
68
+ """
69
+ profile = Path(profile_dir)
70
+ cookies_db = profile / "Default" / "Network" / "Cookies"
71
+ report: dict[str, Any] = {
72
+ "profile_dir": str(profile),
73
+ "exists": profile.exists(),
74
+ "cookies_db": str(cookies_db),
75
+ "cookies_db_exists": cookies_db.exists(),
76
+ "domains": {},
77
+ "error": "",
78
+ }
79
+ if not cookies_db.exists():
80
+ return report
81
+
82
+ try:
83
+ with tempfile.TemporaryDirectory() as tmp:
84
+ copy_path = Path(tmp) / "Cookies"
85
+ shutil.copy2(cookies_db, copy_path)
86
+ conn = sqlite3.connect(copy_path)
87
+ try:
88
+ columns = {
89
+ str(row[1])
90
+ for row in conn.execute("pragma table_info(cookies)").fetchall()
91
+ }
92
+ has_expires = "expires_utc" in columns
93
+ for domain in domains:
94
+ if has_expires:
95
+ rows = conn.execute(
96
+ """
97
+ select
98
+ host_key,
99
+ count(*),
100
+ sum(case when expires_utc = 0 then 1 else 0 end),
101
+ sum(case when expires_utc != 0 then 1 else 0 end),
102
+ max(expires_utc),
103
+ sum(case
104
+ when expires_utc != 0 and expires_utc < ((strftime('%s','now') + ?) * 1000000)
105
+ then 1 else 0 end)
106
+ from cookies
107
+ where host_key like ?
108
+ group by host_key
109
+ order by host_key
110
+ """,
111
+ (CHROME_EPOCH_OFFSET_SECONDS, f"%{domain}"),
112
+ ).fetchall()
113
+ hosts = [
114
+ {
115
+ "host": row[0],
116
+ "cookie_count": int(row[1] or 0),
117
+ "session_cookie_count": int(row[2] or 0),
118
+ "persistent_cookie_count": int(row[3] or 0),
119
+ "latest_expires_utc": int(row[4] or 0),
120
+ "latest_expires_at": chrome_time_to_iso(int(row[4] or 0)),
121
+ "expired_cookie_count": int(row[5] or 0),
122
+ }
123
+ for row in rows
124
+ ]
125
+ else:
126
+ rows = conn.execute(
127
+ """
128
+ select host_key, count(*)
129
+ from cookies
130
+ where host_key like ?
131
+ group by host_key
132
+ order by host_key
133
+ """,
134
+ (f"%{domain}",),
135
+ ).fetchall()
136
+ hosts = [
137
+ {
138
+ "host": row[0],
139
+ "cookie_count": int(row[1]),
140
+ "session_cookie_count": 0,
141
+ "persistent_cookie_count": 0,
142
+ "latest_expires_utc": 0,
143
+ "latest_expires_at": "",
144
+ "expired_cookie_count": 0,
145
+ }
146
+ for row in rows
147
+ ]
148
+ report["domains"][domain] = {
149
+ "cookie_count": sum(int(host["cookie_count"]) for host in hosts),
150
+ "session_cookie_count": sum(int(host["session_cookie_count"]) for host in hosts),
151
+ "persistent_cookie_count": sum(int(host["persistent_cookie_count"]) for host in hosts),
152
+ "expired_cookie_count": sum(int(host["expired_cookie_count"]) for host in hosts),
153
+ "latest_expires_at": max(
154
+ (
155
+ str(host["latest_expires_at"])
156
+ for host in hosts
157
+ if host["latest_expires_at"] not in ("", "session")
158
+ ),
159
+ default="",
160
+ ),
161
+ "hosts": hosts,
162
+ }
163
+ finally:
164
+ conn.close()
165
+ except Exception as exc:
166
+ report["error"] = f"{type(exc).__name__}: {exc}"
167
+ return report
@@ -0,0 +1,210 @@
1
+ """Shared browser runtime discovery for LitLoop.
2
+
3
+ The doctor in this module is intentionally diagnostic only: it does not install
4
+ packages, download browser binaries, launch browsers, or write browser profiles.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import importlib.util
10
+ import json
11
+ import os
12
+ import shlex
13
+ import shutil
14
+ import urllib.request
15
+ from pathlib import Path
16
+ from typing import Any, Callable, Dict, Iterable, List, Optional
17
+
18
+ def _default_profile_dir() -> str:
19
+ """Where persistent browser profiles actually live (auth.py convention)."""
20
+ explicit = os.environ.get("LITLOOP_CHROME_PROFILE_DIR", "").strip()
21
+ if explicit:
22
+ return explicit
23
+ from ..config import DATA_DIR
24
+ return str(Path(DATA_DIR) / "browser_profiles")
25
+
26
+
27
+ def _default_cache_dir() -> str:
28
+ from ..config import DATA_DIR
29
+ return str(Path(DATA_DIR) / "cache" / "browser")
30
+
31
+
32
+ def _default_system_browser_paths() -> List[str]:
33
+ return [
34
+ r"C:\Program Files\Google\Chrome\Application\chrome.exe",
35
+ r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
36
+ r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
37
+ r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
38
+ ]
39
+
40
+
41
+ def _default_current_module_paths() -> List[str]:
42
+ base = Path(__file__).resolve().parent
43
+ return [
44
+ str(base / "cookies.py"),
45
+ str(base / "engine.py"),
46
+ str(base / "login.py"),
47
+ ]
48
+
49
+
50
+ def _split_command(value: str) -> List[str]:
51
+ value = value.strip()
52
+ if not value:
53
+ return []
54
+ if value.startswith("["):
55
+ parsed = json.loads(value)
56
+ return [str(item) for item in parsed]
57
+ return shlex.split(value, posix=os.name != "nt")
58
+
59
+
60
+ def _http_available(url: str) -> bool:
61
+ if not url:
62
+ return False
63
+ try:
64
+ request = urllib.request.Request(url.rstrip("/") + "/health", headers={"Accept": "application/json"})
65
+ with urllib.request.urlopen(request, timeout=0.25) as response:
66
+ return response.status < 500
67
+ except Exception:
68
+ return False
69
+
70
+
71
+ class BrowserRuntimeResolver:
72
+ def __init__(
73
+ self,
74
+ *,
75
+ env: Optional[Dict[str, str]] = None,
76
+ path_lookup: Optional[Callable[[str], Optional[str]]] = None,
77
+ import_exists: Optional[Callable[[str], bool]] = None,
78
+ path_exists: Optional[Callable[[str], bool]] = None,
79
+ http_available: Optional[Callable[[str], bool]] = None,
80
+ current_module_paths: Optional[Iterable[str]] = None,
81
+ default_system_browser_paths: Optional[Iterable[str]] = None,
82
+ ) -> None:
83
+ self.env = dict(os.environ if env is None else env)
84
+ self.path_lookup = path_lookup or shutil.which
85
+ self.import_exists = import_exists or (lambda name: importlib.util.find_spec(name) is not None)
86
+ self.path_exists = path_exists or (lambda path: Path(path).exists())
87
+ self.http_available = http_available or _http_available
88
+ self.current_module_paths = list(
89
+ _default_current_module_paths() if current_module_paths is None else current_module_paths
90
+ )
91
+ self.system_browser_paths = list(
92
+ _default_system_browser_paths() if default_system_browser_paths is None else default_system_browser_paths
93
+ )
94
+
95
+ def doctor(self) -> Dict[str, Any]:
96
+ candidates = self._candidates()
97
+ selected = next((item for item in candidates if item.get("available")), None)
98
+ if selected:
99
+ result = dict(selected)
100
+ result["install_needed"] = False
101
+ result["install_hint"] = ""
102
+ else:
103
+ result = {
104
+ "selected": "",
105
+ "source": "",
106
+ "available": False,
107
+ "install_needed": True,
108
+ "install_hint": "No reusable browser runtime found. Suggested explicit install: pip install cloakbrowser, or configure LITLOOP_BROWSER_COMMAND.",
109
+ }
110
+ result["profile_dir"] = _default_profile_dir()
111
+ result["cache_dir"] = _default_cache_dir()
112
+ result["candidates"] = candidates
113
+ return result
114
+
115
+ def _candidates(self) -> List[Dict[str, Any]]:
116
+ candidates: List[Dict[str, Any]] = []
117
+ configured = self.env.get("LITLOOP_BROWSER_COMMAND", "").strip()
118
+ if configured:
119
+ candidates.append(
120
+ {
121
+ "selected": "configured_command",
122
+ "source": "LITLOOP_BROWSER_COMMAND",
123
+ "available": True,
124
+ "command": _split_command(configured),
125
+ }
126
+ )
127
+
128
+ command_path = self.path_lookup("litloop-browser")
129
+ candidates.append(
130
+ {
131
+ "selected": "litloop_browser_command",
132
+ "source": "PATH",
133
+ "available": bool(command_path),
134
+ "path": command_path or "",
135
+ }
136
+ )
137
+ candidates.append(
138
+ {
139
+ "selected": "litloop_browser_package",
140
+ "source": "python_package",
141
+ "available": bool(self.import_exists("litloop_browser")),
142
+ "package": "litloop_browser",
143
+ }
144
+ )
145
+ candidates.append(self._current_litloop())
146
+ for package in ("cloakbrowser", "playwright"):
147
+ candidates.append(
148
+ {
149
+ "selected": f"{package}_package",
150
+ "source": "python_package",
151
+ "available": bool(self.import_exists(package)),
152
+ "package": package,
153
+ }
154
+ )
155
+ system_browser = self._system_browser()
156
+ candidates.append(
157
+ {
158
+ "selected": "system_browser",
159
+ "source": "system_browser",
160
+ "available": bool(system_browser),
161
+ "browser_binary": system_browser or "",
162
+ }
163
+ )
164
+ return candidates
165
+
166
+ def _current_litloop(self) -> Dict[str, Any]:
167
+ modules = [path for path in self.current_module_paths if self.path_exists(path)]
168
+ return {
169
+ "selected": "litloop_browser",
170
+ "source": "current_litloop",
171
+ "available": bool(modules),
172
+ "modules": [path.replace("\\", "/") for path in modules],
173
+ }
174
+
175
+ def _system_browser(self) -> str:
176
+ for path in self.system_browser_paths:
177
+ if self.path_exists(path):
178
+ return path.replace("\\", "/")
179
+ for name in ("chrome", "chrome.exe", "msedge", "msedge.exe"):
180
+ found = self.path_lookup(name)
181
+ if found:
182
+ return found.replace("\\", "/")
183
+ return ""
184
+
185
+
186
+ def doctor() -> Dict[str, Any]:
187
+ result = BrowserRuntimeResolver().doctor()
188
+ # Surface cloakbrowser version freshness. cloakbrowser iterates on
189
+ # anti-detection patches fast; surfacing staleness here lets users
190
+ # diagnose silent bot-detection failures via litloop browser-doctor.
191
+ try:
192
+ from ..cli import _cloakbrowser_version_status, CLOAKBROWSER_RECOMMENDED_MIN
193
+ cb_status, cb_detail = _cloakbrowser_version_status()
194
+ result["cloakbrowser_version"] = {
195
+ "status": cb_status,
196
+ "detail": cb_detail,
197
+ "recommended_min": CLOAKBROWSER_RECOMMENDED_MIN,
198
+ }
199
+ except Exception as exc: # pragma: no cover - defensive
200
+ result["cloakbrowser_version"] = {"status": "unknown", "detail": str(exc)}
201
+ return result
202
+
203
+
204
+ def main(argv: Optional[List[str]] = None) -> int:
205
+ print(json.dumps(doctor(), ensure_ascii=False, indent=2))
206
+ return 0
207
+
208
+
209
+ if __name__ == "__main__":
210
+ raise SystemExit(main())