litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,139 @@
1
+ """Authorized SAGE China downloads backed by a saved CARSI session."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ import time
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ import requests
12
+
13
+ from ..config import DATA_DIR
14
+ from ..log import get_logger
15
+ from ..network import USER_AGENT
16
+ from ..download.pdf import _response_looks_pdf, is_pdf_file
17
+ from .publishers import _load_publisher_cookies, _write_pdf_atomic
18
+
19
+ log = get_logger()
20
+
21
+ SAGE_CN_ARTICLE_URL = "https://sage.cnpereading.com/doi/{doi}"
22
+ SAGE_CN_DOWNLOAD_URL = "https://sage.cnpereading.com/website/journal/download"
23
+ _AUTH_COOKIE_NAMES = {"jsessionid", "token", "userinfo"}
24
+
25
+ _ARTICLE_RECORD_RE = re.compile(
26
+ r'articleId\s*":\s*"(?P<article_id>[A-F0-9]{32})"\s*,\s*"doi\s*":\s*"(?P<doi>[^"<]+)',
27
+ re.IGNORECASE,
28
+ )
29
+
30
+
31
+ def extract_article_id(html: str, doi: str) -> str:
32
+ """Return the SAGE CN article id paired with *doi*, ignoring related papers."""
33
+ normalized_html = html.replace(r'\"', '"').replace(r"\/", "/")
34
+ wanted = doi.strip().lower()
35
+ for match in _ARTICLE_RECORD_RE.finditer(normalized_html):
36
+ candidate = match.group("doi").replace("\\", "").strip().lower()
37
+ if candidate == wanted:
38
+ return match.group("article_id")
39
+ return ""
40
+
41
+
42
+ def _cookie_files(config: dict[str, Any]) -> list[Path]:
43
+ cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
44
+ return [
45
+ cache_dir / "carsi_cookies" / "sage-cn.json",
46
+ cache_dir / "carsi_cookies" / "sage.json",
47
+ cache_dir / "publisher_cookies.json",
48
+ ]
49
+
50
+
51
+ def has_saved_sage_cn_session(config: dict[str, Any]) -> bool:
52
+ """Whether an unexpired saved cookie targets the SAGE China domain."""
53
+ now = time.time()
54
+ for cookie_file in _cookie_files(config):
55
+ if not cookie_file.exists():
56
+ continue
57
+ try:
58
+ cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
59
+ except (OSError, ValueError, TypeError):
60
+ continue
61
+ if not isinstance(cookies, list):
62
+ continue
63
+ for cookie in cookies:
64
+ domain = str(cookie.get("domain", "")).lstrip(".").lower()
65
+ try:
66
+ expires = float(cookie.get("expires", 0) or 0)
67
+ except (TypeError, ValueError):
68
+ expires = 0
69
+ if (
70
+ str(cookie.get("name", "")).lower() in _AUTH_COOKIE_NAMES
71
+ and cookie.get("value")
72
+ and (
73
+ domain == "sage.cnpereading.com"
74
+ or domain.endswith(".sage.cnpereading.com")
75
+ )
76
+ and (not expires or expires > now)
77
+ ):
78
+ return True
79
+ return False
80
+
81
+
82
+ def try_sage_cn_authorized(
83
+ doi: str,
84
+ output_path: Path,
85
+ config: dict[str, Any],
86
+ *,
87
+ session: requests.Session | None = None,
88
+ ) -> bool:
89
+ """Download a SAGE CN PDF through a previously authorized CARSI session.
90
+
91
+ SAGE CN does not expose the article PDF at ``/doi/pdf/<doi>``. Its page
92
+ data pairs each DOI with an opaque ``articleId``; the visible PDF button
93
+ then calls ``/website/journal/download?articleId=...``. Replaying that
94
+ site-owned request with the saved institutional cookies avoids browser PDF
95
+ viewer crashes while preserving the user's authorized access.
96
+ """
97
+ if not has_saved_sage_cn_session(config):
98
+ return False
99
+
100
+ own_session = session is None
101
+ client = session or requests.Session()
102
+ if own_session:
103
+ client.trust_env = False
104
+ client.headers.setdefault("User-Agent", USER_AGENT)
105
+ client.headers.setdefault("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
106
+ _load_publisher_cookies(client, config)
107
+
108
+ timeout = int(config.get("publisher_timeout", 60) or 60)
109
+ article_url = SAGE_CN_ARTICLE_URL.format(doi=doi)
110
+ try:
111
+ article_response = client.get(article_url, timeout=timeout, allow_redirects=True)
112
+ if article_response.status_code >= 400:
113
+ return False
114
+ article_id = extract_article_id(article_response.text, doi)
115
+ if not article_id:
116
+ return False
117
+
118
+ pdf_response = client.get(
119
+ SAGE_CN_DOWNLOAD_URL,
120
+ params={"articleId": article_id},
121
+ timeout=timeout,
122
+ stream=True,
123
+ allow_redirects=True,
124
+ headers={"Accept": "application/pdf,*/*", "Referer": article_url},
125
+ )
126
+ if pdf_response.status_code >= 400:
127
+ return False
128
+ iterator = pdf_response.iter_content(chunk_size=8192)
129
+ first = next(iterator, b"")
130
+ if not _response_looks_pdf(pdf_response, first):
131
+ return False
132
+ if not _write_pdf_atomic(output_path, first, iterator):
133
+ return False
134
+ if not is_pdf_file(output_path):
135
+ return False
136
+ log.info(" [SAGE-CN] authorized PDF downloaded via articleId")
137
+ return True
138
+ except (requests.RequestException, OSError, ValueError):
139
+ return False
@@ -0,0 +1,10 @@
1
+ """SciBban source stub — not yet implemented."""
2
+
3
+ from __future__ import annotations
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+
8
+ def try_scibban(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
9
+ """Try to download from SciBban. Currently not implemented."""
10
+ return None