litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Authorized SAGE China downloads backed by a saved CARSI session."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
|
|
13
|
+
from ..config import DATA_DIR
|
|
14
|
+
from ..log import get_logger
|
|
15
|
+
from ..network import USER_AGENT
|
|
16
|
+
from ..download.pdf import _response_looks_pdf, is_pdf_file
|
|
17
|
+
from .publishers import _load_publisher_cookies, _write_pdf_atomic
|
|
18
|
+
|
|
19
|
+
log = get_logger()
|
|
20
|
+
|
|
21
|
+
SAGE_CN_ARTICLE_URL = "https://sage.cnpereading.com/doi/{doi}"
|
|
22
|
+
SAGE_CN_DOWNLOAD_URL = "https://sage.cnpereading.com/website/journal/download"
|
|
23
|
+
_AUTH_COOKIE_NAMES = {"jsessionid", "token", "userinfo"}
|
|
24
|
+
|
|
25
|
+
_ARTICLE_RECORD_RE = re.compile(
|
|
26
|
+
r'articleId\s*":\s*"(?P<article_id>[A-F0-9]{32})"\s*,\s*"doi\s*":\s*"(?P<doi>[^"<]+)',
|
|
27
|
+
re.IGNORECASE,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def extract_article_id(html: str, doi: str) -> str:
|
|
32
|
+
"""Return the SAGE CN article id paired with *doi*, ignoring related papers."""
|
|
33
|
+
normalized_html = html.replace(r'\"', '"').replace(r"\/", "/")
|
|
34
|
+
wanted = doi.strip().lower()
|
|
35
|
+
for match in _ARTICLE_RECORD_RE.finditer(normalized_html):
|
|
36
|
+
candidate = match.group("doi").replace("\\", "").strip().lower()
|
|
37
|
+
if candidate == wanted:
|
|
38
|
+
return match.group("article_id")
|
|
39
|
+
return ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _cookie_files(config: dict[str, Any]) -> list[Path]:
|
|
43
|
+
cache_dir = Path(config.get("cache_dir", str(DATA_DIR / "cache")))
|
|
44
|
+
return [
|
|
45
|
+
cache_dir / "carsi_cookies" / "sage-cn.json",
|
|
46
|
+
cache_dir / "carsi_cookies" / "sage.json",
|
|
47
|
+
cache_dir / "publisher_cookies.json",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def has_saved_sage_cn_session(config: dict[str, Any]) -> bool:
|
|
52
|
+
"""Whether an unexpired saved cookie targets the SAGE China domain."""
|
|
53
|
+
now = time.time()
|
|
54
|
+
for cookie_file in _cookie_files(config):
|
|
55
|
+
if not cookie_file.exists():
|
|
56
|
+
continue
|
|
57
|
+
try:
|
|
58
|
+
cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
|
|
59
|
+
except (OSError, ValueError, TypeError):
|
|
60
|
+
continue
|
|
61
|
+
if not isinstance(cookies, list):
|
|
62
|
+
continue
|
|
63
|
+
for cookie in cookies:
|
|
64
|
+
domain = str(cookie.get("domain", "")).lstrip(".").lower()
|
|
65
|
+
try:
|
|
66
|
+
expires = float(cookie.get("expires", 0) or 0)
|
|
67
|
+
except (TypeError, ValueError):
|
|
68
|
+
expires = 0
|
|
69
|
+
if (
|
|
70
|
+
str(cookie.get("name", "")).lower() in _AUTH_COOKIE_NAMES
|
|
71
|
+
and cookie.get("value")
|
|
72
|
+
and (
|
|
73
|
+
domain == "sage.cnpereading.com"
|
|
74
|
+
or domain.endswith(".sage.cnpereading.com")
|
|
75
|
+
)
|
|
76
|
+
and (not expires or expires > now)
|
|
77
|
+
):
|
|
78
|
+
return True
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def try_sage_cn_authorized(
|
|
83
|
+
doi: str,
|
|
84
|
+
output_path: Path,
|
|
85
|
+
config: dict[str, Any],
|
|
86
|
+
*,
|
|
87
|
+
session: requests.Session | None = None,
|
|
88
|
+
) -> bool:
|
|
89
|
+
"""Download a SAGE CN PDF through a previously authorized CARSI session.
|
|
90
|
+
|
|
91
|
+
SAGE CN does not expose the article PDF at ``/doi/pdf/<doi>``. Its page
|
|
92
|
+
data pairs each DOI with an opaque ``articleId``; the visible PDF button
|
|
93
|
+
then calls ``/website/journal/download?articleId=...``. Replaying that
|
|
94
|
+
site-owned request with the saved institutional cookies avoids browser PDF
|
|
95
|
+
viewer crashes while preserving the user's authorized access.
|
|
96
|
+
"""
|
|
97
|
+
if not has_saved_sage_cn_session(config):
|
|
98
|
+
return False
|
|
99
|
+
|
|
100
|
+
own_session = session is None
|
|
101
|
+
client = session or requests.Session()
|
|
102
|
+
if own_session:
|
|
103
|
+
client.trust_env = False
|
|
104
|
+
client.headers.setdefault("User-Agent", USER_AGENT)
|
|
105
|
+
client.headers.setdefault("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
|
106
|
+
_load_publisher_cookies(client, config)
|
|
107
|
+
|
|
108
|
+
timeout = int(config.get("publisher_timeout", 60) or 60)
|
|
109
|
+
article_url = SAGE_CN_ARTICLE_URL.format(doi=doi)
|
|
110
|
+
try:
|
|
111
|
+
article_response = client.get(article_url, timeout=timeout, allow_redirects=True)
|
|
112
|
+
if article_response.status_code >= 400:
|
|
113
|
+
return False
|
|
114
|
+
article_id = extract_article_id(article_response.text, doi)
|
|
115
|
+
if not article_id:
|
|
116
|
+
return False
|
|
117
|
+
|
|
118
|
+
pdf_response = client.get(
|
|
119
|
+
SAGE_CN_DOWNLOAD_URL,
|
|
120
|
+
params={"articleId": article_id},
|
|
121
|
+
timeout=timeout,
|
|
122
|
+
stream=True,
|
|
123
|
+
allow_redirects=True,
|
|
124
|
+
headers={"Accept": "application/pdf,*/*", "Referer": article_url},
|
|
125
|
+
)
|
|
126
|
+
if pdf_response.status_code >= 400:
|
|
127
|
+
return False
|
|
128
|
+
iterator = pdf_response.iter_content(chunk_size=8192)
|
|
129
|
+
first = next(iterator, b"")
|
|
130
|
+
if not _response_looks_pdf(pdf_response, first):
|
|
131
|
+
return False
|
|
132
|
+
if not _write_pdf_atomic(output_path, first, iterator):
|
|
133
|
+
return False
|
|
134
|
+
if not is_pdf_file(output_path):
|
|
135
|
+
return False
|
|
136
|
+
log.info(" [SAGE-CN] authorized PDF downloaded via articleId")
|
|
137
|
+
return True
|
|
138
|
+
except (requests.RequestException, OSError, ValueError):
|
|
139
|
+
return False
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""SciBban source stub — not yet implemented."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def try_scibban(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
|
|
9
|
+
"""Try to download from SciBban. Currently not implemented."""
|
|
10
|
+
return None
|