litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/http_utils.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Shared HTTP utilities with retry logic."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
import urllib3
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
# Auto-detect local HTTP connector and disable SSL verification warnings.
|
|
13
|
+
if os.environ.get("HTTP_PROXY") or os.environ.get("HTTPS_PROXY") or \
|
|
14
|
+
os.environ.get("http_proxy") or os.environ.get("https_proxy"):
|
|
15
|
+
_SSL_VERIFY = False
|
|
16
|
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
17
|
+
else:
|
|
18
|
+
_SSL_VERIFY = True
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def request_with_retry(
|
|
22
|
+
method: str,
|
|
23
|
+
url: str,
|
|
24
|
+
*,
|
|
25
|
+
max_retries: int = 3,
|
|
26
|
+
retry_backoff: float = 2.0,
|
|
27
|
+
**kwargs,
|
|
28
|
+
) -> requests.Response:
|
|
29
|
+
"""HTTP request with exponential backoff retry on 429/5xx/network errors."""
|
|
30
|
+
kwargs.setdefault("timeout", 30)
|
|
31
|
+
kwargs.setdefault("verify", _SSL_VERIFY)
|
|
32
|
+
|
|
33
|
+
for attempt in range(max_retries + 1):
|
|
34
|
+
try:
|
|
35
|
+
resp = requests.request(method, url, **kwargs)
|
|
36
|
+
if resp.status_code == 429 or resp.status_code >= 500:
|
|
37
|
+
if attempt < max_retries:
|
|
38
|
+
wait = retry_backoff ** attempt
|
|
39
|
+
logger.warning(
|
|
40
|
+
"HTTP %d for %s, retrying in %.1fs (attempt %d/%d)",
|
|
41
|
+
resp.status_code, url, wait, attempt + 1, max_retries,
|
|
42
|
+
)
|
|
43
|
+
time.sleep(wait)
|
|
44
|
+
continue
|
|
45
|
+
return resp
|
|
46
|
+
except requests.RequestException as e:
|
|
47
|
+
if attempt < max_retries:
|
|
48
|
+
wait = retry_backoff ** attempt
|
|
49
|
+
logger.warning(
|
|
50
|
+
"Request error for %s: %s, retrying in %.1fs (attempt %d/%d)",
|
|
51
|
+
url, e, wait, attempt + 1, max_retries,
|
|
52
|
+
)
|
|
53
|
+
time.sleep(wait)
|
|
54
|
+
else:
|
|
55
|
+
raise
|
|
56
|
+
|
|
57
|
+
return requests.request(method, url, **kwargs)
|
litloop/identifiers.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""DOI and arXiv identifier parsing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import urllib.parse
|
|
7
|
+
|
|
8
|
+
ARXIV_RE = re.compile(r"^(?:arxiv:)?(?P<id>\d{4}\.\d{4,5})(?:v\d+)?$", re.I)
|
|
9
|
+
ARXIV_DOI_RE = re.compile(r"10\.48550/arxiv\.(?P<id>\d{4}\.\d{4,5})(?:v\d+)?", re.I)
|
|
10
|
+
OLD_ARXIV_RE = re.compile(r"^(?:arxiv:)?(?P<id>[a-z-]+(?:\.[A-Z]{2})?/\d{7})(?:v\d+)?$", re.I)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def canonical_doi(value: str | None) -> str:
|
|
14
|
+
"""Canonicalize a DOI: strip, URL-decode, strip prefixes, and lower-case."""
|
|
15
|
+
if not value:
|
|
16
|
+
return ""
|
|
17
|
+
v = urllib.parse.unquote(str(value).strip())
|
|
18
|
+
v = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", v, flags=re.I)
|
|
19
|
+
v = re.sub(r"^doi\.org/", "", v, flags=re.I)
|
|
20
|
+
v = re.sub(r"^doi:\s*", "", v, flags=re.I)
|
|
21
|
+
return v.strip().lower()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def normalize_doi(value: str) -> str:
|
|
25
|
+
value = value.strip()
|
|
26
|
+
value = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", value, flags=re.I)
|
|
27
|
+
value = re.sub(r"^doi:\s*", "", value, flags=re.I)
|
|
28
|
+
return urllib.parse.unquote(value).strip()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def normalize_arxiv_id(value: str) -> str | None:
|
|
32
|
+
raw = value.strip()
|
|
33
|
+
lower = raw.lower()
|
|
34
|
+
if lower.startswith("http://arxiv.org/abs/") or lower.startswith("https://arxiv.org/abs/"):
|
|
35
|
+
raw = raw.rsplit("/", 1)[-1]
|
|
36
|
+
elif lower.startswith("http://arxiv.org/pdf/") or lower.startswith("https://arxiv.org/pdf/"):
|
|
37
|
+
raw = raw.rsplit("/", 1)[-1].removesuffix(".pdf")
|
|
38
|
+
|
|
39
|
+
doi_match = ARXIV_DOI_RE.search(raw)
|
|
40
|
+
if doi_match:
|
|
41
|
+
return doi_match.group("id")
|
|
42
|
+
|
|
43
|
+
match = ARXIV_RE.match(raw)
|
|
44
|
+
if match:
|
|
45
|
+
return match.group("id")
|
|
46
|
+
|
|
47
|
+
old_match = OLD_ARXIV_RE.match(raw)
|
|
48
|
+
if old_match:
|
|
49
|
+
return old_match.group("id")
|
|
50
|
+
|
|
51
|
+
return None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def is_arxiv_identifier(value: str) -> bool:
|
|
55
|
+
return normalize_arxiv_id(value) is not None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# Anywhere in a line: a DOI (bare, doi:, doi.org link, or inside a publisher
|
|
59
|
+
# URL), an arXiv link, or "arXiv:2401.01234".
|
|
60
|
+
_DOI_IN_TEXT = re.compile(r"\b10\.\d{4,9}/[^\s\"'<>]+")
|
|
61
|
+
_ARXIV_IN_TEXT = re.compile(r"(?:arxiv\.org/(?:abs|pdf)/|arxiv:\s*)(\d{4}\.\d{4,5})(?:v\d+)?", re.I)
|
|
62
|
+
_BARE_ARXIV_LINE = re.compile(r"^\s*(\d{4}\.\d{4,5})(?:v\d+)?\s*$")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def extract_identifiers(text: str) -> tuple[list[str], list[str]]:
|
|
66
|
+
"""Papers in pasted text, for a download list: (DOIs, lines with none).
|
|
67
|
+
|
|
68
|
+
One line may hold several. DOIs come back canonical (lower case, no
|
|
69
|
+
prefix); arXiv papers as their DataCite DOI 10.48550/arxiv.<id>, which the
|
|
70
|
+
downloads treat as the same paper as the bare id. Duplicates are dropped,
|
|
71
|
+
first occurrence kept. Lines with nothing recognisable come back as they
|
|
72
|
+
were, so the user can see what was not understood.
|
|
73
|
+
"""
|
|
74
|
+
found: list[str] = []
|
|
75
|
+
seen: set[str] = set()
|
|
76
|
+
unrecognized: list[str] = []
|
|
77
|
+
|
|
78
|
+
def add(doi: str) -> None:
|
|
79
|
+
if doi and doi not in seen:
|
|
80
|
+
seen.add(doi)
|
|
81
|
+
found.append(doi)
|
|
82
|
+
|
|
83
|
+
# Copied from PDFs: Unicode hyphens and dashes inside DOIs, non-breaking and
|
|
84
|
+
# zero-width spaces.
|
|
85
|
+
from .terms.text import ascii_dashes
|
|
86
|
+
text = ascii_dashes(text or "")
|
|
87
|
+
text = re.sub("[ ]", " ", text)
|
|
88
|
+
for line in text.splitlines():
|
|
89
|
+
if not line.strip():
|
|
90
|
+
continue
|
|
91
|
+
hits = 0
|
|
92
|
+
for m in _DOI_IN_TEXT.finditer(urllib.parse.unquote(line)):
|
|
93
|
+
doi = m.group(0).rstrip(".,;:)]}>")
|
|
94
|
+
doi = re.sub(r"(?i)(\.pdf|/pdf|/full|/abstract|/epdf)$", "", doi)
|
|
95
|
+
arxiv = ARXIV_DOI_RE.fullmatch(doi)
|
|
96
|
+
add(f"10.48550/arxiv.{arxiv.group('id')}" if arxiv else canonical_doi(doi))
|
|
97
|
+
hits += 1
|
|
98
|
+
for m in _ARXIV_IN_TEXT.finditer(line):
|
|
99
|
+
add(f"10.48550/arxiv.{m.group(1)}")
|
|
100
|
+
hits += 1
|
|
101
|
+
bare = _BARE_ARXIV_LINE.match(line)
|
|
102
|
+
if bare and not hits:
|
|
103
|
+
add(f"10.48550/arxiv.{bare.group(1)}")
|
|
104
|
+
hits += 1
|
|
105
|
+
if not hits:
|
|
106
|
+
unrecognized.append(line.strip())
|
|
107
|
+
return found, unrecognized
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def validate_doi(doi: str) -> tuple[bool, str]:
|
|
111
|
+
"""Check if a DOI resolves via doi.org. Returns (valid, resolved_url or error)."""
|
|
112
|
+
import requests
|
|
113
|
+
from .network import user_agent
|
|
114
|
+
try:
|
|
115
|
+
resp = requests.head(
|
|
116
|
+
f"https://doi.org/{urllib.parse.quote(doi, safe='')}",
|
|
117
|
+
timeout=10,
|
|
118
|
+
allow_redirects=True,
|
|
119
|
+
headers={"User-Agent": user_agent()},
|
|
120
|
+
)
|
|
121
|
+
if resp.status_code == 200:
|
|
122
|
+
return True, resp.url
|
|
123
|
+
elif resp.status_code == 404:
|
|
124
|
+
return False, "DOI not found (404)"
|
|
125
|
+
else:
|
|
126
|
+
return True, f"status={resp.status_code}"
|
|
127
|
+
except requests.Timeout:
|
|
128
|
+
return True, "timeout (assume valid)"
|
|
129
|
+
except Exception as e:
|
|
130
|
+
return True, f"check failed: {e}"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def safe_filename(identifier: str) -> str:
|
|
134
|
+
return re.sub(r"[^A-Za-z0-9._-]+", "_", identifier).strip("_") or "paper"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Institutional access subpackage for LitLoop."""
|