litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/http_utils.py ADDED
@@ -0,0 +1,57 @@
1
+ """Shared HTTP utilities with retry logic."""
2
+
3
+ import logging
4
+ import os
5
+ import time
6
+ import urllib3
7
+
8
+ import requests
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+ # Auto-detect local HTTP connector and disable SSL verification warnings.
13
+ if os.environ.get("HTTP_PROXY") or os.environ.get("HTTPS_PROXY") or \
14
+ os.environ.get("http_proxy") or os.environ.get("https_proxy"):
15
+ _SSL_VERIFY = False
16
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
17
+ else:
18
+ _SSL_VERIFY = True
19
+
20
+
21
+ def request_with_retry(
22
+ method: str,
23
+ url: str,
24
+ *,
25
+ max_retries: int = 3,
26
+ retry_backoff: float = 2.0,
27
+ **kwargs,
28
+ ) -> requests.Response:
29
+ """HTTP request with exponential backoff retry on 429/5xx/network errors."""
30
+ kwargs.setdefault("timeout", 30)
31
+ kwargs.setdefault("verify", _SSL_VERIFY)
32
+
33
+ for attempt in range(max_retries + 1):
34
+ try:
35
+ resp = requests.request(method, url, **kwargs)
36
+ if resp.status_code == 429 or resp.status_code >= 500:
37
+ if attempt < max_retries:
38
+ wait = retry_backoff ** attempt
39
+ logger.warning(
40
+ "HTTP %d for %s, retrying in %.1fs (attempt %d/%d)",
41
+ resp.status_code, url, wait, attempt + 1, max_retries,
42
+ )
43
+ time.sleep(wait)
44
+ continue
45
+ return resp
46
+ except requests.RequestException as e:
47
+ if attempt < max_retries:
48
+ wait = retry_backoff ** attempt
49
+ logger.warning(
50
+ "Request error for %s: %s, retrying in %.1fs (attempt %d/%d)",
51
+ url, e, wait, attempt + 1, max_retries,
52
+ )
53
+ time.sleep(wait)
54
+ else:
55
+ raise
56
+
57
+ return requests.request(method, url, **kwargs)
litloop/identifiers.py ADDED
@@ -0,0 +1,134 @@
1
+ """DOI and arXiv identifier parsing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ import urllib.parse
7
+
8
+ ARXIV_RE = re.compile(r"^(?:arxiv:)?(?P<id>\d{4}\.\d{4,5})(?:v\d+)?$", re.I)
9
+ ARXIV_DOI_RE = re.compile(r"10\.48550/arxiv\.(?P<id>\d{4}\.\d{4,5})(?:v\d+)?", re.I)
10
+ OLD_ARXIV_RE = re.compile(r"^(?:arxiv:)?(?P<id>[a-z-]+(?:\.[A-Z]{2})?/\d{7})(?:v\d+)?$", re.I)
11
+
12
+
13
+ def canonical_doi(value: str | None) -> str:
14
+ """Canonicalize a DOI: strip, URL-decode, strip prefixes, and lower-case."""
15
+ if not value:
16
+ return ""
17
+ v = urllib.parse.unquote(str(value).strip())
18
+ v = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", v, flags=re.I)
19
+ v = re.sub(r"^doi\.org/", "", v, flags=re.I)
20
+ v = re.sub(r"^doi:\s*", "", v, flags=re.I)
21
+ return v.strip().lower()
22
+
23
+
24
+ def normalize_doi(value: str) -> str:
25
+ value = value.strip()
26
+ value = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", value, flags=re.I)
27
+ value = re.sub(r"^doi:\s*", "", value, flags=re.I)
28
+ return urllib.parse.unquote(value).strip()
29
+
30
+
31
+ def normalize_arxiv_id(value: str) -> str | None:
32
+ raw = value.strip()
33
+ lower = raw.lower()
34
+ if lower.startswith("http://arxiv.org/abs/") or lower.startswith("https://arxiv.org/abs/"):
35
+ raw = raw.rsplit("/", 1)[-1]
36
+ elif lower.startswith("http://arxiv.org/pdf/") or lower.startswith("https://arxiv.org/pdf/"):
37
+ raw = raw.rsplit("/", 1)[-1].removesuffix(".pdf")
38
+
39
+ doi_match = ARXIV_DOI_RE.search(raw)
40
+ if doi_match:
41
+ return doi_match.group("id")
42
+
43
+ match = ARXIV_RE.match(raw)
44
+ if match:
45
+ return match.group("id")
46
+
47
+ old_match = OLD_ARXIV_RE.match(raw)
48
+ if old_match:
49
+ return old_match.group("id")
50
+
51
+ return None
52
+
53
+
54
+ def is_arxiv_identifier(value: str) -> bool:
55
+ return normalize_arxiv_id(value) is not None
56
+
57
+
58
+ # Anywhere in a line: a DOI (bare, doi:, doi.org link, or inside a publisher
59
+ # URL), an arXiv link, or "arXiv:2401.01234".
60
+ _DOI_IN_TEXT = re.compile(r"\b10\.\d{4,9}/[^\s\"'<>]+")
61
+ _ARXIV_IN_TEXT = re.compile(r"(?:arxiv\.org/(?:abs|pdf)/|arxiv:\s*)(\d{4}\.\d{4,5})(?:v\d+)?", re.I)
62
+ _BARE_ARXIV_LINE = re.compile(r"^\s*(\d{4}\.\d{4,5})(?:v\d+)?\s*$")
63
+
64
+
65
+ def extract_identifiers(text: str) -> tuple[list[str], list[str]]:
66
+ """Papers in pasted text, for a download list: (DOIs, lines with none).
67
+
68
+ One line may hold several. DOIs come back canonical (lower case, no
69
+ prefix); arXiv papers as their DataCite DOI 10.48550/arxiv.<id>, which the
70
+ downloads treat as the same paper as the bare id. Duplicates are dropped,
71
+ first occurrence kept. Lines with nothing recognisable come back as they
72
+ were, so the user can see what was not understood.
73
+ """
74
+ found: list[str] = []
75
+ seen: set[str] = set()
76
+ unrecognized: list[str] = []
77
+
78
+ def add(doi: str) -> None:
79
+ if doi and doi not in seen:
80
+ seen.add(doi)
81
+ found.append(doi)
82
+
83
+ # Copied from PDFs: Unicode hyphens and dashes inside DOIs, non-breaking and
84
+ # zero-width spaces.
85
+ from .terms.text import ascii_dashes
86
+ text = ascii_dashes(text or "")
87
+ text = re.sub("[ ​‌‍]", " ", text)
88
+ for line in text.splitlines():
89
+ if not line.strip():
90
+ continue
91
+ hits = 0
92
+ for m in _DOI_IN_TEXT.finditer(urllib.parse.unquote(line)):
93
+ doi = m.group(0).rstrip(".,;:)]}>")
94
+ doi = re.sub(r"(?i)(\.pdf|/pdf|/full|/abstract|/epdf)$", "", doi)
95
+ arxiv = ARXIV_DOI_RE.fullmatch(doi)
96
+ add(f"10.48550/arxiv.{arxiv.group('id')}" if arxiv else canonical_doi(doi))
97
+ hits += 1
98
+ for m in _ARXIV_IN_TEXT.finditer(line):
99
+ add(f"10.48550/arxiv.{m.group(1)}")
100
+ hits += 1
101
+ bare = _BARE_ARXIV_LINE.match(line)
102
+ if bare and not hits:
103
+ add(f"10.48550/arxiv.{bare.group(1)}")
104
+ hits += 1
105
+ if not hits:
106
+ unrecognized.append(line.strip())
107
+ return found, unrecognized
108
+
109
+
110
+ def validate_doi(doi: str) -> tuple[bool, str]:
111
+ """Check if a DOI resolves via doi.org. Returns (valid, resolved_url or error)."""
112
+ import requests
113
+ from .network import user_agent
114
+ try:
115
+ resp = requests.head(
116
+ f"https://doi.org/{urllib.parse.quote(doi, safe='')}",
117
+ timeout=10,
118
+ allow_redirects=True,
119
+ headers={"User-Agent": user_agent()},
120
+ )
121
+ if resp.status_code == 200:
122
+ return True, resp.url
123
+ elif resp.status_code == 404:
124
+ return False, "DOI not found (404)"
125
+ else:
126
+ return True, f"status={resp.status_code}"
127
+ except requests.Timeout:
128
+ return True, "timeout (assume valid)"
129
+ except Exception as e:
130
+ return True, f"check failed: {e}"
131
+
132
+
133
+ def safe_filename(identifier: str) -> str:
134
+ return re.sub(r"[^A-Za-z0-9._-]+", "_", identifier).strip("_") or "paper"
@@ -0,0 +1 @@
1
+ """Institutional access subpackage for LitLoop."""