litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,221 @@
1
+ """Local DOI index, cache validation, and PDF auto-rename helpers.
2
+
3
+ Extracted on 2026-11-20 from ``sources/__init__.py``; all exported names are
4
+ re-exported by ``sources/__init__.py``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import sys
11
+ import time
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from ..identifiers import safe_filename
16
+ from ..log import get_logger
17
+
18
+ log = get_logger()
19
+
20
+ __all__ = [
21
+ "_EXPLICIT_STRATEGIES",
22
+ "_index_entry",
23
+ "_index_entry_is_fresh",
24
+ "_index_strategy_compatible",
25
+ "_norm_path",
26
+ "_files_claimed_by_other_dois",
27
+ "_pdf_mentions_doi",
28
+ "_same_dir",
29
+ "_update_doi_index",
30
+ "_auto_rename",
31
+ ]
32
+
33
+
34
+ def _pkg() -> Any:
35
+ return sys.modules[__package__]
36
+
37
+
38
+ #: Strategies that encode an explicit user source preference. For these, a
39
+ #: cached file obtained through a different strategy is treated as a miss so
40
+ #: the requested channel is actually used; the default strategies reuse cache.
41
+ _EXPLICIT_STRATEGIES = ("scihub_only", "legal_only", "scihub_first")
42
+
43
+
44
+ def _index_entry(entry: Any) -> dict[str, Any] | None:
45
+ """Normalize a .doi_index entry to a record dict.
46
+
47
+ Legacy ``str -> path`` entries have no provenance and no timestamp; they
48
+ are treated as fresh (ts=None) and upgraded on the next cache hit.
49
+ """
50
+ if isinstance(entry, str):
51
+ return {"file": entry, "source": "", "strategy": "", "ts": None}
52
+ if isinstance(entry, dict):
53
+ ts = entry.get("ts")
54
+ try:
55
+ ts_value = float(ts) if ts is not None else None
56
+ except (TypeError, ValueError):
57
+ ts_value = None
58
+ return {
59
+ "file": str(entry.get("file") or ""),
60
+ "source": str(entry.get("source") or ""),
61
+ "strategy": str(entry.get("strategy") or ""),
62
+ "ts": ts_value,
63
+ }
64
+ return None
65
+
66
+
67
+ def _index_entry_is_fresh(entry: dict[str, Any], config: dict[str, Any]) -> bool:
68
+ """Apply the cache TTL (cache_ttl_hours) to an index entry."""
69
+ try:
70
+ ttl_hours = float(config.get("cache_ttl_hours", 168))
71
+ except (TypeError, ValueError):
72
+ ttl_hours = 168.0
73
+ if ttl_hours <= 0 or entry.get("ts") is None:
74
+ return True
75
+ return (time.time() - entry["ts"]) <= ttl_hours * 3600
76
+
77
+
78
+ def _index_strategy_compatible(entry: dict[str, Any], config: dict[str, Any]) -> bool:
79
+ """Honor an explicitly requested download strategy over cached files."""
80
+ current = str(config.get("download_strategy") or "fastest").strip()
81
+ if current not in _pkg()._EXPLICIT_STRATEGIES:
82
+ return True
83
+ recorded = str(entry.get("strategy") or "").strip()
84
+ return recorded == current
85
+
86
+
87
+ def _norm_path(p: Path | str) -> str:
88
+ return str(Path(p).resolve()).lower()
89
+
90
+
91
+ def _files_claimed_by_other_dois(target_dir: Path, doi: str) -> set[str]:
92
+ """Files the DOI index already assigns to a different DOI."""
93
+ try:
94
+ idx = json.loads((target_dir / ".doi_index.json").read_text(encoding="utf-8"))
95
+ except Exception:
96
+ return set()
97
+ claimed = set()
98
+ for key, raw in idx.items():
99
+ entry = _index_entry(raw)
100
+ if key != doi and entry and entry.get("file"):
101
+ claimed.add(_norm_path(entry["file"]))
102
+ return claimed
103
+
104
+
105
+ def _pdf_mentions_doi(path: Path, doi: str, pages: int = 3) -> bool:
106
+ """True when the DOI appears in the first pages' text (publisher PDFs print it)."""
107
+ from ..download.pdf import doi_printed
108
+ return doi_printed(path, doi, pages)
109
+
110
+
111
+ def _same_dir(a: Path, b: Path) -> bool:
112
+ try:
113
+ return a.resolve() == b.resolve()
114
+ except OSError:
115
+ return str(a) == str(b)
116
+
117
+
118
+ def _update_doi_index(
119
+ target_dir: Path,
120
+ doi: str,
121
+ file_path: Path,
122
+ *,
123
+ source: str = "",
124
+ strategy: str = "",
125
+ config: dict[str, Any] | None = None,
126
+ ) -> None:
127
+ """Update the DOI→file index for dedup.
128
+
129
+ Records carry provenance (``source``/``strategy``) and a timestamp so
130
+ stale or channel-mismatched entries can be invalidated by TTL and by
131
+ explicit strategy changes instead of manual ``rm``. Existing entries keep
132
+ their recorded source/strategy when the caller does not supply new ones
133
+ (e.g. a rename that just moves the file).
134
+ """
135
+ doi_index = target_dir / ".doi_index.json"
136
+ try:
137
+ idx: dict[str, Any] = {}
138
+ if doi_index.exists():
139
+ idx = json.loads(doi_index.read_text(encoding="utf-8"))
140
+ existing = _index_entry(idx.get(doi)) if doi in idx else None
141
+ if existing:
142
+ source = source or existing["source"]
143
+ strategy = strategy or existing["strategy"]
144
+ idx[doi] = {
145
+ "file": str(file_path),
146
+ "source": source or "",
147
+ "strategy": strategy or (config or {}).get("download_strategy", "") or "",
148
+ "ts": time.time(),
149
+ }
150
+ doi_index.write_text(json.dumps(idx, indent=2, ensure_ascii=False), encoding="utf-8")
151
+ except Exception:
152
+ pass
153
+
154
+
155
+ def _auto_rename(result: dict[str, Any], identifier: str, config: dict[str, Any], doi: str = "", target_dir: Path | None = None) -> None:
156
+ """Auto-rename downloaded PDF based on metadata."""
157
+ from ..download.rename import rename_format
158
+ if rename_format(config) == "off":
159
+ return
160
+ file_path = Path(result.get("file", ""))
161
+ if not file_path.exists():
162
+ return
163
+ if file_path.suffix.lower() != ".pdf":
164
+ # Non-PDF artifacts (e.g. the ElsevierAPI text/plain full text) keep
165
+ # their own extension — rename_pdf would force a .pdf name onto them.
166
+ log.info(f" Non-PDF artifact ({file_path.suffix or 'no ext'}) — skipping auto-rename")
167
+ return
168
+ # Use cached metadata or fetch from Crossref
169
+ from ..citations.formats import fetch_metadata
170
+ _doi = doi or result.get("doi", identifier)
171
+ metadata = fetch_metadata(_doi, config)
172
+ if metadata:
173
+ new_path = _pkg().rename_pdf(file_path, metadata, doi=_doi, fmt=rename_format(config))
174
+ if new_path and result.get("whole_book") and "整本书" not in new_path.stem:
175
+ # A chapter delivered as its whole book says so in its name.
176
+ tagged = new_path.with_name(f"{new_path.stem}(整本书){new_path.suffix}")
177
+ try:
178
+ new_path.rename(tagged)
179
+ new_path = tagged
180
+ except OSError:
181
+ pass
182
+ if new_path and result.get("preprint") and "_preprint" not in new_path.stem:
183
+ # A preprint of the paper is not the published PDF: say so in the
184
+ # name, so it is not mistaken for the version of record later.
185
+ tagged = new_path.with_name(f"{new_path.stem}_preprint{new_path.suffix}")
186
+ try:
187
+ new_path.rename(tagged)
188
+ new_path = tagged
189
+ except OSError:
190
+ pass
191
+ if new_path and new_path != file_path:
192
+ result["file"] = str(new_path)
193
+ result["renamed"] = True
194
+ log.info(f" Renamed: {file_path.name} -> {new_path.name}")
195
+ # Racing leaves one "<doi>_<Source>.pdf" copy per source that also
196
+ # succeeded; the winner is now saved under its final name.
197
+ if _doi:
198
+ removed = 0
199
+ for dup in new_path.parent.glob(f"{safe_filename(_doi)}_*.pdf"):
200
+ if dup != new_path:
201
+ try:
202
+ dup.unlink()
203
+ removed += 1
204
+ except OSError:
205
+ pass
206
+ if removed:
207
+ log.info(f" Removed {removed} duplicate source cop{'y' if removed == 1 else 'ies'}")
208
+ # Update DOI→file index for dedup
209
+ if target_dir and _doi:
210
+ _pkg()._update_doi_index(
211
+ target_dir,
212
+ _doi,
213
+ new_path,
214
+ source=result.get("source") or "",
215
+ strategy=config.get("download_strategy", ""),
216
+ config=config,
217
+ )
218
+ else:
219
+ log.info(f" Kept original name: {file_path.name}")
220
+ else:
221
+ log.info(f" No metadata for rename, keeping: {file_path.name}")
@@ -0,0 +1,290 @@
1
+ """Phase 1/2 source assembly, institutional routing sequence, watchdog, and failure guidance.
2
+
3
+ Split out of `sources/__init__.py` (2026-11-20); `sources/__init__.py` re-exports
4
+ all symbols below so callers and tests keep importing from there.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import sys
10
+ import threading
11
+ import time
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from ..identifiers import safe_filename
16
+ from ..log import get_logger
17
+ from .core_api import try_core
18
+ from .crossref import try_crossref_page_scrape
19
+ from .europepmc import try_europepmc, try_pmc
20
+ from .libgen import try_libgen
21
+ from .oa_discovery import try_doaj
22
+ from .openaire import try_openaire
23
+ from .openalex import try_openalex_content_api, try_openalex_oa
24
+ from .publishers import get_publisher_fast_sources
25
+ from .scibban import try_scibban
26
+ from .scihub import try_scihub
27
+ from .semantic_scholar import try_semanticscholar
28
+ from .unpaywall import try_unpaywall
29
+
30
+ log = get_logger()
31
+
32
+
33
+ def _pkg() -> Any:
34
+ return sys.modules[__package__]
35
+
36
+
37
+ def _build_free_sources(doi: str, config: dict[str, Any]) -> list[tuple[Any, str]]:
38
+ """Build Phase 1 sources: all free/OA/grey sources, sorted by adaptive score.
39
+
40
+ Respects config['download_strategy']:
41
+ - fastest: all sources, EMA-sorted (default)
42
+ - scihub_first: all sources, but grey sources sorted first
43
+ - scihub_only: only SciBban + Sci-Hub + LibGen
44
+ - oa_first: all sources, but OA sources sorted before grey
45
+ - legal_only: exclude grey sources (SciBban, Sci-Hub, LibGen)
46
+ """
47
+ from .scoring import sort_sources
48
+
49
+ strategy = config.get("download_strategy", "fastest")
50
+
51
+ publisher_fast = get_publisher_fast_sources(doi)
52
+ _fast_names = {label for _, label in publisher_fast}
53
+
54
+ extra_fast = []
55
+ for fn, label in [
56
+ (try_unpaywall, "Unpaywall"),
57
+ (try_openalex_oa, "OpenAlexOA"),
58
+ (try_semanticscholar, "SemanticScholar"),
59
+ (try_openaire, "OpenAIRE"),
60
+ ]:
61
+ if label not in _fast_names:
62
+ extra_fast.append((fn, label))
63
+
64
+ from .elsevier_api import fulltext_enabled
65
+ if not fulltext_enabled(config): # off in settings, or the key has no full-text entitlement
66
+ publisher_fast = [(fn, label) for fn, label in publisher_fast if label != "ElsevierAPI"]
67
+ legal_sources = publisher_fast + extra_fast
68
+ # Every publisher's own site is tried here, in phase 1, from this exit:
69
+ # those without a browser lane of their own (PNAS, Optica, World Scientific,
70
+ # unlisted ones) get the generic one. Open-access hosts need none.
71
+ if not any("Browser" in label for label in _fast_names):
72
+ from .publishers import get_publisher
73
+ if get_publisher(doi) not in _pkg()._OPEN_HOSTS:
74
+ from .publishers import _FN_MAP
75
+ legal_sources.append((_FN_MAP["GenericBrowser"], "GenericBrowser"))
76
+ # Publishers whose site is not worth phase 1 (publisher_site_skip_prefixes):
77
+ # the school routes reach the same content faster.
78
+ skip = tuple(str(p).strip().lower().rstrip("/") + "/" for p in config.get("publisher_site_skip_prefixes") or [] if p)
79
+ if skip and doi.lower().startswith(skip):
80
+ legal_sources = [(fn, label) for fn, label in legal_sources if "Browser" not in label]
81
+ log.info(" Publisher site skipped in phase 1 (publisher_site_skip_prefixes)")
82
+ legal_sources += [(try_doaj, "DOAJ"), (try_crossref_page_scrape, "CrossrefPage")]
83
+ legal_sources += [(try_europepmc, "EuropePMC"), (try_core, "CORE")]
84
+ if not config.get("_batch"):
85
+ legal_sources.append((try_pmc, "PMC"))
86
+
87
+ if config.get("openalex_api_key"):
88
+ legal_sources.append((try_openalex_content_api, "OpenAlexContent"))
89
+
90
+ # Grey / shadow-library sources (Sci-Hub, SciBban, LibGen) are all gated
91
+ # by scihub_enabled and are off unless the user turned it on (CHARTER: the
92
+ # user must choose them explicitly; config.DEFAULT_CONFIG has False). A
93
+ # partial config dict without the key therefore means off, never on.
94
+ grey_sources: list[tuple[Any, str]] = []
95
+ if config.get("scihub_enabled", False):
96
+ from .scihub import recently_absent, too_new
97
+ new = too_new(doi, config) # published in or after scihub_skip_from_year
98
+ if not new:
99
+ grey_sources.append((try_scibban, "SciBban")) # a Sci-Hub mirror
100
+ grey_sources.append((try_libgen, "LibGen"))
101
+ if not new and not recently_absent(doi): # Sci-Hub said within the hour it does not have it
102
+ grey_sources.append((try_scihub, "Sci-Hub"))
103
+
104
+ if strategy == "scihub_only":
105
+ # Only Sci-Hub (not SciBban, not LibGen)
106
+ return sort_sources([(try_scihub, "Sci-Hub")]) if config.get("scihub_enabled", False) else []
107
+ elif strategy == "grey_only":
108
+ return sort_sources(grey_sources)
109
+ elif strategy == "legal_only":
110
+ return sort_sources(legal_sources)
111
+ elif strategy == "scihub_first":
112
+ # Grey sources first, then legal
113
+ return sort_sources(grey_sources) + sort_sources(legal_sources)
114
+ elif strategy == "oa_first":
115
+ # Legal sources first, then grey
116
+ return sort_sources(legal_sources) + grey_sources
117
+ else:
118
+ # fastest: all sources EMA-sorted
119
+ return sort_sources(legal_sources + grey_sources)
120
+
121
+
122
+ # Publishers whose papers are open: no site visit from phase 1 is needed.
123
+ _OPEN_HOSTS = frozenset({"MDPI", "arXiv", "bioRxiv", "PLOS", "Frontiers", "BMC", "Beilstein", "EMBO",
124
+ "Research Square", "chemRxiv", "Preprints.org"})
125
+
126
+
127
+ def _webvpn_http(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
128
+ """WebVPN without a browser: the saved session over HTTP (and a campus
129
+ connector's SOCKS5 when one is configured). Asks the user nothing."""
130
+ from . import instsci
131
+ if instsci._is_campus_connector_mode(config):
132
+ result = instsci._try_instsci_socks5(doi, output_path, config)
133
+ if result:
134
+ return result
135
+ if not instsci.instsci_cookie_path(config).exists():
136
+ return None
137
+ return instsci._try_instsci_http(doi, output_path, config)
138
+
139
+
140
+ def _webvpn_browser(doi: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
141
+ from . import instsci
142
+ return instsci._try_instsci_browser(doi, output_path, config)
143
+
144
+
145
+ # The school's routes, tried one after another (institutional_order): the ones
146
+ # that ask the user nothing first, then those that may need a login, so at most
147
+ # one window waits for the user at a time. Each step: (enabled?, function, label);
148
+ # the label decides whether it takes the browser slot (_BROWSER_SOURCE_LABELS).
149
+ _INSTITUTIONAL_STEPS: dict[str, tuple[Any, Any, str]] = {
150
+ "webvpn_http": (lambda c: bool(c.get("vpnsci_enabled")), _webvpn_http, "WebVPN-HTTP"),
151
+ "webvpn_browser": (lambda c: bool(c.get("vpnsci_enabled")), _webvpn_browser, "WebVPN"),
152
+ "carsi": (lambda c: bool(c.get("carsi_enabled") and str(c.get("carsi_idp_name") or "").strip()),
153
+ lambda d, o, c: _pkg().try_carsi(d, o, c), "CARSI"),
154
+ "ezproxy": (lambda c: bool(c.get("ezproxy_enabled")), lambda d, o, c: _pkg().try_ezproxy(d, o, c), "EZProxy"),
155
+ }
156
+ DEFAULT_INSTITUTIONAL_ORDER = ["webvpn_http", "webvpn_browser", "carsi", "ezproxy"]
157
+
158
+
159
+ def _build_institutional_sources(doi: str, config: dict[str, Any], *, use_vpnsci: bool = False) -> list[tuple[Any, str]]:
160
+ """Phase 2: the school's routes that are switched on, in institutional_order.
161
+
162
+ WebVPN follows its setting (use_vpnsci is kept for callers; a single
163
+ download used to leave WebVPN out unless asked). The publisher's own site
164
+ is not here: phase 1 opened it from this same exit.
165
+ """
166
+ order = config.get("institutional_order") or _pkg().DEFAULT_INSTITUTIONAL_ORDER
167
+ out = []
168
+ for name in order:
169
+ step = _pkg()._INSTITUTIONAL_STEPS.get(str(name))
170
+ if step and step[0](config):
171
+ out.append((step[1], step[2]))
172
+ return out
173
+
174
+
175
+ def _run_institutional_sequence(steps: list[tuple[Any, str]], doi: str, target_dir: Path, output_path: Path,
176
+ config: dict[str, Any], failures: list[dict[str, str]] | None = None,
177
+ ) -> dict[str, Any] | None:
178
+ """Try the school's routes one at a time; the first PDF wins."""
179
+ from ..download import progress as progress_reporter
180
+ log.info(f" School routes in turn: {', '.join(label for _, label in steps)}")
181
+ t_seq = time.time()
182
+ for fn, label in steps:
183
+ if _pkg()._cancelled():
184
+ return None
185
+ progress_reporter.item_stage(doi, f"正在通过学校认证下载:{_pkg()._STEP_TEXT.get(label, label)}")
186
+ src_output = target_dir / f"{safe_filename(doi)}_{label}.pdf"
187
+ done = threading.Event()
188
+ threading.Thread(target=_pkg()._step_watchdog, args=(doi, label, done, config), daemon=True).start()
189
+ try:
190
+ result = _pkg()._try_source(fn, doi, src_output, config, label)
191
+ finally:
192
+ done.set()
193
+ if result and result.get("success"):
194
+ final_path = Path(result.get("file", ""))
195
+ if final_path != output_path and final_path.exists():
196
+ if output_path.exists():
197
+ output_path.unlink()
198
+ final_path.rename(output_path)
199
+ result["file"] = str(output_path)
200
+ log.info(f" OK {label}")
201
+ log.info(f" [timing] school routes: {time.time() - t_seq:.1f}s")
202
+ return result
203
+ if failures is not None:
204
+ _pkg()._record_source_failure(failures, label, result or {"success": False, "error": "no PDF"})
205
+ if src_output.exists():
206
+ src_output.unlink(missing_ok=True)
207
+ log.info(f" [timing] school routes: {time.time() - t_seq:.1f}s, none had it")
208
+ return None
209
+
210
+
211
+ def _step_watchdog(doi: str, label: str, done: threading.Event, config: dict[str, Any]) -> None:
212
+ """End a school route that has gone quiet: after institutional_step_seconds
213
+ without a result (a login the user is doing keeps it alive), kill the
214
+ browsers it opened, so its blocked browser calls fail and the next route
215
+ runs. 2026-10-17: a WebVPN browser step hung over 10 minutes on a page
216
+ that never finished loading, with no timeout of its own firing."""
217
+ from ..browser.backend import close_browsers_since
218
+ limit = float(config.get("institutional_step_seconds", 240) or 240)
219
+ since = time.perf_counter() # close_browsers_since's clock
220
+ deadline = time.time() + limit
221
+ while not done.wait(5):
222
+ if _pkg()._user_waiting_for(doi):
223
+ deadline = max(deadline, time.time() + 60) # the user is at it: give them time
224
+ continue
225
+ if time.time() >= deadline:
226
+ n = close_browsers_since(since, owner=doi)
227
+ log.info(f" [timing] {label}: nothing after {limit:.0f}s; closed {n} browser(s), next route")
228
+ return
229
+
230
+
231
+ _STEP_TEXT = {"WebVPN-HTTP": "WebVPN 直接下载", "WebVPN": "WebVPN 浏览器", "CARSI": "CARSI", "EZProxy": "EZProxy"}
232
+
233
+
234
+ def _build_failure_guidance(doi: str, config: dict[str, Any], error_type: str = "") -> list[str]:
235
+ """Build actionable guidance when all download sources fail.
236
+
237
+ error_type: what this paper's browser strategies reported (its DownloadAttempt).
238
+ """
239
+ import os
240
+ tips = []
241
+
242
+ if error_type == "paywall":
243
+ tips.append("此论文需要机构订阅才能下载")
244
+ tips.append(f"→ 运行 litloop_login(identifier=\"{doi}\") 打开浏览器登录")
245
+ tips.append("→ 在浏览器中点击 'Access through your institution' 选择你的机构")
246
+ tips.append("→ 登录后关闭浏览器,cookies 自动保存")
247
+ tips.append("→ 重新运行下载命令即可")
248
+ elif error_type == "cloudflare_blocked":
249
+ tips.append("Cloudflare 防护阻止了访问")
250
+ tips.append("→ 安装 CloakBrowser (pip install cloakbrowser) 绕过反爬")
251
+ tips.append("→ 或配置代理: litloop config_set network_proxy \"socks5://127.0.0.1:1080\"")
252
+ elif error_type == "browser_unavailable":
253
+ tips.append("CloakBrowser 不可用,无法使用浏览器下载策略")
254
+ tips.append("→ 运行: pip install cloakbrowser")
255
+
256
+ # Check litloop proxy config
257
+ cfg_proxy = config.get("network_proxy", "")
258
+ env_proxy = os.environ.get("LITLOOP_PROXY", "")
259
+
260
+ if cfg_proxy or env_proxy:
261
+ active = env_proxy or cfg_proxy
262
+ tips.append(f"当前代理: {active} — 如果 Sci-Hub 仍不通,尝试更换代理地址")
263
+ else:
264
+ # Check if system has proxy that litloop ignores
265
+ sys_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or ""
266
+ if sys_proxy:
267
+ tips.append(f"检测到系统代理 {sys_proxy},但 litloop 未使用")
268
+ tips.append(f"→ 运行: litloop config_set network_proxy \"{sys_proxy}\"")
269
+ else:
270
+ tips.append("未配置代理 — 如果网络受限,运行: litloop config_set network_proxy \"socks5://127.0.0.1:1080\"")
271
+
272
+ # Check Tor
273
+ try:
274
+ from ..tor.proxy import check_tor_circuit
275
+ if not check_tor_circuit(config):
276
+ tips.append("Tor 未运行 → 运行: litloop tor_start(匿名访问 Sci-Hub/LibGen)")
277
+ except Exception:
278
+ pass
279
+
280
+ # Check WebVPN
281
+ if not config.get("vpnsci_enabled"):
282
+ tips.append("有高校账号?运行: litloop config_set vpnsci_enabled true")
283
+
284
+ # Manual fallback
285
+ tips.append(f"手动下载: https://sci-hub.ru/{doi}")
286
+
287
+ # Network diagnostic
288
+ tips.append("运行网络诊断: litloop network_diagnose")
289
+
290
+ return tips
@@ -0,0 +1,90 @@
1
+ """ALTCHA proof-of-work solved over plain HTTP (no browser).
2
+
3
+ ALTCHA (used by sci-hub.ru) is a proof-of-work gate, not a human test: the
4
+ page's widget fetches a challenge (algorithm, salt, target hash, maxnumber) and
5
+ the client searches for the number n with hash(salt + n) == target, then posts
6
+ it back. Browsers do this in JS; doing the same in Python skips the browser
7
+ window entirely (~1-3 s). Ported from the user's academic-paper-downloader
8
+ skill (scripts/scihub_solver.py).
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import base64
14
+ import hashlib
15
+ import json
16
+ import re
17
+ import time
18
+ from typing import Any
19
+
20
+ import requests
21
+
22
+ from ..log import get_logger
23
+
24
+ log = get_logger()
25
+
26
+ _CHALLENGE_URL_RE = re.compile(r'challengeurl\s*=\s*["\']([^"\']+)["\']', re.I)
27
+ # sci-hub.ru posts the answer to its own solution URL, whose id differs from
28
+ # the challenge id: read it from the page script, don't derive it.
29
+ _SOLUTION_URL_RE = re.compile(r"""fetch\(\s*['"]([^'"]*/solution/[^'"]*)['"]""", re.I)
30
+ _HASHES = {"SHA-256": hashlib.sha256, "SHA-1": hashlib.sha1, "SHA-512": hashlib.sha512}
31
+
32
+
33
+ def is_altcha_page(html: str) -> bool:
34
+ low = html.lower()
35
+ return "altcha" in low and bool(_CHALLENGE_URL_RE.search(html))
36
+
37
+
38
+ def solve(challenge: dict[str, Any], max_seconds: float = 20.0) -> str | None:
39
+ """Return the base64 ALTCHA payload for a challenge, or None."""
40
+ algorithm = str(challenge.get("algorithm") or "SHA-256").upper()
41
+ target = str(challenge.get("challenge") or "")
42
+ salt = str(challenge.get("salt") or "")
43
+ hasher = _HASHES.get(algorithm)
44
+ if not target or not salt or hasher is None:
45
+ return None
46
+ max_number = int(challenge.get("maxNumber") or challenge.get("maxnumber") or 1_000_000)
47
+ deadline = time.monotonic() + max_seconds
48
+ prefix = salt.encode("utf-8")
49
+ for n in range(max_number + 1):
50
+ if hasher(prefix + str(n).encode()).hexdigest() == target:
51
+ payload = {"algorithm": algorithm, "challenge": target, "number": n,
52
+ "salt": salt, "signature": challenge.get("signature", "")}
53
+ return base64.b64encode(json.dumps(payload).encode("utf-8")).decode("ascii")
54
+ if n % 50_000 == 0 and time.monotonic() > deadline:
55
+ return None
56
+ return None
57
+
58
+
59
+ def pass_gate(session: requests.Session, html: str, page_url: str,
60
+ timeout: tuple[int, int] = (10, 25)) -> str | None:
61
+ """Solve the ALTCHA gate on ``page_url`` and return the page HTML behind it."""
62
+ m = _CHALLENGE_URL_RE.search(html)
63
+ if not m:
64
+ return None
65
+ origin = "{0.scheme}://{0.netloc}".format(requests.utils.urlparse(page_url))
66
+ challenge_url = requests.compat.urljoin(page_url, m.group(1))
67
+ try:
68
+ t0 = time.monotonic()
69
+ challenge = session.get(challenge_url, headers={"Referer": page_url}, timeout=timeout).json()
70
+ payload = solve(challenge)
71
+ if not payload:
72
+ log.info(" [altcha-http] no solution within the time budget")
73
+ return None
74
+ sm = _SOLUTION_URL_RE.search(html)
75
+ solution_url = (requests.compat.urljoin(page_url, sm.group(1)) if sm
76
+ else challenge_url.replace("/challenge/", "/solution/"))
77
+ answer = session.post(
78
+ solution_url,
79
+ json={"captcha": payload},
80
+ headers={"Origin": origin, "Referer": page_url},
81
+ timeout=timeout,
82
+ ).json()
83
+ if answer.get("success") is not True:
84
+ log.info(f" [altcha-http] solution rejected: {str(answer)[:80]}")
85
+ return None
86
+ log.info(f" [altcha-http] solved in {time.monotonic() - t0:.1f}s")
87
+ return session.get(page_url, headers={"Referer": page_url}, timeout=timeout).text
88
+ except Exception as exc:
89
+ log.info(f" [altcha-http] failed: {type(exc).__name__}: {exc}")
90
+ return None
@@ -0,0 +1,45 @@
1
+ """arXiv direct PDF download."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from ..identifiers import normalize_arxiv_id
9
+ from ..network import fetch
10
+ from ..download.pdf import _response_looks_pdf, is_pdf_file, success
11
+
12
+
13
+ def try_arxiv(identifier: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
14
+ arxiv_id = normalize_arxiv_id(identifier)
15
+ if not arxiv_id:
16
+ return None
17
+ url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
18
+ return download_arxiv_pdf(url, output_path, config)
19
+
20
+
21
+ def download_arxiv_pdf(url: str, output_path: Path, config: dict[str, Any]) -> dict[str, Any] | None:
22
+ try:
23
+ resp = fetch(url, config, stream=True)
24
+ if resp.status_code >= 400:
25
+ return None
26
+
27
+ iterator = resp.iter_content(chunk_size=8192)
28
+ first_chunk = next(iterator, b"")
29
+ if not _response_looks_pdf(resp, first_chunk):
30
+ return None
31
+
32
+ from .publishers import _write_pdf_atomic
33
+ if not _write_pdf_atomic(output_path, first_chunk, iterator):
34
+ return None
35
+
36
+ if is_pdf_file(output_path):
37
+ return success(output_path.stem, output_path, "arXiv")
38
+ else:
39
+ try:
40
+ output_path.unlink(missing_ok=True)
41
+ except OSError:
42
+ pass
43
+ except Exception:
44
+ return None
45
+ return None