litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Paper list parser for APA references, BibTeX, and DOI lists."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class PaperEntry:
|
|
13
|
+
title: str
|
|
14
|
+
authors: list[str] = field(default_factory=list)
|
|
15
|
+
year: int | None = None
|
|
16
|
+
doi: str | None = None
|
|
17
|
+
journal: str | None = None
|
|
18
|
+
raw: str = ""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse_paper_list(file_path: str | Path) -> list[PaperEntry]:
|
|
22
|
+
"""Auto-detect format and parse paper list."""
|
|
23
|
+
path = Path(file_path)
|
|
24
|
+
if not path.exists():
|
|
25
|
+
raise FileNotFoundError(f"File not found: {file_path}")
|
|
26
|
+
|
|
27
|
+
suffix = path.suffix.lower()
|
|
28
|
+
if suffix in (".csv", ".xlsx", ".tsv", ".tab"):
|
|
29
|
+
from ..download.pipeline import entries_from_table, read_table
|
|
30
|
+
|
|
31
|
+
return _entries_from_queue(entries_from_table(read_table(path)))
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
content = path.read_text(encoding="utf-8")
|
|
35
|
+
except UnicodeDecodeError:
|
|
36
|
+
content = path.read_text(encoding="latin-1")
|
|
37
|
+
|
|
38
|
+
if suffix == ".bib":
|
|
39
|
+
return _parse_bib_to_entries(content)
|
|
40
|
+
if suffix in (".txt", ".md"):
|
|
41
|
+
# Try APA format first (has author-year patterns)
|
|
42
|
+
if re.search(r"[A-Z][a-z\u00C0-\u024F]+,\s+[A-Z]\.", content):
|
|
43
|
+
return parse_apa_references(content)
|
|
44
|
+
# Fall back to DOI list
|
|
45
|
+
return _parse_doi_list(content)
|
|
46
|
+
|
|
47
|
+
# Default: try APA, then DOI list
|
|
48
|
+
if re.search(r"[A-Z][a-z\u00C0-\u024F]+,\s+[A-Z]\.", content):
|
|
49
|
+
return parse_apa_references(content)
|
|
50
|
+
return _parse_doi_list(content)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _entries_from_queue(queue_entries) -> list[PaperEntry]:
|
|
54
|
+
"""Convert pipeline QueueEntry objects to PaperEntry (skipping unresolved)."""
|
|
55
|
+
out = []
|
|
56
|
+
for e in queue_entries:
|
|
57
|
+
if e.unresolved or not e.identifier:
|
|
58
|
+
continue
|
|
59
|
+
out.append(PaperEntry(title=e.title, doi=e.identifier, raw=e.raw or e.identifier))
|
|
60
|
+
return out
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def parse_apa_references(text: str) -> list[PaperEntry]:
|
|
64
|
+
"""Parse APA-style reference list."""
|
|
65
|
+
entries = []
|
|
66
|
+
raw_entries = _split_apa_entries(text)
|
|
67
|
+
for raw in raw_entries:
|
|
68
|
+
entry = _parse_single_apa(raw)
|
|
69
|
+
if entry:
|
|
70
|
+
entries.append(entry)
|
|
71
|
+
return entries
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _split_apa_entries(text: str) -> list[str]:
|
|
75
|
+
"""Split continuous APA text by DOI URL boundaries.
|
|
76
|
+
|
|
77
|
+
Each APA entry ends with a DOI URL. Find DOI URLs and use them as
|
|
78
|
+
entry boundaries, assigning each DOI URL to the following entry.
|
|
79
|
+
"""
|
|
80
|
+
# Find all DOI URL positions
|
|
81
|
+
doi_positions = [m.end() for m in re.finditer(r"https://doi\.org/\S+", text)]
|
|
82
|
+
if not doi_positions:
|
|
83
|
+
return [text.strip()] if text.strip() else []
|
|
84
|
+
|
|
85
|
+
# Split after each DOI URL
|
|
86
|
+
parts = []
|
|
87
|
+
for i, end_pos in enumerate(doi_positions):
|
|
88
|
+
start = doi_positions[i - 1] if i > 0 else 0
|
|
89
|
+
# Find the next author pattern after this DOI
|
|
90
|
+
rest = text[end_pos:]
|
|
91
|
+
# Skip whitespace between DOI and next entry
|
|
92
|
+
m = re.match(r"\s+", rest)
|
|
93
|
+
split_pos = end_pos + (m.end() if m else 0)
|
|
94
|
+
parts.append(text[start:split_pos].strip())
|
|
95
|
+
|
|
96
|
+
# Add any remaining text after the last DOI
|
|
97
|
+
last_end = doi_positions[-1]
|
|
98
|
+
rest = text[last_end:]
|
|
99
|
+
m = re.match(r"\s+", rest)
|
|
100
|
+
remaining_start = last_end + (m.end() if m else 0)
|
|
101
|
+
if remaining_start < len(text):
|
|
102
|
+
remaining = text[remaining_start:].strip()
|
|
103
|
+
if remaining:
|
|
104
|
+
parts.append(remaining)
|
|
105
|
+
|
|
106
|
+
return [p for p in parts if p]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _parse_single_apa(raw: str) -> PaperEntry | None:
|
|
110
|
+
"""Parse a single APA reference entry."""
|
|
111
|
+
if len(raw) < 20:
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
# Extract year
|
|
115
|
+
year_match = re.search(r"\((\d{4})[a-z]?\)", raw)
|
|
116
|
+
year = int(year_match.group(1)) if year_match else None
|
|
117
|
+
|
|
118
|
+
# Extract DOI
|
|
119
|
+
doi = None
|
|
120
|
+
doi_match = re.search(r"https?://doi\.org/(\S+)", raw)
|
|
121
|
+
if doi_match:
|
|
122
|
+
doi = _normalize_doi_string(doi_match.group(1))
|
|
123
|
+
|
|
124
|
+
# Extract title: between year and journal (or volume number)
|
|
125
|
+
title = ""
|
|
126
|
+
if year_match:
|
|
127
|
+
after_year = raw[year_match.end():].strip()
|
|
128
|
+
# Skip leading period/punctuation after year
|
|
129
|
+
after_year = after_year.lstrip(". ")
|
|
130
|
+
# Title ends at period before journal name
|
|
131
|
+
title_match = re.match(r"(.+?)\.\s+", after_year)
|
|
132
|
+
if title_match:
|
|
133
|
+
title = title_match.group(1).strip()
|
|
134
|
+
|
|
135
|
+
# Extract first author
|
|
136
|
+
authors = []
|
|
137
|
+
author_match = re.match(r"([A-Z\u00C0-\u024F][a-z\u00C0-\u024F-]+,\s+[A-Z]\.[^,(&]*)", raw)
|
|
138
|
+
if author_match:
|
|
139
|
+
authors = [author_match.group(1).strip()]
|
|
140
|
+
|
|
141
|
+
return PaperEntry(
|
|
142
|
+
title=title,
|
|
143
|
+
authors=authors,
|
|
144
|
+
year=year,
|
|
145
|
+
doi=doi,
|
|
146
|
+
raw=raw[:200],
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _normalize_doi_string(doi: str) -> str | None:
|
|
151
|
+
"""Fix DOI format: unicode hyphens to ASCII, remove internal spaces."""
|
|
152
|
+
if not doi:
|
|
153
|
+
return None
|
|
154
|
+
from ..terms.text import ascii_dashes
|
|
155
|
+
doi = ascii_dashes(doi)
|
|
156
|
+
# Remove internal spaces
|
|
157
|
+
doi = re.sub(r"\s+", "", doi)
|
|
158
|
+
# Strip trailing punctuation
|
|
159
|
+
doi = doi.rstrip(".,;:")
|
|
160
|
+
# Strip trailing author names (e.g. "10.1002/ird.2673Hamed" → "10.1002/ird.2673")
|
|
161
|
+
# Author names start with uppercase letter followed by lowercase letters
|
|
162
|
+
doi = re.sub(r"[A-Z][a-z\u00C0-\u024F]{1,}$", "", doi)
|
|
163
|
+
# Validate
|
|
164
|
+
if re.match(r"^10\.\d{4,}/", doi):
|
|
165
|
+
return doi
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _parse_bib_to_entries(content: str) -> list[PaperEntry]:
|
|
170
|
+
"""Parse BibTeX content into PaperEntry list."""
|
|
171
|
+
from .bib_import import parse_bib_text
|
|
172
|
+
raw_entries = parse_bib_text(content, require_doi=False)
|
|
173
|
+
entries = []
|
|
174
|
+
for e in raw_entries:
|
|
175
|
+
doi = e.get("doi", "")
|
|
176
|
+
if doi:
|
|
177
|
+
doi = _normalize_doi_string(doi)
|
|
178
|
+
authors_str = e.get("author", e.get("authors", ""))
|
|
179
|
+
authors = [a.strip() for a in authors_str.split(" and ")] if authors_str else []
|
|
180
|
+
year_str = e.get("year", "")
|
|
181
|
+
year = int(year_str) if year_str.isdigit() else None
|
|
182
|
+
entries.append(PaperEntry(
|
|
183
|
+
title=e.get("title", ""),
|
|
184
|
+
authors=authors,
|
|
185
|
+
year=year,
|
|
186
|
+
doi=doi,
|
|
187
|
+
journal=e.get("journal", ""),
|
|
188
|
+
raw=f"@{e.get('type', 'article')}{{{e.get('cite_key', '')}",
|
|
189
|
+
))
|
|
190
|
+
return entries
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _parse_doi_list(content: str) -> list[PaperEntry]:
|
|
194
|
+
"""Parse plain text DOI list."""
|
|
195
|
+
entries = []
|
|
196
|
+
for part in re.split(r"[,\n;]+", content):
|
|
197
|
+
doi = part.strip()
|
|
198
|
+
if re.match(r"^10\.\d{4,}/", doi):
|
|
199
|
+
entries.append(PaperEntry(doi=doi, raw=doi))
|
|
200
|
+
return entries
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Export a screening session's papers: BibTeX, RIS or a DOI list, from the record alone (no network).
|
|
2
|
+
|
|
3
|
+
Which papers: "selected" (the user's final picks, the default) or "high" (rated 高,
|
|
4
|
+
the user's rating first). Files go to <output_dir>/exports/.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ..search import screening
|
|
15
|
+
from ..config import load_config
|
|
16
|
+
|
|
17
|
+
FORMATS = {"bibtex": "bib", "ris": "ris", "doi": "txt"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def papers(session: str, which: str = "selected") -> list[dict[str, Any]]:
|
|
21
|
+
items = screening.load(session)["items"]
|
|
22
|
+
if which == "high":
|
|
23
|
+
return [it for it in items if screening._rating(it) == "高"]
|
|
24
|
+
return [it for it in items if it.get("decision") == "selected"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _key(it: dict[str, Any], used: set[str]) -> str:
|
|
28
|
+
first = (it.get("authors") or ["anon"])[0]
|
|
29
|
+
surname = re.sub(r"[^A-Za-z]", "", first.split()[-1] if first.split() else "anon") or "anon"
|
|
30
|
+
word = next((w for w in re.findall(r"[A-Za-z]{4,}", it.get("title") or "")), "paper")
|
|
31
|
+
base = f"{surname}{it.get('year') or ''}{word}".lower()
|
|
32
|
+
key, n = base, 1
|
|
33
|
+
while key in used:
|
|
34
|
+
n += 1
|
|
35
|
+
key = f"{base}{n}"
|
|
36
|
+
used.add(key)
|
|
37
|
+
return key
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _bib_escape(s: str) -> str:
|
|
41
|
+
return str(s).replace("{", "\\{").replace("}", "\\}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def bibtex(items: list[dict[str, Any]]) -> str:
|
|
45
|
+
used: set[str] = set()
|
|
46
|
+
out = []
|
|
47
|
+
for it in items:
|
|
48
|
+
fields = {"title": it.get("title"), "author": " and ".join(it.get("authors") or []) or None,
|
|
49
|
+
"journal": it.get("journal"), "year": it.get("year"), "doi": it.get("doi")}
|
|
50
|
+
body = ",\n".join(f" {k} = {{{_bib_escape(v)}}}" for k, v in fields.items() if v)
|
|
51
|
+
out.append(f"@article{{{_key(it, used)},\n{body}\n}}")
|
|
52
|
+
return "\n\n".join(out) + "\n"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def ris(items: list[dict[str, Any]]) -> str:
|
|
56
|
+
out = []
|
|
57
|
+
for it in items:
|
|
58
|
+
lines = ["TY - JOUR", f"TI - {it.get('title') or ''}"]
|
|
59
|
+
lines += [f"AU - {a}" for a in it.get("authors") or []]
|
|
60
|
+
for tag, key in (("JO", "journal"), ("PY", "year"), ("DO", "doi")):
|
|
61
|
+
if it.get(key):
|
|
62
|
+
lines.append(f"{tag} - {it[key]}")
|
|
63
|
+
lines.append("ER - ")
|
|
64
|
+
out.append("\n".join(lines))
|
|
65
|
+
return "\n\n".join(out) + "\n"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def doi_list(items: list[dict[str, Any]]) -> str:
|
|
69
|
+
return "\n".join(it["doi"] for it in items if it.get("doi")) + "\n"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def export(session: str, fmt: str = "bibtex", which: str = "selected") -> dict[str, Any]:
|
|
73
|
+
if fmt not in FORMATS:
|
|
74
|
+
raise ValueError(f"format must be one of {sorted(FORMATS)}")
|
|
75
|
+
items = papers(session, which)
|
|
76
|
+
if not items:
|
|
77
|
+
raise ValueError("no papers to export")
|
|
78
|
+
text = {"bibtex": bibtex, "ris": ris, "doi": doi_list}[fmt](items)
|
|
79
|
+
folder = Path(str(load_config().get("output_dir") or "")).expanduser() / "exports"
|
|
80
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
81
|
+
name = f"{session[:15]}-{which}-{time.strftime('%H%M%S')}.{FORMATS[fmt]}"
|
|
82
|
+
path = folder / name
|
|
83
|
+
path.write_text(text, encoding="utf-8", newline="\n")
|
|
84
|
+
return {"file": str(path), "count": len(items)}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def prisma_export(session: str) -> dict[str, Any]:
|
|
88
|
+
"""Write the session's flow counts as Markdown under <output_dir>/exports/."""
|
|
89
|
+
text = screening.prisma_markdown(session)
|
|
90
|
+
folder = Path(str(load_config().get("output_dir") or "")).expanduser() / "exports"
|
|
91
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
path = folder / f"{session[:15]}-prisma-{time.strftime('%H%M%S')}.md"
|
|
93
|
+
path.write_text(text, encoding="utf-8", newline="\n")
|
|
94
|
+
return {"file": str(path)}
|
litloop/cli/__init__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""CLI package — commands grouped by domain.
|
|
2
|
+
|
|
3
|
+
The actual command implementations live in ``commands.py``.
|
|
4
|
+
This package exists so future command modules can be added under ``cli/``
|
|
5
|
+
without changing the public import path.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Re-export the Typer app and all commands from the core module
|
|
9
|
+
from .commands import app # noqa: F401
|
|
10
|
+
|
|
11
|
+
# Trigger command registration by importing the core module
|
|
12
|
+
from . import commands # noqa: F401
|
litloop/cli/_utils.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""CLI shared utilities and app singleton."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
# Fix Windows console encoding for Unicode output
|
|
10
|
+
if sys.platform == "win32":
|
|
11
|
+
os.environ.setdefault("PYTHONIOENCODING", "utf-8")
|
|
12
|
+
try:
|
|
13
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
14
|
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
|
15
|
+
except Exception:
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
import typer
|
|
19
|
+
from rich.console import Console
|
|
20
|
+
from rich.table import Table
|
|
21
|
+
|
|
22
|
+
from ..config import load_config, save_config
|
|
23
|
+
|
|
24
|
+
app = typer.Typer(
|
|
25
|
+
name="litloop",
|
|
26
|
+
help="Fetch academic papers via institutional access, Open Access, or arXiv.",
|
|
27
|
+
no_args_is_help=True,
|
|
28
|
+
)
|
|
29
|
+
console = Console()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _setup_logging(verbose: bool = False):
|
|
33
|
+
level = logging.DEBUG if verbose else logging.INFO
|
|
34
|
+
logging.basicConfig(
|
|
35
|
+
level=level,
|
|
36
|
+
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
37
|
+
datefmt="%H:%M:%S",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _ensure_email(config: dict):
|
|
42
|
+
if not config.get("email", ""):
|
|
43
|
+
console.print("[yellow]Email not configured (needed for Unpaywall OA detection).[/yellow]")
|
|
44
|
+
email = typer.prompt("Enter your email address")
|
|
45
|
+
config["email"] = email
|
|
46
|
+
save_config(config)
|
|
47
|
+
console.print(f"[green]Email saved: {email}[/green]")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _school_type_label(school_type: str) -> str:
|
|
51
|
+
return {
|
|
52
|
+
"webvpn": "CampusPortal",
|
|
53
|
+
"easyconnect": "CampusConnector",
|
|
54
|
+
"atrust": "CampusConnector",
|
|
55
|
+
"ezproxy": "LibraryPortal",
|
|
56
|
+
}.get(school_type, school_type)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _read_dois(path: Path) -> list[str]:
|
|
60
|
+
return [
|
|
61
|
+
line.strip()
|
|
62
|
+
for line in path.read_text(encoding="utf-8").splitlines()
|
|
63
|
+
if line.strip() and not line.strip().startswith("#")
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _print_batch_summary(summary: dict, *, extra_lines: list[str] | None = None) -> None:
|
|
68
|
+
console.print(
|
|
69
|
+
f"[bold]Done:[/bold] {summary['success']}/{summary['count']} verified PDFs, "
|
|
70
|
+
f"{summary.get('unverified', 0)} unverified PDFs."
|
|
71
|
+
)
|
|
72
|
+
console.print(f"[dim]PDF dir: {summary['pdf_dir']}[/dim]")
|
|
73
|
+
console.print(f"[dim]Manifest: {summary['manifest']}[/dim]")
|
|
74
|
+
if summary.get("auto_stopped"):
|
|
75
|
+
n = summary.get("ip_blocked_count", 0)
|
|
76
|
+
console.print(
|
|
77
|
+
f"[yellow bold]⚠ 已自动停止:[/yellow bold]连续检测到 IP 被出版商封禁"
|
|
78
|
+
f"({n} 篇返回 ip_blocked),剩余任务已取消。"
|
|
79
|
+
)
|
|
80
|
+
console.print(
|
|
81
|
+
"[yellow]应对:调低 batch_workers、增大 request_delay_min/max,"
|
|
82
|
+
"或参考 README「被封 IP」章节。[/yellow]"
|
|
83
|
+
)
|
|
84
|
+
for line in extra_lines or []:
|
|
85
|
+
console.print(line)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _installed_package_version(name: str) -> str:
|
|
89
|
+
import importlib.metadata
|
|
90
|
+
try:
|
|
91
|
+
return importlib.metadata.version(name)
|
|
92
|
+
except importlib.metadata.PackageNotFoundError:
|
|
93
|
+
return ""
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
CLOAKBROWSER_RECOMMENDED_MIN = "0.5.9"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _parse_version_tuple(v: str) -> tuple[int, ...]:
|
|
100
|
+
parts: list[int] = []
|
|
101
|
+
for token in (v or "").split("."):
|
|
102
|
+
digits = "".join(ch for ch in token if ch.isdigit())
|
|
103
|
+
parts.append(int(digits) if digits else 0)
|
|
104
|
+
return tuple(parts)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _cloakbrowser_version_status() -> tuple[str, str]:
|
|
108
|
+
version = _installed_package_version("cloakbrowser")
|
|
109
|
+
if not version:
|
|
110
|
+
return ("not_installed", "not installed (pip install 'litloop[cloakbrowser]')")
|
|
111
|
+
if _parse_version_tuple(version) < _parse_version_tuple(CLOAKBROWSER_RECOMMENDED_MIN):
|
|
112
|
+
hint = f"{version} (outdated, recommend >={CLOAKBROWSER_RECOMMENDED_MIN}: pip install -U cloakbrowser)"
|
|
113
|
+
return ("outdated", hint)
|
|
114
|
+
return ("ok", version)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _doctor_checks() -> list[tuple[str, str, str]]:
|
|
118
|
+
import shutil
|
|
119
|
+
checks: list[tuple[str, str, str]] = [
|
|
120
|
+
("Python runtime", "ok", sys.executable),
|
|
121
|
+
]
|
|
122
|
+
for command in ("litloop", "litloop-mcp"):
|
|
123
|
+
path = shutil.which(command)
|
|
124
|
+
checks.append((command, "ok" if path else "warning", path or "not found on PATH"))
|
|
125
|
+
for package in ("litloop", "pymupdf"):
|
|
126
|
+
version = _installed_package_version(package)
|
|
127
|
+
checks.append((f"package: {package}", "ok" if version else "warning", version or "not installed"))
|
|
128
|
+
cb_status, cb_detail = _cloakbrowser_version_status()
|
|
129
|
+
checks.append(("package: cloakbrowser", cb_status if cb_status != "not_installed" else "warning", cb_detail))
|
|
130
|
+
try:
|
|
131
|
+
from ..browser.cloakbrowser_compat import configure_builtin_cloakbrowser
|
|
132
|
+
cache_dir = configure_builtin_cloakbrowser(create_dir=False)
|
|
133
|
+
status = "ok" if cache_dir.exists() else "warning"
|
|
134
|
+
detail = str(cache_dir) if cache_dir.exists() else f"not downloaded yet: {cache_dir}"
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
status = "warning"
|
|
137
|
+
detail = f"cache check failed: {exc}"
|
|
138
|
+
checks.append(("CloakBrowser cache", status, detail))
|
|
139
|
+
return checks
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _apply_school_config(cfg: dict, school: str):
|
|
143
|
+
from ..institutional.schools import get_school
|
|
144
|
+
entry = get_school(school)
|
|
145
|
+
cfg["instsci_school"] = entry.name
|
|
146
|
+
if entry.school_type == "ezproxy":
|
|
147
|
+
cfg["ezproxy_login_url"] = entry.host
|
|
148
|
+
cfg["instsci_base_url"] = ""
|
|
149
|
+
else:
|
|
150
|
+
cfg["instsci_base_url"] = entry.host
|
|
151
|
+
cfg["ezproxy_login_url"] = ""
|
|
152
|
+
return entry
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _access_url(cfg: dict) -> str:
|
|
156
|
+
return cfg.get("ezproxy_login_url", "") or cfg.get("instsci_base_url", "")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _configured_subscription_institution(cfg: dict) -> str:
|
|
160
|
+
return (cfg.get("carsi_idp_name", "") or cfg.get("instsci_school", "") or "").strip()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _resolve_subscription_institution(
|
|
164
|
+
cfg: dict,
|
|
165
|
+
institution: str,
|
|
166
|
+
*,
|
|
167
|
+
prompt: bool = True,
|
|
168
|
+
) -> str:
|
|
169
|
+
explicit = institution.strip()
|
|
170
|
+
if explicit:
|
|
171
|
+
return explicit
|
|
172
|
+
configured = _configured_subscription_institution(cfg)
|
|
173
|
+
if configured:
|
|
174
|
+
return configured
|
|
175
|
+
if not prompt:
|
|
176
|
+
console.print(
|
|
177
|
+
"[red]Subscription institution is required.[/red] "
|
|
178
|
+
"Pass --institution or run: instsci setup --school \"Your Institution\""
|
|
179
|
+
)
|
|
180
|
+
raise typer.Exit(1)
|
|
181
|
+
console.print(
|
|
182
|
+
"[yellow]Subscription institution is required for closed-access publisher PDFs.[/yellow]"
|
|
183
|
+
)
|
|
184
|
+
console.print(
|
|
185
|
+
"[dim]Use the institution that owns your subscription, e.g. the name shown in "
|
|
186
|
+
"OpenAthens/Shibboleth/CARSI login pages.[/dim]"
|
|
187
|
+
)
|
|
188
|
+
value = typer.prompt("Subscription institution").strip()
|
|
189
|
+
if not value:
|
|
190
|
+
console.print("[red]Subscription institution cannot be empty.[/red]")
|
|
191
|
+
raise typer.Exit(1)
|
|
192
|
+
cfg["carsi_enabled"] = True
|
|
193
|
+
cfg["carsi_idp_name"] = value
|
|
194
|
+
save_config(cfg)
|
|
195
|
+
return value
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _path_status(path_value: str) -> tuple[str, str]:
|
|
199
|
+
if not path_value:
|
|
200
|
+
return "missing", ""
|
|
201
|
+
path = Path(path_value)
|
|
202
|
+
return ("ok" if path.exists() else "missing", str(path))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _show_setup_check(cfg: dict) -> bool:
|
|
206
|
+
checks: list[tuple[str, str, str]] = []
|
|
207
|
+
checks.append(("School", "ok" if cfg.get("instsci_school", "") else "missing", cfg.get("instsci_school", "") or "set with --school"))
|
|
208
|
+
checks.append(("Access URL", "ok" if _access_url(cfg) else "missing", _access_url(cfg) or "derived from --school"))
|
|
209
|
+
federated_ready = (not cfg.get("carsi_enabled", False)) or bool(cfg.get("carsi_idp_name", ""))
|
|
210
|
+
checks.append((
|
|
211
|
+
"Federated login",
|
|
212
|
+
"ok" if federated_ready else "missing",
|
|
213
|
+
cfg.get("carsi_idp_name", "") or ("disabled" if not cfg.get("carsi_enabled", False) else "set with --federated-school"),
|
|
214
|
+
))
|
|
215
|
+
for label, path_value in [
|
|
216
|
+
("Output dir", cfg.get("output_dir", "")),
|
|
217
|
+
("Cache dir", cfg.get("cache_dir", "")),
|
|
218
|
+
("Chrome profile", cfg.get("chrome_profile_dir", "")),
|
|
219
|
+
("Session dir", cfg.get("carsi_cookie_dir", "")),
|
|
220
|
+
]:
|
|
221
|
+
status, detail = _path_status(path_value)
|
|
222
|
+
checks.append((label, status, detail))
|
|
223
|
+
|
|
224
|
+
table = Table(title="InstSci Environment Check")
|
|
225
|
+
table.add_column("Item", width=18)
|
|
226
|
+
table.add_column("Status", width=10)
|
|
227
|
+
table.add_column("Detail", overflow="fold")
|
|
228
|
+
ready = True
|
|
229
|
+
for label, status, detail in checks:
|
|
230
|
+
if status != "ok":
|
|
231
|
+
ready = False
|
|
232
|
+
style = "green" if status == "ok" else "yellow"
|
|
233
|
+
table.add_row(label, f"[{style}]{status}[/{style}]", detail)
|
|
234
|
+
console.print(table)
|
|
235
|
+
return ready
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _run_federated_login(publisher: str, url: str, force: bool, verbose: bool) -> None:
|
|
239
|
+
_setup_logging(verbose)
|
|
240
|
+
config = load_config()
|
|
241
|
+
|
|
242
|
+
if not config.get("carsi_enabled", ""):
|
|
243
|
+
console.print("[red]Federated login is not enabled. Run: instsci config-cmd --federated-enable --federated-school \"你的学校名\"[/red]")
|
|
244
|
+
raise typer.Exit(1)
|
|
245
|
+
if not config.get("carsi_idp_name", ""):
|
|
246
|
+
console.print("[red]Federated login school not set. Run: instsci config-cmd --federated-school \"你的学校名\"[/red]")
|
|
247
|
+
raise typer.Exit(1)
|
|
248
|
+
if not publisher and url:
|
|
249
|
+
from ..institutional.carsi_login import detect_publisher
|
|
250
|
+
publisher = detect_publisher(url) or ""
|
|
251
|
+
if not publisher:
|
|
252
|
+
console.print("[yellow]Available publishers:[/yellow]")
|
|
253
|
+
console.print(" sciencedirect, springer, wiley, ieee, tandfonline, nature")
|
|
254
|
+
publisher = typer.prompt("Enter publisher name")
|
|
255
|
+
|
|
256
|
+
from ..institutional.carsi_login import CARSIClient
|
|
257
|
+
carsi = CARSIClient(config)
|
|
258
|
+
try:
|
|
259
|
+
console.print(f"[bold]Federated login for: {publisher}[/bold]")
|
|
260
|
+
console.print(f"[dim]School: {config.get('carsi_idp_name', '')}[/dim]")
|
|
261
|
+
if carsi.login(publisher, force=force):
|
|
262
|
+
console.print("[green]Federated access session established![/green]")
|
|
263
|
+
else:
|
|
264
|
+
console.print("[red]Federated login failed.[/red]")
|
|
265
|
+
raise typer.Exit(1)
|
|
266
|
+
finally:
|
|
267
|
+
carsi.close()
|