litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
"""Discovery layer: ScanSci Find CLI facade.
|
|
2
|
+
|
|
3
|
+
Brings ScanSci Find's six capability areas into litloop by delegating to
|
|
4
|
+
the ``scansci-find`` CLI at runtime (no code copy):
|
|
5
|
+
|
|
6
|
+
- 13-source federated search with domain routing (OpenAlex/Crossref/S2/arXiv/
|
|
7
|
+
PubMed/EuropePMC/DBLP/... plus research-data sources)
|
|
8
|
+
- Protocol workflow: plan / estimate / smoke / calibrate
|
|
9
|
+
- Citation chasing: expand_citations (Semantic Scholar + OpenCitations)
|
|
10
|
+
- Identifier verification: verify (DOI/PMID/arXiv)
|
|
11
|
+
- OA resolution: resolve-oa (Unpaywall)
|
|
12
|
+
- Audit artifacts: coverage / PRISMA / download_queue.json
|
|
13
|
+
|
|
14
|
+
The facade degrades gracefully: every call reports a clear error when the
|
|
15
|
+
``scansci-find`` CLI is missing, and ``search`` callers fall back to the
|
|
16
|
+
built-in three-source search.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
import tempfile
|
|
27
|
+
import time
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
CLI_NAME = "scansci-find"
|
|
34
|
+
DEFAULT_TIMEOUT = 180 # seconds; citation chasing / systematic searches are slow
|
|
35
|
+
# Single-identifier verify / OA resolve must never inherit the systematic-review
|
|
36
|
+
# budget: a lightweight subprocess that is not back within this budget is a
|
|
37
|
+
# dependency problem, not a slow search.
|
|
38
|
+
LIGHT_TIMEOUT = 45
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class DiscoveryUnavailableError(RuntimeError):
|
|
42
|
+
"""Raised when the scansci-find CLI is missing or broken."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class DiscoveryTimeoutError(RuntimeError):
|
|
46
|
+
"""scansci-find exceeded its budget; carries structured fields for MCP."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, message: str, *, command: str, timeout_seconds: int,
|
|
49
|
+
retryable: bool = True) -> None:
|
|
50
|
+
super().__init__(message)
|
|
51
|
+
self.command = command
|
|
52
|
+
self.timeout_seconds = timeout_seconds
|
|
53
|
+
self.retryable = retryable
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _which_cli() -> str | None:
|
|
57
|
+
"""Locate the scansci-find CLI on PATH, else next to this interpreter.
|
|
58
|
+
|
|
59
|
+
MCP clients often launch the server by absolute path without the virtual
|
|
60
|
+
environment's Scripts/bin directory on PATH, so a CLI installed into the
|
|
61
|
+
same venv would otherwise be reported missing.
|
|
62
|
+
"""
|
|
63
|
+
import sys
|
|
64
|
+
return shutil.which(CLI_NAME) or shutil.which(CLI_NAME, path=str(Path(sys.executable).parent))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _cli_path() -> str:
|
|
68
|
+
path = _which_cli()
|
|
69
|
+
if not path:
|
|
70
|
+
raise DiscoveryUnavailableError(
|
|
71
|
+
f"'{CLI_NAME}' not found on PATH. Install ScanSci Find: "
|
|
72
|
+
"pip install git+https://github.com/dysprosium231/easyseek.git "
|
|
73
|
+
"(not on PyPI; this fork also works without the private Journal Scout component). "
|
|
74
|
+
"Alternatively use the local fallbacks: search / verify / resolve-oa / build-queue."
|
|
75
|
+
)
|
|
76
|
+
return path
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def find_cli_available() -> bool:
|
|
80
|
+
"""Check whether the scansci-find CLI is installed and responds."""
|
|
81
|
+
path = _which_cli()
|
|
82
|
+
if not path:
|
|
83
|
+
return False
|
|
84
|
+
try:
|
|
85
|
+
result = subprocess.run(
|
|
86
|
+
[path, "sources"],
|
|
87
|
+
capture_output=True, text=True, timeout=30,
|
|
88
|
+
env=_build_env(),
|
|
89
|
+
)
|
|
90
|
+
return result.returncode == 0 and "catalog" in (result.stdout or "")
|
|
91
|
+
except Exception:
|
|
92
|
+
return False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _build_env() -> dict[str, str]:
|
|
96
|
+
"""Merge litloop config credentials into the subprocess environment.
|
|
97
|
+
|
|
98
|
+
ScanSci Find reads OPENALEX_API_KEY / UNPAYWALL_EMAIL / SEMANTIC_SCHOLAR_API_KEY
|
|
99
|
+
from the environment; pass through what litloop already configures.
|
|
100
|
+
"""
|
|
101
|
+
env = dict(os.environ)
|
|
102
|
+
try:
|
|
103
|
+
from ..config import load_config
|
|
104
|
+
cfg = load_config()
|
|
105
|
+
openalex_key = str(cfg.get("openalex_api_key") or "").strip()
|
|
106
|
+
if openalex_key and not env.get("OPENALEX_API_KEY"):
|
|
107
|
+
env["OPENALEX_API_KEY"] = openalex_key
|
|
108
|
+
email = str(cfg.get("email") or "").strip()
|
|
109
|
+
if email and "@" in email and not env.get("UNPAYWALL_EMAIL"):
|
|
110
|
+
env["UNPAYWALL_EMAIL"] = email
|
|
111
|
+
except Exception:
|
|
112
|
+
pass
|
|
113
|
+
return env
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _run(args: list[str], *, timeout: int = DEFAULT_TIMEOUT) -> dict[str, Any]:
|
|
117
|
+
"""Run scansci-find and parse its stdout JSON.
|
|
118
|
+
|
|
119
|
+
Raises DiscoveryUnavailableError when the CLI is missing, and RuntimeError
|
|
120
|
+
with the CLI's stderr when it fails.
|
|
121
|
+
"""
|
|
122
|
+
cli = _cli_path()
|
|
123
|
+
try:
|
|
124
|
+
proc = subprocess.run(
|
|
125
|
+
[cli, *args],
|
|
126
|
+
capture_output=True, text=True, timeout=timeout,
|
|
127
|
+
env=_build_env(),
|
|
128
|
+
)
|
|
129
|
+
except subprocess.TimeoutExpired:
|
|
130
|
+
raise DiscoveryTimeoutError(
|
|
131
|
+
f"scansci-find {' '.join(args[:2])} timed out after {timeout}s",
|
|
132
|
+
command=" ".join(args[:2]), timeout_seconds=timeout,
|
|
133
|
+
) from None
|
|
134
|
+
except OSError as exc:
|
|
135
|
+
raise DiscoveryUnavailableError(f"Failed to run scansci-find: {exc}") from None
|
|
136
|
+
|
|
137
|
+
if proc.returncode != 0:
|
|
138
|
+
detail = (proc.stderr or proc.stdout or "").strip()[-500:]
|
|
139
|
+
raise RuntimeError(
|
|
140
|
+
f"scansci-find {' '.join(args[:2])} failed (exit {proc.returncode}): {detail}"
|
|
141
|
+
)
|
|
142
|
+
try:
|
|
143
|
+
return json.loads(proc.stdout or "{}")
|
|
144
|
+
except json.JSONDecodeError:
|
|
145
|
+
# Not JSON — some commands print plain text (e.g. warnings); wrap it.
|
|
146
|
+
return {"stdout": proc.stdout, "stderr": proc.stderr}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# ---------------------------------------------------------------------------
|
|
150
|
+
# Protocol workflow proxies
|
|
151
|
+
# ---------------------------------------------------------------------------
|
|
152
|
+
|
|
153
|
+
def plan(query: str, *, domain: str = "general", depth: str = "standard",
|
|
154
|
+
question: str = "", year_from: int | None = None, year_to: int | None = None) -> dict[str, Any]:
|
|
155
|
+
"""Build an auditable search protocol without running a search."""
|
|
156
|
+
args = ["plan", query, "--domain", domain, "--depth", depth]
|
|
157
|
+
if question:
|
|
158
|
+
args += ["--question", question]
|
|
159
|
+
if year_from is not None:
|
|
160
|
+
args += ["--year-from", str(year_from)]
|
|
161
|
+
if year_to is not None:
|
|
162
|
+
args += ["--year-to", str(year_to)]
|
|
163
|
+
return _run(args)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def estimate(query: str, *, domain: str = "general", depth: str = "standard",
|
|
167
|
+
question: str = "", year_from: int | None = None, year_to: int | None = None) -> dict[str, Any]:
|
|
168
|
+
"""Estimate result volume before spending a full search budget."""
|
|
169
|
+
args = ["estimate", query, "--domain", domain, "--depth", depth]
|
|
170
|
+
if question:
|
|
171
|
+
args += ["--question", question]
|
|
172
|
+
if year_from is not None:
|
|
173
|
+
args += ["--year-from", str(year_from)]
|
|
174
|
+
if year_to is not None:
|
|
175
|
+
args += ["--year-to", str(year_to)]
|
|
176
|
+
return _run(args)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def smoke(query: str, *, domain: str = "general",
|
|
180
|
+
records_per_source: int = 4) -> dict[str, Any]:
|
|
181
|
+
"""Fetch a few records per source and validate the candidate contract."""
|
|
182
|
+
return _run(["smoke", query, "--domain", domain,
|
|
183
|
+
"--records-per-source", str(records_per_source)])
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def calibrate(query: str, *, domain: str = "general", depth: str = "standard",
|
|
187
|
+
sample_size: int = 100) -> dict[str, Any]:
|
|
188
|
+
"""Run a bounded calibration sample before a high-recall search."""
|
|
189
|
+
return _run(["calibrate", query, "--domain", domain, "--depth", depth,
|
|
190
|
+
"--sample-size", str(sample_size)])
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# ---------------------------------------------------------------------------
|
|
194
|
+
# Search / citation chasing
|
|
195
|
+
# ---------------------------------------------------------------------------
|
|
196
|
+
|
|
197
|
+
def search(query: str, out_dir: str | Path, *, domain: str = "general",
|
|
198
|
+
depth: str = "quick", limit: int = 20, year_from: int | None = None,
|
|
199
|
+
year_to: int | None = None, sources: list[str] | None = None,
|
|
200
|
+
expand_citations: bool = False, citation_source: str = "semantic",
|
|
201
|
+
citation_rounds: int | None = None, verify_identifiers: bool = False,
|
|
202
|
+
resolve_oa: bool = False, find_preprints: bool = False,
|
|
203
|
+
code_links: bool = False, sort: str = "") -> dict[str, Any]:
|
|
204
|
+
"""Run a ScanSci Find search into ``out_dir`` and return its artifacts.
|
|
205
|
+
|
|
206
|
+
Returns {"out": str, "total": int, "candidates": [...], "download_queue": [...]}.
|
|
207
|
+
"""
|
|
208
|
+
out = Path(out_dir)
|
|
209
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
210
|
+
args = ["search", query, "--out", str(out), "--domain", domain,
|
|
211
|
+
"--depth", depth, "--limit", str(limit)]
|
|
212
|
+
if year_from is not None:
|
|
213
|
+
args += ["--year-from", str(year_from)]
|
|
214
|
+
if year_to is not None:
|
|
215
|
+
args += ["--year-to", str(year_to)]
|
|
216
|
+
if sources:
|
|
217
|
+
args += ["--source", ",".join(sources)]
|
|
218
|
+
if expand_citations:
|
|
219
|
+
args += ["--expand-citations", "--citation-source", citation_source]
|
|
220
|
+
if citation_rounds is not None:
|
|
221
|
+
args += ["--citation-rounds", str(citation_rounds)]
|
|
222
|
+
if verify_identifiers:
|
|
223
|
+
args += ["--verify-identifiers"]
|
|
224
|
+
if resolve_oa:
|
|
225
|
+
args += ["--resolve-oa"]
|
|
226
|
+
if find_preprints:
|
|
227
|
+
args += ["--find-preprints"]
|
|
228
|
+
if code_links:
|
|
229
|
+
args += ["--code-links"]
|
|
230
|
+
if sort:
|
|
231
|
+
args += ["--sort", sort]
|
|
232
|
+
summary = _run(args, timeout=600 if expand_citations else DEFAULT_TIMEOUT)
|
|
233
|
+
|
|
234
|
+
payload: dict[str, Any] = {
|
|
235
|
+
"out": str(out),
|
|
236
|
+
"total": int(summary.get("total", 0)),
|
|
237
|
+
"candidates": _read_json(out / "candidates.json", default=[]),
|
|
238
|
+
"download_queue": _read_json(out / "download_queue.json", default=[]),
|
|
239
|
+
}
|
|
240
|
+
for key in ("coverage_report.json", "prisma.json", "source_report.json"):
|
|
241
|
+
payload[key.replace(".json", "")] = _read_json(out / key, default={})
|
|
242
|
+
return payload
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def expand_citations(query: str, out_dir: str | Path, *, rounds: int = 1,
|
|
246
|
+
citation_source: str = "semantic", depth: str = "standard",
|
|
247
|
+
limit: int = 20, domain: str = "general") -> dict[str, Any]:
|
|
248
|
+
"""Search plus backward/forward citation chasing rounds."""
|
|
249
|
+
return search(
|
|
250
|
+
query, out_dir, domain=domain, depth=depth, limit=limit,
|
|
251
|
+
expand_citations=True, citation_source=citation_source,
|
|
252
|
+
citation_rounds=rounds,
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# ---------------------------------------------------------------------------
|
|
257
|
+
# Verification / OA resolution
|
|
258
|
+
# ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
def verify(candidates: list[dict[str, Any]] | str, *,
|
|
261
|
+
limit: int | None = None, timeout: int = LIGHT_TIMEOUT) -> dict[str, Any]:
|
|
262
|
+
"""Verify DOI/PMID/arXiv identifiers against authoritative APIs."""
|
|
263
|
+
return _identify_pipeline("verify", candidates, limit=limit, timeout=timeout)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def resolve_oa(candidates: list[dict[str, Any]] | str, *,
|
|
267
|
+
limit: int | None = None, timeout: int = LIGHT_TIMEOUT) -> dict[str, Any]:
|
|
268
|
+
"""Resolve DOI open-access locations through Unpaywall."""
|
|
269
|
+
reason = _check_unpaywall_email()
|
|
270
|
+
if reason:
|
|
271
|
+
return {
|
|
272
|
+
"status": "blocked_configuration",
|
|
273
|
+
"provider": "unpaywall",
|
|
274
|
+
"stage": "resolve_oa",
|
|
275
|
+
"reason": reason,
|
|
276
|
+
"hint": "Unpaywall 免费 API 需要邮箱:litloop_config(key='email', value='you@example.com') 后重试",
|
|
277
|
+
}
|
|
278
|
+
return _identify_pipeline("resolve-oa", candidates, limit=limit, timeout=timeout)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _check_unpaywall_email() -> str | None:
|
|
282
|
+
"""Return a reason string when Unpaywall email is missing, else None.
|
|
283
|
+
|
|
284
|
+
Checked before launching the subprocess so a misconfigured batch is
|
|
285
|
+
reported once, up front, instead of failing per-identifier downstream.
|
|
286
|
+
"""
|
|
287
|
+
if _build_env().get("UNPAYWALL_EMAIL"):
|
|
288
|
+
return None
|
|
289
|
+
return "missing Unpaywall email (unpaywall API requires a contact email)"
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _identify_pipeline(command: str, candidates: list[dict[str, Any]] | str,
|
|
293
|
+
*, limit: int | None = None,
|
|
294
|
+
timeout: int = LIGHT_TIMEOUT) -> dict[str, Any]:
|
|
295
|
+
"""Run verify/resolve-oa over a candidate list (path or inline JSON)."""
|
|
296
|
+
temp_created = False
|
|
297
|
+
if isinstance(candidates, str) and Path(candidates).exists():
|
|
298
|
+
input_path = Path(candidates)
|
|
299
|
+
else:
|
|
300
|
+
with tempfile.NamedTemporaryFile(
|
|
301
|
+
"w", suffix=".json", delete=False, encoding="utf-8"
|
|
302
|
+
) as tmp:
|
|
303
|
+
if isinstance(candidates, str):
|
|
304
|
+
tmp.write(candidates)
|
|
305
|
+
else:
|
|
306
|
+
json.dump(candidates, tmp, ensure_ascii=False)
|
|
307
|
+
input_path = Path(tmp.name)
|
|
308
|
+
temp_created = True
|
|
309
|
+
args = [command, str(input_path)]
|
|
310
|
+
if limit is not None:
|
|
311
|
+
args += ["--limit", str(limit)]
|
|
312
|
+
try:
|
|
313
|
+
return _run(args, timeout=timeout)
|
|
314
|
+
finally:
|
|
315
|
+
if temp_created:
|
|
316
|
+
try:
|
|
317
|
+
input_path.unlink(missing_ok=True)
|
|
318
|
+
except Exception:
|
|
319
|
+
pass
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# ---------------------------------------------------------------------------
|
|
323
|
+
# Queue building / legacy field adaptation
|
|
324
|
+
# ---------------------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
def build_download_queue(out_dir: str | Path) -> list[str]:
|
|
327
|
+
"""Extract download identifiers from a find output dir's download_queue.json.
|
|
328
|
+
|
|
329
|
+
Returns a list of DOI/arXiv identifiers ready for ``litloop batch``.
|
|
330
|
+
"""
|
|
331
|
+
out = Path(out_dir)
|
|
332
|
+
queue = _read_json(out / "download_queue.json", default=[])
|
|
333
|
+
identifiers: list[str] = []
|
|
334
|
+
for entry in queue:
|
|
335
|
+
if not isinstance(entry, dict):
|
|
336
|
+
continue
|
|
337
|
+
identifier = entry.get("identifier") or ""
|
|
338
|
+
if identifier:
|
|
339
|
+
identifiers.append(identifier)
|
|
340
|
+
else:
|
|
341
|
+
for key in ("doi", "arxiv_id"):
|
|
342
|
+
value = entry.get(key)
|
|
343
|
+
if value:
|
|
344
|
+
identifiers.append(str(value))
|
|
345
|
+
break
|
|
346
|
+
return identifiers
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def build_preprint_fallbacks(out_dir: str | Path) -> dict[str, list[str]]:
|
|
350
|
+
"""Map queue DOIs to their matched preprint arXiv IDs for fallback retries.
|
|
351
|
+
|
|
352
|
+
Reads ``preprint_identifiers`` from each download_queue entry so a batch
|
|
353
|
+
that fails on a paywalled DOI can retry the open preprint version.
|
|
354
|
+
"""
|
|
355
|
+
out = Path(out_dir)
|
|
356
|
+
queue = _read_json(out / "download_queue.json", default=[])
|
|
357
|
+
fallbacks: dict[str, list[str]] = {}
|
|
358
|
+
for entry in queue:
|
|
359
|
+
if not isinstance(entry, dict):
|
|
360
|
+
continue
|
|
361
|
+
identifier = str(entry.get("identifier") or entry.get("doi") or "").strip()
|
|
362
|
+
preprint_values = []
|
|
363
|
+
for item in entry.get("preprint_identifiers") or []:
|
|
364
|
+
value = str(item.get("value") or "").strip()
|
|
365
|
+
if value:
|
|
366
|
+
preprint_values.append(value)
|
|
367
|
+
if identifier and preprint_values:
|
|
368
|
+
fallbacks[identifier] = preprint_values
|
|
369
|
+
return fallbacks
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def to_legacy_results(candidates: list[dict[str, Any]], *, limit: int = 20) -> list[dict[str, Any]]:
|
|
373
|
+
"""Adapt ScanSci Find candidates to the built-in search result fields.
|
|
374
|
+
|
|
375
|
+
Keeps the existing return contract (doi/title/authors/year/cited_by_count/
|
|
376
|
+
is_oa/oa_url/source) so downstream callers (MCP, litloop_html) are unchanged.
|
|
377
|
+
"""
|
|
378
|
+
results: list[dict[str, Any]] = []
|
|
379
|
+
for candidate in candidates[:limit]:
|
|
380
|
+
if not isinstance(candidate, dict):
|
|
381
|
+
continue
|
|
382
|
+
source_hits = candidate.get("source_hits") or []
|
|
383
|
+
links = candidate.get("links") or {}
|
|
384
|
+
access = candidate.get("access") or {}
|
|
385
|
+
oa_url = links.get("pdf") or links.get("fulltext") or candidate.get("oa_url") or ""
|
|
386
|
+
results.append({
|
|
387
|
+
"doi": candidate.get("doi") or None,
|
|
388
|
+
"arxiv_id": candidate.get("arxiv_id") or None,
|
|
389
|
+
"title": candidate.get("title") or "",
|
|
390
|
+
"authors": candidate.get("authors") or [],
|
|
391
|
+
"year": candidate.get("year"),
|
|
392
|
+
"cited_by_count": int(candidate.get("citation_count") or 0),
|
|
393
|
+
"sources": list(source_hits),
|
|
394
|
+
"is_oa": bool(oa_url or access.get("oa_status")),
|
|
395
|
+
"oa_url": oa_url or None,
|
|
396
|
+
"source": "discovery",
|
|
397
|
+
"abstract": candidate.get("abstract") or "",
|
|
398
|
+
"journal": candidate.get("journal") or "",
|
|
399
|
+
"issn": [x for x in (candidate.get("issn"), candidate.get("eissn")) if x],
|
|
400
|
+
})
|
|
401
|
+
return results
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _read_json(path: Path, *, default: Any) -> Any:
|
|
405
|
+
try:
|
|
406
|
+
if not path.exists():
|
|
407
|
+
return default
|
|
408
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
409
|
+
except Exception:
|
|
410
|
+
return default
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Screening sessions: criteria, per-paper relevance + reason, the user's picks.
|
|
2
|
+
|
|
3
|
+
The agent reads search results, rates each paper (高/中/低/不相关) with a
|
|
4
|
+
one-line reason, and saves them here; the user then chooses which to keep.
|
|
5
|
+
Each session is one JSON file (the record) plus a Markdown table (to read)
|
|
6
|
+
under ``<litloop home>/screening``. Saving again into a session merges by
|
|
7
|
+
DOI, keeps earlier decisions and appends the search to its history, so a
|
|
8
|
+
topic can be searched in several rounds and later continued.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
import time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from .. import config as _config
|
|
19
|
+
from .. import db
|
|
20
|
+
from .. import store
|
|
21
|
+
|
|
22
|
+
RELEVANCE = db.RELEVANCE
|
|
23
|
+
ABSTRACT_CHARS = db.ABSTRACT_CHARS
|
|
24
|
+
_RANK = {r: i for i, r in enumerate(RELEVANCE)}
|
|
25
|
+
_EN = {"high": "高", "medium": "中", "low": "低", "irrelevant": "不相关", "none": "不相关"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def screening_dir() -> Path:
|
|
29
|
+
d = Path(_config.DATA_DIR) / "screening"
|
|
30
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
31
|
+
return d
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _slug(text: str) -> str:
|
|
35
|
+
s = re.sub(r"[^\w一-鿿]+", "-", text.strip().lower()).strip("-")
|
|
36
|
+
return s[:40] or "search"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _key(item: dict[str, Any]) -> str:
|
|
40
|
+
from ..terms.text import paper_key
|
|
41
|
+
return paper_key(item)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _norm_relevance(value: Any) -> str:
|
|
45
|
+
v = str(value or "").strip()
|
|
46
|
+
v = _EN.get(v.lower(), v)
|
|
47
|
+
if v not in _RANK:
|
|
48
|
+
raise ValueError(f"relevance must be one of {RELEVANCE}, got {value!r}")
|
|
49
|
+
return v
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def load(session: str) -> dict[str, Any]:
|
|
53
|
+
return db.load_screening(session)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _rating(item: dict[str, Any]) -> str:
|
|
57
|
+
"""The rating that counts: the user's, set in the app, overrides the agent's."""
|
|
58
|
+
return item.get("user_relevance") or item["relevance"]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _write_markdown(data: dict[str, Any]) -> dict[str, str]:
|
|
62
|
+
md_path = screening_dir() / f"{data['id']}.md"
|
|
63
|
+
store.write_text(md_path, render_markdown(data))
|
|
64
|
+
return {"markdown": str(md_path)}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def save(question: str, items: list[dict[str, Any]], criteria: list[str] | str | None = None,
|
|
68
|
+
query: str = "", mode: str = "quick", session: str | None = None,
|
|
69
|
+
search_params: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
70
|
+
"""Create a session, or merge ITEMS into SESSION. Returns id, paths, counts."""
|
|
71
|
+
res = db.save_screening(
|
|
72
|
+
question,
|
|
73
|
+
items,
|
|
74
|
+
criteria=criteria,
|
|
75
|
+
query=query,
|
|
76
|
+
mode=mode,
|
|
77
|
+
session=session,
|
|
78
|
+
search_params=search_params,
|
|
79
|
+
)
|
|
80
|
+
record = res.pop("record")
|
|
81
|
+
paths = _write_markdown(record)
|
|
82
|
+
return {**res, **paths}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def translate_wanted(config: dict[str, Any] | None = None) -> list[str]:
|
|
86
|
+
"""The ratings whose abstract is translated without asking (config abstract_translation)."""
|
|
87
|
+
from ..config import load_config
|
|
88
|
+
value = str((config or load_config()).get("abstract_translation", "高") or "off")
|
|
89
|
+
if value == "all":
|
|
90
|
+
return list(RELEVANCE)
|
|
91
|
+
return [r for r in RELEVANCE if r in value]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def untranslated(session: str, config: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
95
|
+
"""Items the agent should still translate: only when the agent is the translator
|
|
96
|
+
(translation_provider "agent"), rated in translate_wanted, with an abstract."""
|
|
97
|
+
from ..desktop import translate as _translate
|
|
98
|
+
if _translate.by_api(config):
|
|
99
|
+
return []
|
|
100
|
+
wanted = translate_wanted(config)
|
|
101
|
+
return [_brief(it) for it in load(session)["items"]
|
|
102
|
+
if _rating(it) in wanted and it.get("abstract") and not it.get("abstract_zh")]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def translate(session: str, items: list[dict[str, Any]]) -> dict[str, Any]:
|
|
106
|
+
"""Store the agent's Chinese abstracts: items=[{doi or n, abstract_zh}]. Ratings stay as they are."""
|
|
107
|
+
res = db.translate_screening(session, items)
|
|
108
|
+
return {**res, "left": untranslated(session)}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _brief(it: dict[str, Any]) -> dict[str, Any]:
|
|
112
|
+
return {"n": it["n"], "relevance": it["relevance"], "title": str(it.get("title") or "")[:90],
|
|
113
|
+
"doi": it.get("doi")}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def decide(session: str, selected: list[int | str], rejected: list[int | str] | None = None,
|
|
117
|
+
out_file: str | None = None) -> dict[str, Any]:
|
|
118
|
+
"""Record the user's whole choice by item number or DOI, replacing any
|
|
119
|
+
earlier decision (a corrected pick must not leave the wrong one behind);
|
|
120
|
+
optionally write a download queue."""
|
|
121
|
+
res = db.decide_screening(session, selected, rejected)
|
|
122
|
+
if "error" in res:
|
|
123
|
+
return res
|
|
124
|
+
record = res.pop("record")
|
|
125
|
+
paths = _write_markdown(record)
|
|
126
|
+
result: dict[str, Any] = {**res, **paths}
|
|
127
|
+
chosen = [it for it in record["items"] if it.get("decision") == "selected"]
|
|
128
|
+
if out_file and chosen:
|
|
129
|
+
from ..download.pipeline import QueueEntry, predict_channel, write_queue
|
|
130
|
+
qe = [QueueEntry(identifier=it["doi"], channel=predict_channel(it["doi"]), title=it.get("title", ""))
|
|
131
|
+
for it in chosen if it.get("doi")]
|
|
132
|
+
result["queue_file"] = str(write_queue(qe, out_file))
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def list_sessions(limit: int = 20) -> list[dict[str, Any]]:
|
|
137
|
+
return db.list_sessions(limit)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def prisma(session: str) -> dict[str, Any]:
|
|
141
|
+
"""The review's flow counts from what this session recorded (PRISMA-style).
|
|
142
|
+
|
|
143
|
+
Records identified are what the searches returned; duplicates are those that
|
|
144
|
+
were already in the session; screening is every paper the user was shown.
|
|
145
|
+
Exclusions are counted separately for the user's "不要" and for the low
|
|
146
|
+
ratings the agent gave without the user deciding, with the reasons attached.
|
|
147
|
+
"""
|
|
148
|
+
data = load(session)
|
|
149
|
+
items = data.get("items", [])
|
|
150
|
+
searches = [{"query": s.get("query", ""), "time": s.get("time", ""),
|
|
151
|
+
"results": int(s.get("results") or 0), "new": int(s.get("new") or 0)}
|
|
152
|
+
for s in data.get("searches", [])]
|
|
153
|
+
selected = [it for it in items if it.get("decision") == "selected"]
|
|
154
|
+
rejected = [it for it in items if it.get("decision") == "rejected"]
|
|
155
|
+
low = [it for it in items if it.get("decision") is None and _rating(it) in ("低", "不相关")]
|
|
156
|
+
reasons: dict[str, int] = {}
|
|
157
|
+
for it in rejected + low:
|
|
158
|
+
# The user's own reason outranks the agent's when they gave one.
|
|
159
|
+
text = " ".join(str(it.get("reject_reason") or it.get("reason") or "").split())[:120]
|
|
160
|
+
if text:
|
|
161
|
+
reasons[text] = reasons.get(text, 0) + 1
|
|
162
|
+
return {
|
|
163
|
+
"session": data.get("id", session), "question": data.get("question", ""),
|
|
164
|
+
"mode": data.get("mode", "quick"), "created": data.get("created", ""),
|
|
165
|
+
"updated": data.get("updated", ""), "searches": searches,
|
|
166
|
+
"identified": sum(s["results"] for s in searches),
|
|
167
|
+
"duplicates": sum(max(0, s["results"] - s["new"]) for s in searches) if searches else 0,
|
|
168
|
+
"screened": len(items),
|
|
169
|
+
"by_relevance": {r: sum(1 for it in items if _rating(it) == r) for r in RELEVANCE},
|
|
170
|
+
"included": len(selected),
|
|
171
|
+
"excluded": len(rejected),
|
|
172
|
+
"low_without_decision": len(low),
|
|
173
|
+
"reasons": [{"reason": k, "count": v} for k, v in sorted(reasons.items(), key=lambda kv: -kv[1])[:10]],
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def history(limit: int = 500) -> list[dict[str, Any]]:
|
|
178
|
+
"""Every screening session for the app's history: what was asked, how far it got, what was kept.
|
|
179
|
+
|
|
180
|
+
"selected" lists the final picks (title, year, journal, DOI), newest session first.
|
|
181
|
+
"""
|
|
182
|
+
return db.history(limit)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
_DECISION = {"selected": "✅ 选中", "rejected": "✗ 不要", None: ""}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def render_markdown(data: dict[str, Any]) -> str:
|
|
189
|
+
lines = [f"# 文献筛选:{data.get('question') or ''}", "",
|
|
190
|
+
f"- 会话:`{data['id']}`({data.get('mode', '')}),更新于 {data.get('updated', '')}"]
|
|
191
|
+
if data.get("criteria"):
|
|
192
|
+
lines += ["- 筛选标准:", *[f" - {c}" for c in data["criteria"]]]
|
|
193
|
+
if data.get("searches"):
|
|
194
|
+
lines.append("- 检索记录:" + ";".join(f"`{s['query']}`({s['time']},{s['results']} 篇)"
|
|
195
|
+
for s in data["searches"]))
|
|
196
|
+
lines += ["", "| # | 相关度 | 文章 | 年份 | 期刊 / 分区 | 理由 | 选择 |",
|
|
197
|
+
"|---|---|---|---|---|---|---|"]
|
|
198
|
+
for it in data["items"]:
|
|
199
|
+
title = str(it.get("title") or "").replace("|", "/")
|
|
200
|
+
if it.get("doi"):
|
|
201
|
+
title = f"[{title}](https://doi.org/{it['doi']})"
|
|
202
|
+
flags = ("⚠已撤稿 " if it.get("retracted") else "") + ("⚠关注声明 " if it.get("concern") else "") \
|
|
203
|
+
+ ("(仅凭标题)" if it.get("abstract_missing") else "")
|
|
204
|
+
venue = " ".join(x for x in (str(it.get("journal") or ""), str(it.get("journal_rank") or "")) if x)
|
|
205
|
+
lines.append(f"| {it['n']} | {it['relevance']} | {title} | {it.get('year') or ''} | "
|
|
206
|
+
f"{venue.replace('|', '/')} | {flags}{str(it.get('reason') or '').replace('|', '/')} | "
|
|
207
|
+
f"{_DECISION.get(it.get('decision'), '')} |")
|
|
208
|
+
return "\n".join(lines) + "\n"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def prisma_markdown(session: str) -> str:
|
|
212
|
+
"""The flow counts as a Markdown table, for a manuscript's record."""
|
|
213
|
+
d = prisma(session)
|
|
214
|
+
lines = [f"# 检索流程:{d['question'] or '(没有记录检索问题)'}", "",
|
|
215
|
+
"| 阶段 | 篇数 |", "| --- | --- |",
|
|
216
|
+
f"| 检索命中 | {d['identified']} |", f"| 去重后 | {d['screened']} |",
|
|
217
|
+
f"| 已筛 | {d['screened']} |", f"| 纳入 | {d['included']} |",
|
|
218
|
+
f"| 排除 | {d['excluded']} |", f"| 低相关未定 | {d['low_without_decision']} |", ""]
|
|
219
|
+
if d["searches"]:
|
|
220
|
+
lines += ["## 检索", "", "| 时间 | 检索词 | 返回 | 新增 |", "| --- | --- | --- | --- |"]
|
|
221
|
+
lines += [f"| {s['time']} | {s['query']} | {s['results']} | {s['new']} |" for s in d["searches"]]
|
|
222
|
+
lines.append("")
|
|
223
|
+
if d["reasons"]:
|
|
224
|
+
lines += ["## 排除理由", ""] + [f"- {r['reason']}({r['count']} 篇)" for r in d["reasons"]] + [""]
|
|
225
|
+
return "\n".join(lines)
|