litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,410 @@
1
+ """Discovery layer: ScanSci Find CLI facade.
2
+
3
+ Brings ScanSci Find's six capability areas into litloop by delegating to
4
+ the ``scansci-find`` CLI at runtime (no code copy):
5
+
6
+ - 13-source federated search with domain routing (OpenAlex/Crossref/S2/arXiv/
7
+ PubMed/EuropePMC/DBLP/... plus research-data sources)
8
+ - Protocol workflow: plan / estimate / smoke / calibrate
9
+ - Citation chasing: expand_citations (Semantic Scholar + OpenCitations)
10
+ - Identifier verification: verify (DOI/PMID/arXiv)
11
+ - OA resolution: resolve-oa (Unpaywall)
12
+ - Audit artifacts: coverage / PRISMA / download_queue.json
13
+
14
+ The facade degrades gracefully: every call reports a clear error when the
15
+ ``scansci-find`` CLI is missing, and ``search`` callers fall back to the
16
+ built-in three-source search.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import logging
23
+ import os
24
+ import shutil
25
+ import subprocess
26
+ import tempfile
27
+ import time
28
+ from pathlib import Path
29
+ from typing import Any
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+ CLI_NAME = "scansci-find"
34
+ DEFAULT_TIMEOUT = 180 # seconds; citation chasing / systematic searches are slow
35
+ # Single-identifier verify / OA resolve must never inherit the systematic-review
36
+ # budget: a lightweight subprocess that is not back within this budget is a
37
+ # dependency problem, not a slow search.
38
+ LIGHT_TIMEOUT = 45
39
+
40
+
41
+ class DiscoveryUnavailableError(RuntimeError):
42
+ """Raised when the scansci-find CLI is missing or broken."""
43
+
44
+
45
+ class DiscoveryTimeoutError(RuntimeError):
46
+ """scansci-find exceeded its budget; carries structured fields for MCP."""
47
+
48
+ def __init__(self, message: str, *, command: str, timeout_seconds: int,
49
+ retryable: bool = True) -> None:
50
+ super().__init__(message)
51
+ self.command = command
52
+ self.timeout_seconds = timeout_seconds
53
+ self.retryable = retryable
54
+
55
+
56
+ def _which_cli() -> str | None:
57
+ """Locate the scansci-find CLI on PATH, else next to this interpreter.
58
+
59
+ MCP clients often launch the server by absolute path without the virtual
60
+ environment's Scripts/bin directory on PATH, so a CLI installed into the
61
+ same venv would otherwise be reported missing.
62
+ """
63
+ import sys
64
+ return shutil.which(CLI_NAME) or shutil.which(CLI_NAME, path=str(Path(sys.executable).parent))
65
+
66
+
67
+ def _cli_path() -> str:
68
+ path = _which_cli()
69
+ if not path:
70
+ raise DiscoveryUnavailableError(
71
+ f"'{CLI_NAME}' not found on PATH. Install ScanSci Find: "
72
+ "pip install git+https://github.com/dysprosium231/easyseek.git "
73
+ "(not on PyPI; this fork also works without the private Journal Scout component). "
74
+ "Alternatively use the local fallbacks: search / verify / resolve-oa / build-queue."
75
+ )
76
+ return path
77
+
78
+
79
+ def find_cli_available() -> bool:
80
+ """Check whether the scansci-find CLI is installed and responds."""
81
+ path = _which_cli()
82
+ if not path:
83
+ return False
84
+ try:
85
+ result = subprocess.run(
86
+ [path, "sources"],
87
+ capture_output=True, text=True, timeout=30,
88
+ env=_build_env(),
89
+ )
90
+ return result.returncode == 0 and "catalog" in (result.stdout or "")
91
+ except Exception:
92
+ return False
93
+
94
+
95
+ def _build_env() -> dict[str, str]:
96
+ """Merge litloop config credentials into the subprocess environment.
97
+
98
+ ScanSci Find reads OPENALEX_API_KEY / UNPAYWALL_EMAIL / SEMANTIC_SCHOLAR_API_KEY
99
+ from the environment; pass through what litloop already configures.
100
+ """
101
+ env = dict(os.environ)
102
+ try:
103
+ from ..config import load_config
104
+ cfg = load_config()
105
+ openalex_key = str(cfg.get("openalex_api_key") or "").strip()
106
+ if openalex_key and not env.get("OPENALEX_API_KEY"):
107
+ env["OPENALEX_API_KEY"] = openalex_key
108
+ email = str(cfg.get("email") or "").strip()
109
+ if email and "@" in email and not env.get("UNPAYWALL_EMAIL"):
110
+ env["UNPAYWALL_EMAIL"] = email
111
+ except Exception:
112
+ pass
113
+ return env
114
+
115
+
116
+ def _run(args: list[str], *, timeout: int = DEFAULT_TIMEOUT) -> dict[str, Any]:
117
+ """Run scansci-find and parse its stdout JSON.
118
+
119
+ Raises DiscoveryUnavailableError when the CLI is missing, and RuntimeError
120
+ with the CLI's stderr when it fails.
121
+ """
122
+ cli = _cli_path()
123
+ try:
124
+ proc = subprocess.run(
125
+ [cli, *args],
126
+ capture_output=True, text=True, timeout=timeout,
127
+ env=_build_env(),
128
+ )
129
+ except subprocess.TimeoutExpired:
130
+ raise DiscoveryTimeoutError(
131
+ f"scansci-find {' '.join(args[:2])} timed out after {timeout}s",
132
+ command=" ".join(args[:2]), timeout_seconds=timeout,
133
+ ) from None
134
+ except OSError as exc:
135
+ raise DiscoveryUnavailableError(f"Failed to run scansci-find: {exc}") from None
136
+
137
+ if proc.returncode != 0:
138
+ detail = (proc.stderr or proc.stdout or "").strip()[-500:]
139
+ raise RuntimeError(
140
+ f"scansci-find {' '.join(args[:2])} failed (exit {proc.returncode}): {detail}"
141
+ )
142
+ try:
143
+ return json.loads(proc.stdout or "{}")
144
+ except json.JSONDecodeError:
145
+ # Not JSON — some commands print plain text (e.g. warnings); wrap it.
146
+ return {"stdout": proc.stdout, "stderr": proc.stderr}
147
+
148
+
149
+ # ---------------------------------------------------------------------------
150
+ # Protocol workflow proxies
151
+ # ---------------------------------------------------------------------------
152
+
153
+ def plan(query: str, *, domain: str = "general", depth: str = "standard",
154
+ question: str = "", year_from: int | None = None, year_to: int | None = None) -> dict[str, Any]:
155
+ """Build an auditable search protocol without running a search."""
156
+ args = ["plan", query, "--domain", domain, "--depth", depth]
157
+ if question:
158
+ args += ["--question", question]
159
+ if year_from is not None:
160
+ args += ["--year-from", str(year_from)]
161
+ if year_to is not None:
162
+ args += ["--year-to", str(year_to)]
163
+ return _run(args)
164
+
165
+
166
+ def estimate(query: str, *, domain: str = "general", depth: str = "standard",
167
+ question: str = "", year_from: int | None = None, year_to: int | None = None) -> dict[str, Any]:
168
+ """Estimate result volume before spending a full search budget."""
169
+ args = ["estimate", query, "--domain", domain, "--depth", depth]
170
+ if question:
171
+ args += ["--question", question]
172
+ if year_from is not None:
173
+ args += ["--year-from", str(year_from)]
174
+ if year_to is not None:
175
+ args += ["--year-to", str(year_to)]
176
+ return _run(args)
177
+
178
+
179
+ def smoke(query: str, *, domain: str = "general",
180
+ records_per_source: int = 4) -> dict[str, Any]:
181
+ """Fetch a few records per source and validate the candidate contract."""
182
+ return _run(["smoke", query, "--domain", domain,
183
+ "--records-per-source", str(records_per_source)])
184
+
185
+
186
+ def calibrate(query: str, *, domain: str = "general", depth: str = "standard",
187
+ sample_size: int = 100) -> dict[str, Any]:
188
+ """Run a bounded calibration sample before a high-recall search."""
189
+ return _run(["calibrate", query, "--domain", domain, "--depth", depth,
190
+ "--sample-size", str(sample_size)])
191
+
192
+
193
+ # ---------------------------------------------------------------------------
194
+ # Search / citation chasing
195
+ # ---------------------------------------------------------------------------
196
+
197
+ def search(query: str, out_dir: str | Path, *, domain: str = "general",
198
+ depth: str = "quick", limit: int = 20, year_from: int | None = None,
199
+ year_to: int | None = None, sources: list[str] | None = None,
200
+ expand_citations: bool = False, citation_source: str = "semantic",
201
+ citation_rounds: int | None = None, verify_identifiers: bool = False,
202
+ resolve_oa: bool = False, find_preprints: bool = False,
203
+ code_links: bool = False, sort: str = "") -> dict[str, Any]:
204
+ """Run a ScanSci Find search into ``out_dir`` and return its artifacts.
205
+
206
+ Returns {"out": str, "total": int, "candidates": [...], "download_queue": [...]}.
207
+ """
208
+ out = Path(out_dir)
209
+ out.mkdir(parents=True, exist_ok=True)
210
+ args = ["search", query, "--out", str(out), "--domain", domain,
211
+ "--depth", depth, "--limit", str(limit)]
212
+ if year_from is not None:
213
+ args += ["--year-from", str(year_from)]
214
+ if year_to is not None:
215
+ args += ["--year-to", str(year_to)]
216
+ if sources:
217
+ args += ["--source", ",".join(sources)]
218
+ if expand_citations:
219
+ args += ["--expand-citations", "--citation-source", citation_source]
220
+ if citation_rounds is not None:
221
+ args += ["--citation-rounds", str(citation_rounds)]
222
+ if verify_identifiers:
223
+ args += ["--verify-identifiers"]
224
+ if resolve_oa:
225
+ args += ["--resolve-oa"]
226
+ if find_preprints:
227
+ args += ["--find-preprints"]
228
+ if code_links:
229
+ args += ["--code-links"]
230
+ if sort:
231
+ args += ["--sort", sort]
232
+ summary = _run(args, timeout=600 if expand_citations else DEFAULT_TIMEOUT)
233
+
234
+ payload: dict[str, Any] = {
235
+ "out": str(out),
236
+ "total": int(summary.get("total", 0)),
237
+ "candidates": _read_json(out / "candidates.json", default=[]),
238
+ "download_queue": _read_json(out / "download_queue.json", default=[]),
239
+ }
240
+ for key in ("coverage_report.json", "prisma.json", "source_report.json"):
241
+ payload[key.replace(".json", "")] = _read_json(out / key, default={})
242
+ return payload
243
+
244
+
245
+ def expand_citations(query: str, out_dir: str | Path, *, rounds: int = 1,
246
+ citation_source: str = "semantic", depth: str = "standard",
247
+ limit: int = 20, domain: str = "general") -> dict[str, Any]:
248
+ """Search plus backward/forward citation chasing rounds."""
249
+ return search(
250
+ query, out_dir, domain=domain, depth=depth, limit=limit,
251
+ expand_citations=True, citation_source=citation_source,
252
+ citation_rounds=rounds,
253
+ )
254
+
255
+
256
+ # ---------------------------------------------------------------------------
257
+ # Verification / OA resolution
258
+ # ---------------------------------------------------------------------------
259
+
260
+ def verify(candidates: list[dict[str, Any]] | str, *,
261
+ limit: int | None = None, timeout: int = LIGHT_TIMEOUT) -> dict[str, Any]:
262
+ """Verify DOI/PMID/arXiv identifiers against authoritative APIs."""
263
+ return _identify_pipeline("verify", candidates, limit=limit, timeout=timeout)
264
+
265
+
266
+ def resolve_oa(candidates: list[dict[str, Any]] | str, *,
267
+ limit: int | None = None, timeout: int = LIGHT_TIMEOUT) -> dict[str, Any]:
268
+ """Resolve DOI open-access locations through Unpaywall."""
269
+ reason = _check_unpaywall_email()
270
+ if reason:
271
+ return {
272
+ "status": "blocked_configuration",
273
+ "provider": "unpaywall",
274
+ "stage": "resolve_oa",
275
+ "reason": reason,
276
+ "hint": "Unpaywall 免费 API 需要邮箱:litloop_config(key='email', value='you@example.com') 后重试",
277
+ }
278
+ return _identify_pipeline("resolve-oa", candidates, limit=limit, timeout=timeout)
279
+
280
+
281
+ def _check_unpaywall_email() -> str | None:
282
+ """Return a reason string when Unpaywall email is missing, else None.
283
+
284
+ Checked before launching the subprocess so a misconfigured batch is
285
+ reported once, up front, instead of failing per-identifier downstream.
286
+ """
287
+ if _build_env().get("UNPAYWALL_EMAIL"):
288
+ return None
289
+ return "missing Unpaywall email (unpaywall API requires a contact email)"
290
+
291
+
292
+ def _identify_pipeline(command: str, candidates: list[dict[str, Any]] | str,
293
+ *, limit: int | None = None,
294
+ timeout: int = LIGHT_TIMEOUT) -> dict[str, Any]:
295
+ """Run verify/resolve-oa over a candidate list (path or inline JSON)."""
296
+ temp_created = False
297
+ if isinstance(candidates, str) and Path(candidates).exists():
298
+ input_path = Path(candidates)
299
+ else:
300
+ with tempfile.NamedTemporaryFile(
301
+ "w", suffix=".json", delete=False, encoding="utf-8"
302
+ ) as tmp:
303
+ if isinstance(candidates, str):
304
+ tmp.write(candidates)
305
+ else:
306
+ json.dump(candidates, tmp, ensure_ascii=False)
307
+ input_path = Path(tmp.name)
308
+ temp_created = True
309
+ args = [command, str(input_path)]
310
+ if limit is not None:
311
+ args += ["--limit", str(limit)]
312
+ try:
313
+ return _run(args, timeout=timeout)
314
+ finally:
315
+ if temp_created:
316
+ try:
317
+ input_path.unlink(missing_ok=True)
318
+ except Exception:
319
+ pass
320
+
321
+
322
+ # ---------------------------------------------------------------------------
323
+ # Queue building / legacy field adaptation
324
+ # ---------------------------------------------------------------------------
325
+
326
+ def build_download_queue(out_dir: str | Path) -> list[str]:
327
+ """Extract download identifiers from a find output dir's download_queue.json.
328
+
329
+ Returns a list of DOI/arXiv identifiers ready for ``litloop batch``.
330
+ """
331
+ out = Path(out_dir)
332
+ queue = _read_json(out / "download_queue.json", default=[])
333
+ identifiers: list[str] = []
334
+ for entry in queue:
335
+ if not isinstance(entry, dict):
336
+ continue
337
+ identifier = entry.get("identifier") or ""
338
+ if identifier:
339
+ identifiers.append(identifier)
340
+ else:
341
+ for key in ("doi", "arxiv_id"):
342
+ value = entry.get(key)
343
+ if value:
344
+ identifiers.append(str(value))
345
+ break
346
+ return identifiers
347
+
348
+
349
+ def build_preprint_fallbacks(out_dir: str | Path) -> dict[str, list[str]]:
350
+ """Map queue DOIs to their matched preprint arXiv IDs for fallback retries.
351
+
352
+ Reads ``preprint_identifiers`` from each download_queue entry so a batch
353
+ that fails on a paywalled DOI can retry the open preprint version.
354
+ """
355
+ out = Path(out_dir)
356
+ queue = _read_json(out / "download_queue.json", default=[])
357
+ fallbacks: dict[str, list[str]] = {}
358
+ for entry in queue:
359
+ if not isinstance(entry, dict):
360
+ continue
361
+ identifier = str(entry.get("identifier") or entry.get("doi") or "").strip()
362
+ preprint_values = []
363
+ for item in entry.get("preprint_identifiers") or []:
364
+ value = str(item.get("value") or "").strip()
365
+ if value:
366
+ preprint_values.append(value)
367
+ if identifier and preprint_values:
368
+ fallbacks[identifier] = preprint_values
369
+ return fallbacks
370
+
371
+
372
+ def to_legacy_results(candidates: list[dict[str, Any]], *, limit: int = 20) -> list[dict[str, Any]]:
373
+ """Adapt ScanSci Find candidates to the built-in search result fields.
374
+
375
+ Keeps the existing return contract (doi/title/authors/year/cited_by_count/
376
+ is_oa/oa_url/source) so downstream callers (MCP, litloop_html) are unchanged.
377
+ """
378
+ results: list[dict[str, Any]] = []
379
+ for candidate in candidates[:limit]:
380
+ if not isinstance(candidate, dict):
381
+ continue
382
+ source_hits = candidate.get("source_hits") or []
383
+ links = candidate.get("links") or {}
384
+ access = candidate.get("access") or {}
385
+ oa_url = links.get("pdf") or links.get("fulltext") or candidate.get("oa_url") or ""
386
+ results.append({
387
+ "doi": candidate.get("doi") or None,
388
+ "arxiv_id": candidate.get("arxiv_id") or None,
389
+ "title": candidate.get("title") or "",
390
+ "authors": candidate.get("authors") or [],
391
+ "year": candidate.get("year"),
392
+ "cited_by_count": int(candidate.get("citation_count") or 0),
393
+ "sources": list(source_hits),
394
+ "is_oa": bool(oa_url or access.get("oa_status")),
395
+ "oa_url": oa_url or None,
396
+ "source": "discovery",
397
+ "abstract": candidate.get("abstract") or "",
398
+ "journal": candidate.get("journal") or "",
399
+ "issn": [x for x in (candidate.get("issn"), candidate.get("eissn")) if x],
400
+ })
401
+ return results
402
+
403
+
404
+ def _read_json(path: Path, *, default: Any) -> Any:
405
+ try:
406
+ if not path.exists():
407
+ return default
408
+ return json.loads(path.read_text(encoding="utf-8"))
409
+ except Exception:
410
+ return default
@@ -0,0 +1,225 @@
1
+ """Screening sessions: criteria, per-paper relevance + reason, the user's picks.
2
+
3
+ The agent reads search results, rates each paper (高/中/低/不相关) with a
4
+ one-line reason, and saves them here; the user then chooses which to keep.
5
+ Each session is one JSON file (the record) plus a Markdown table (to read)
6
+ under ``<litloop home>/screening``. Saving again into a session merges by
7
+ DOI, keeps earlier decisions and appends the search to its history, so a
8
+ topic can be searched in several rounds and later continued.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ import time
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from .. import config as _config
19
+ from .. import db
20
+ from .. import store
21
+
22
+ RELEVANCE = db.RELEVANCE
23
+ ABSTRACT_CHARS = db.ABSTRACT_CHARS
24
+ _RANK = {r: i for i, r in enumerate(RELEVANCE)}
25
+ _EN = {"high": "高", "medium": "中", "low": "低", "irrelevant": "不相关", "none": "不相关"}
26
+
27
+
28
+ def screening_dir() -> Path:
29
+ d = Path(_config.DATA_DIR) / "screening"
30
+ d.mkdir(parents=True, exist_ok=True)
31
+ return d
32
+
33
+
34
+ def _slug(text: str) -> str:
35
+ s = re.sub(r"[^\w一-鿿]+", "-", text.strip().lower()).strip("-")
36
+ return s[:40] or "search"
37
+
38
+
39
+ def _key(item: dict[str, Any]) -> str:
40
+ from ..terms.text import paper_key
41
+ return paper_key(item)
42
+
43
+
44
+ def _norm_relevance(value: Any) -> str:
45
+ v = str(value or "").strip()
46
+ v = _EN.get(v.lower(), v)
47
+ if v not in _RANK:
48
+ raise ValueError(f"relevance must be one of {RELEVANCE}, got {value!r}")
49
+ return v
50
+
51
+
52
+ def load(session: str) -> dict[str, Any]:
53
+ return db.load_screening(session)
54
+
55
+
56
+ def _rating(item: dict[str, Any]) -> str:
57
+ """The rating that counts: the user's, set in the app, overrides the agent's."""
58
+ return item.get("user_relevance") or item["relevance"]
59
+
60
+
61
+ def _write_markdown(data: dict[str, Any]) -> dict[str, str]:
62
+ md_path = screening_dir() / f"{data['id']}.md"
63
+ store.write_text(md_path, render_markdown(data))
64
+ return {"markdown": str(md_path)}
65
+
66
+
67
+ def save(question: str, items: list[dict[str, Any]], criteria: list[str] | str | None = None,
68
+ query: str = "", mode: str = "quick", session: str | None = None,
69
+ search_params: dict[str, Any] | None = None) -> dict[str, Any]:
70
+ """Create a session, or merge ITEMS into SESSION. Returns id, paths, counts."""
71
+ res = db.save_screening(
72
+ question,
73
+ items,
74
+ criteria=criteria,
75
+ query=query,
76
+ mode=mode,
77
+ session=session,
78
+ search_params=search_params,
79
+ )
80
+ record = res.pop("record")
81
+ paths = _write_markdown(record)
82
+ return {**res, **paths}
83
+
84
+
85
+ def translate_wanted(config: dict[str, Any] | None = None) -> list[str]:
86
+ """The ratings whose abstract is translated without asking (config abstract_translation)."""
87
+ from ..config import load_config
88
+ value = str((config or load_config()).get("abstract_translation", "高") or "off")
89
+ if value == "all":
90
+ return list(RELEVANCE)
91
+ return [r for r in RELEVANCE if r in value]
92
+
93
+
94
+ def untranslated(session: str, config: dict[str, Any] | None = None) -> list[dict[str, Any]]:
95
+ """Items the agent should still translate: only when the agent is the translator
96
+ (translation_provider "agent"), rated in translate_wanted, with an abstract."""
97
+ from ..desktop import translate as _translate
98
+ if _translate.by_api(config):
99
+ return []
100
+ wanted = translate_wanted(config)
101
+ return [_brief(it) for it in load(session)["items"]
102
+ if _rating(it) in wanted and it.get("abstract") and not it.get("abstract_zh")]
103
+
104
+
105
+ def translate(session: str, items: list[dict[str, Any]]) -> dict[str, Any]:
106
+ """Store the agent's Chinese abstracts: items=[{doi or n, abstract_zh}]. Ratings stay as they are."""
107
+ res = db.translate_screening(session, items)
108
+ return {**res, "left": untranslated(session)}
109
+
110
+
111
+ def _brief(it: dict[str, Any]) -> dict[str, Any]:
112
+ return {"n": it["n"], "relevance": it["relevance"], "title": str(it.get("title") or "")[:90],
113
+ "doi": it.get("doi")}
114
+
115
+
116
+ def decide(session: str, selected: list[int | str], rejected: list[int | str] | None = None,
117
+ out_file: str | None = None) -> dict[str, Any]:
118
+ """Record the user's whole choice by item number or DOI, replacing any
119
+ earlier decision (a corrected pick must not leave the wrong one behind);
120
+ optionally write a download queue."""
121
+ res = db.decide_screening(session, selected, rejected)
122
+ if "error" in res:
123
+ return res
124
+ record = res.pop("record")
125
+ paths = _write_markdown(record)
126
+ result: dict[str, Any] = {**res, **paths}
127
+ chosen = [it for it in record["items"] if it.get("decision") == "selected"]
128
+ if out_file and chosen:
129
+ from ..download.pipeline import QueueEntry, predict_channel, write_queue
130
+ qe = [QueueEntry(identifier=it["doi"], channel=predict_channel(it["doi"]), title=it.get("title", ""))
131
+ for it in chosen if it.get("doi")]
132
+ result["queue_file"] = str(write_queue(qe, out_file))
133
+ return result
134
+
135
+
136
+ def list_sessions(limit: int = 20) -> list[dict[str, Any]]:
137
+ return db.list_sessions(limit)
138
+
139
+
140
+ def prisma(session: str) -> dict[str, Any]:
141
+ """The review's flow counts from what this session recorded (PRISMA-style).
142
+
143
+ Records identified are what the searches returned; duplicates are those that
144
+ were already in the session; screening is every paper the user was shown.
145
+ Exclusions are counted separately for the user's "不要" and for the low
146
+ ratings the agent gave without the user deciding, with the reasons attached.
147
+ """
148
+ data = load(session)
149
+ items = data.get("items", [])
150
+ searches = [{"query": s.get("query", ""), "time": s.get("time", ""),
151
+ "results": int(s.get("results") or 0), "new": int(s.get("new") or 0)}
152
+ for s in data.get("searches", [])]
153
+ selected = [it for it in items if it.get("decision") == "selected"]
154
+ rejected = [it for it in items if it.get("decision") == "rejected"]
155
+ low = [it for it in items if it.get("decision") is None and _rating(it) in ("低", "不相关")]
156
+ reasons: dict[str, int] = {}
157
+ for it in rejected + low:
158
+ # The user's own reason outranks the agent's when they gave one.
159
+ text = " ".join(str(it.get("reject_reason") or it.get("reason") or "").split())[:120]
160
+ if text:
161
+ reasons[text] = reasons.get(text, 0) + 1
162
+ return {
163
+ "session": data.get("id", session), "question": data.get("question", ""),
164
+ "mode": data.get("mode", "quick"), "created": data.get("created", ""),
165
+ "updated": data.get("updated", ""), "searches": searches,
166
+ "identified": sum(s["results"] for s in searches),
167
+ "duplicates": sum(max(0, s["results"] - s["new"]) for s in searches) if searches else 0,
168
+ "screened": len(items),
169
+ "by_relevance": {r: sum(1 for it in items if _rating(it) == r) for r in RELEVANCE},
170
+ "included": len(selected),
171
+ "excluded": len(rejected),
172
+ "low_without_decision": len(low),
173
+ "reasons": [{"reason": k, "count": v} for k, v in sorted(reasons.items(), key=lambda kv: -kv[1])[:10]],
174
+ }
175
+
176
+
177
+ def history(limit: int = 500) -> list[dict[str, Any]]:
178
+ """Every screening session for the app's history: what was asked, how far it got, what was kept.
179
+
180
+ "selected" lists the final picks (title, year, journal, DOI), newest session first.
181
+ """
182
+ return db.history(limit)
183
+
184
+
185
+ _DECISION = {"selected": "✅ 选中", "rejected": "✗ 不要", None: ""}
186
+
187
+
188
+ def render_markdown(data: dict[str, Any]) -> str:
189
+ lines = [f"# 文献筛选:{data.get('question') or ''}", "",
190
+ f"- 会话:`{data['id']}`({data.get('mode', '')}),更新于 {data.get('updated', '')}"]
191
+ if data.get("criteria"):
192
+ lines += ["- 筛选标准:", *[f" - {c}" for c in data["criteria"]]]
193
+ if data.get("searches"):
194
+ lines.append("- 检索记录:" + ";".join(f"`{s['query']}`({s['time']},{s['results']} 篇)"
195
+ for s in data["searches"]))
196
+ lines += ["", "| # | 相关度 | 文章 | 年份 | 期刊 / 分区 | 理由 | 选择 |",
197
+ "|---|---|---|---|---|---|---|"]
198
+ for it in data["items"]:
199
+ title = str(it.get("title") or "").replace("|", "/")
200
+ if it.get("doi"):
201
+ title = f"[{title}](https://doi.org/{it['doi']})"
202
+ flags = ("⚠已撤稿 " if it.get("retracted") else "") + ("⚠关注声明 " if it.get("concern") else "") \
203
+ + ("(仅凭标题)" if it.get("abstract_missing") else "")
204
+ venue = " ".join(x for x in (str(it.get("journal") or ""), str(it.get("journal_rank") or "")) if x)
205
+ lines.append(f"| {it['n']} | {it['relevance']} | {title} | {it.get('year') or ''} | "
206
+ f"{venue.replace('|', '/')} | {flags}{str(it.get('reason') or '').replace('|', '/')} | "
207
+ f"{_DECISION.get(it.get('decision'), '')} |")
208
+ return "\n".join(lines) + "\n"
209
+
210
+
211
+ def prisma_markdown(session: str) -> str:
212
+ """The flow counts as a Markdown table, for a manuscript's record."""
213
+ d = prisma(session)
214
+ lines = [f"# 检索流程:{d['question'] or '(没有记录检索问题)'}", "",
215
+ "| 阶段 | 篇数 |", "| --- | --- |",
216
+ f"| 检索命中 | {d['identified']} |", f"| 去重后 | {d['screened']} |",
217
+ f"| 已筛 | {d['screened']} |", f"| 纳入 | {d['included']} |",
218
+ f"| 排除 | {d['excluded']} |", f"| 低相关未定 | {d['low_without_decision']} |", ""]
219
+ if d["searches"]:
220
+ lines += ["## 检索", "", "| 时间 | 检索词 | 返回 | 新增 |", "| --- | --- | --- | --- |"]
221
+ lines += [f"| {s['time']} | {s['query']} | {s['results']} | {s['new']} |" for s in d["searches"]]
222
+ lines.append("")
223
+ if d["reasons"]:
224
+ lines += ["## 排除理由", ""] + [f"- {r['reason']}({r['count']} 篇)" for r in d["reasons"]] + [""]
225
+ return "\n".join(lines)