litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1102 @@
1
+ """Paper search via OpenAlex, Semantic Scholar, and Crossref."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import math
7
+ import os
8
+ import re
9
+ import tempfile
10
+ from concurrent.futures import ThreadPoolExecutor, as_completed
11
+ from typing import Any
12
+
13
+ import requests
14
+
15
+ from ..terms import text
16
+ from ..config import load_config
17
+ from ..network import user_agent
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+ _SEARCH_TIMEOUT = 30 # seconds, longer than default because these are public APIs
22
+ _USER_AGENT = user_agent("(https://github.com/dysprosium231/litloop-paper-agent)")
23
+
24
+
25
+ class _AcademicSession(requests.Session):
26
+ """Adds the OpenAlex API key to api.openalex.org requests only.
27
+
28
+ OpenAlex requires an API key on every call (since 2026-02); the same
29
+ session also talks to other APIs, which must never receive the key.
30
+ """
31
+
32
+ def request(self, method, url, *args, **kwargs): # type: ignore[override]
33
+ if str(url).startswith("https://api.openalex.org/"):
34
+ try:
35
+ key = str(load_config().get("openalex_api_key") or "").strip()
36
+ except Exception:
37
+ key = ""
38
+ if key:
39
+ params = dict(kwargs.get("params") or {})
40
+ params.setdefault("api_key", key)
41
+ kwargs["params"] = params
42
+ resp = super().request(method, url, *args, **kwargs)
43
+ from .. import quota # count keyword searches too (10 credits each), not only deep search
44
+ quota.record("openalex", resp)
45
+ return resp
46
+ return super().request(method, url, *args, **kwargs)
47
+
48
+
49
+ def _plain_session() -> requests.Session:
50
+ """A requests session without proxy, for public academic APIs."""
51
+ s = _AcademicSession()
52
+ s.headers.update({"User-Agent": _USER_AGENT, "mailto": "litloop@example.invalid"})
53
+ return s
54
+
55
+
56
+ def _note(errors: list[str] | None, message: str) -> None:
57
+ """Remember why a source returned nothing, so the caller can say it."""
58
+ logger.warning("search: %s", message)
59
+ if errors is not None and message not in errors:
60
+ errors.append(message)
61
+
62
+
63
+ def _body_reason(resp: Any) -> str:
64
+ """The short reason an API gave, e.g. "Rate limit exceeded"."""
65
+ try:
66
+ data = resp.json()
67
+ except Exception:
68
+ return str(resp.reason or "")[:80]
69
+ if isinstance(data, dict):
70
+ return str(data.get("error") or data.get("message") or "")[:120]
71
+ return ""
72
+
73
+
74
+ def _reconstruct_abstract(inverted_index: dict | None) -> str:
75
+ if not isinstance(inverted_index, dict):
76
+ return ""
77
+ word_positions = []
78
+ for word, positions in inverted_index.items():
79
+ if isinstance(positions, list):
80
+ for pos in positions:
81
+ if isinstance(pos, int):
82
+ word_positions.append((pos, word))
83
+ word_positions.sort()
84
+ # No cut here: clean_abstract caps very long ones, keeping the conclusion.
85
+ return " ".join(w for _, w in word_positions)
86
+
87
+
88
+ def _search_openalex(
89
+ query: str, limit: int = 10,
90
+ year_from: int | None = None, year_to: int | None = None,
91
+ sort: str | None = None,
92
+ errors: list[str] | None = None,
93
+ ) -> list[dict[str, Any]]:
94
+ try:
95
+ session = _plain_session()
96
+ params: dict[str, Any] = {"per_page": limit}
97
+ # OpenAlex rejects keyless requests since 2026-02.
98
+ key = str(load_config().get("openalex_api_key") or "").strip()
99
+ if key:
100
+ params["api_key"] = key
101
+ filters = []
102
+ if query:
103
+ # Title + abstract only: `search=` also matches full text and, with
104
+ # no API key, answers 429 ("Anonymous search is temporarily
105
+ # rate-limited") often enough that a fresh install searches nothing.
106
+ filters.append("title_and_abstract.search:" + query.replace(",", " "))
107
+ if year_from or year_to:
108
+ y_from = year_from or 1900
109
+ y_to = year_to or 2026
110
+ filters.append(f"publication_year:{y_from}-{y_to}")
111
+ if filters:
112
+ params["filter"] = ",".join(filters)
113
+ if sort:
114
+ sort_key = sort if ":" in sort else f"{sort}:desc"
115
+ params["sort"] = sort_key
116
+ resp = session.get(
117
+ "https://api.openalex.org/works",
118
+ params=params,
119
+ timeout=_SEARCH_TIMEOUT,
120
+ )
121
+ if resp.status_code != 200:
122
+ _note(errors, f"OpenAlex {resp.status_code}: {_body_reason(resp)}")
123
+ return []
124
+ data = resp.json()
125
+ except Exception as exc:
126
+ _note(errors, f"OpenAlex 连不上:{type(exc).__name__}")
127
+ return []
128
+ return _parse_openalex_works(data.get("results", []))
129
+
130
+
131
+ def _parse_openalex_works(works: list[dict]) -> list[dict[str, Any]]:
132
+ """Parse OpenAlex works into unified result format."""
133
+ results = []
134
+ for work in works:
135
+ doi_raw = work.get("doi", "") or ""
136
+ doi = doi_raw.replace("https://doi.org/", "") if doi_raw else ""
137
+ if not doi:
138
+ continue
139
+ authors = [
140
+ a.get("author", {}).get("display_name", "")
141
+ for a in (work.get("authorships") or [])[:5]
142
+ ]
143
+ oa = work.get("open_access") or {}
144
+ best_oa = work.get("best_oa_location") or {}
145
+ is_oa = oa.get("is_oa", False)
146
+ oa_url = best_oa.get("pdf_url") or best_oa.get("landing_page_url") or oa.get("oa_url") or ""
147
+ results.append({
148
+ "title": work.get("title", ""),
149
+ "doi": doi,
150
+ "url": work.get("id", ""),
151
+ "authors": authors,
152
+ "year": work.get("publication_year", ""),
153
+ "cited_by_count": work.get("cited_by_count", 0),
154
+ "abstract": _reconstruct_abstract(work.get("abstract_inverted_index")),
155
+ "is_oa": is_oa,
156
+ "oa_url": oa_url,
157
+ "source": "openalex",
158
+ "journal": ((work.get("primary_location") or {}).get("source") or {}).get("display_name") or "",
159
+ "issn": list(((work.get("primary_location") or {}).get("source") or {}).get("issn") or []),
160
+ })
161
+ return results
162
+
163
+
164
+ def _lookup_author_id(name: str) -> tuple[str | None, str, int, int]:
165
+ """Resolve an author name to an OpenAlex author ID.
166
+
167
+ Returns (author_id, display_name, works_count, cited_by_count).
168
+ author_id is None if no match found.
169
+
170
+ Matching strategy:
171
+ 1. Exact case-insensitive name match
172
+ 2. If none, pick by highest works_count
173
+ """
174
+ try:
175
+ session = _plain_session()
176
+ resp = session.get(
177
+ "https://api.openalex.org/authors",
178
+ params={"search": name, "per_page": 10},
179
+ timeout=_SEARCH_TIMEOUT,
180
+ )
181
+ if resp.status_code != 200:
182
+ return None, "", 0, 0
183
+ data = resp.json()
184
+ except Exception:
185
+ return None, "", 0, 0
186
+
187
+ candidates = data.get("results", [])
188
+ if not candidates:
189
+ return None, "", 0, 0
190
+
191
+ name_lower = name.lower().strip()
192
+ # Generate name variants (handle Chinese/Western name order)
193
+ name_parts = name_lower.split()
194
+ name_variants = {name_lower}
195
+ if len(name_parts) == 2:
196
+ # "Fang Jingyun" ↔ "Jingyun Fang"
197
+ name_variants.add(f"{name_parts[1]} {name_parts[0]}")
198
+ elif len(name_parts) >= 3:
199
+ # Try last-name-first ordering for 3+ word names
200
+ name_variants.add(f"{name_parts[-1]} {' '.join(name_parts[:-1])}")
201
+
202
+ # First: try exact case-insensitive name match (including variants)
203
+ exact_matches = [
204
+ a for a in candidates
205
+ if (a.get("display_name") or "").lower().strip() in name_variants
206
+ ]
207
+ if exact_matches:
208
+ best = max(
209
+ exact_matches,
210
+ key=lambda a: (a.get("cited_by_count", 0), a.get("works_count", 0)),
211
+ )
212
+ else:
213
+ # Fallback: highest cited_by_count (prefer authoritative consolidated profiles)
214
+ best = max(
215
+ candidates,
216
+ key=lambda a: (a.get("cited_by_count", 0), a.get("works_count", 0)),
217
+ )
218
+
219
+ author_id = best.get("id", "").split("/")[-1] if best.get("id") else ""
220
+ display = best.get("display_name", name)
221
+ works = best.get("works_count", 0)
222
+ cited = best.get("cited_by_count", 0)
223
+ return (author_id if author_id else None, display, works, cited)
224
+
225
+
226
+ def _search_openalex_by_author(
227
+ author_id: str,
228
+ limit: int = 10,
229
+ year_from: int | None = None,
230
+ year_to: int | None = None,
231
+ sort: str | None = None,
232
+ ) -> list[dict[str, Any]]:
233
+ """Search OpenAlex works by author ID."""
234
+ try:
235
+ session = _plain_session()
236
+ filters = [f"authorships.author.id:{author_id}"]
237
+ if year_from or year_to:
238
+ y_from = year_from or 1900
239
+ y_to = year_to or 2026
240
+ filters.append(f"publication_year:{y_from}-{y_to}")
241
+
242
+ params: dict[str, Any] = {
243
+ "filter": ",".join(filters),
244
+ "per_page": limit,
245
+ }
246
+ if sort:
247
+ sort_key = sort if ":" in sort else f"{sort}:desc"
248
+ params["sort"] = sort_key
249
+
250
+ resp = session.get(
251
+ "https://api.openalex.org/works",
252
+ params=params,
253
+ timeout=_SEARCH_TIMEOUT,
254
+ )
255
+ if resp.status_code != 200:
256
+ return []
257
+ data = resp.json()
258
+ except Exception:
259
+ return []
260
+ return _parse_openalex_works(data.get("results", []))
261
+
262
+
263
+ def _search_cache_payload(
264
+ cfg: dict[str, Any],
265
+ *,
266
+ query: str,
267
+ limit: int,
268
+ year_from: int | None,
269
+ year_to: int | None,
270
+ sort: str | None,
271
+ author: str | None,
272
+ author_id: str | None,
273
+ max_cas_zone: int | None,
274
+ max_cas_sub_zone: int | None,
275
+ max_jcr_quartile: int | None,
276
+ exclude_warning: bool,
277
+ exclude_retracted: bool,
278
+ ) -> dict[str, Any]:
279
+ from ..terms import glossary
280
+ from . import scansci_find
281
+
282
+ try:
283
+ gp = glossary.path()
284
+ st = gp.stat()
285
+ glossary_sig: tuple[int, int] | None = (st.st_mtime_ns, st.st_size)
286
+ except OSError:
287
+ glossary_sig = None
288
+
289
+ try:
290
+ scansci_avail = bool(scansci_find.find_cli_available())
291
+ except Exception:
292
+ scansci_avail = False
293
+
294
+ email = str(cfg.get("email") or "").strip().lower()
295
+ return {
296
+ "query": " ".join(str(query or "").strip().lower().split()),
297
+ "limit": limit,
298
+ "year_from": year_from,
299
+ "year_to": year_to,
300
+ "sort": sort,
301
+ "author": " ".join(str(author or "").strip().lower().split()) if author else None,
302
+ "author_id": str(author_id or "").strip() or None,
303
+ "max_cas_zone": max_cas_zone,
304
+ "max_cas_sub_zone": max_cas_sub_zone,
305
+ "max_jcr_quartile": max_jcr_quartile,
306
+ "exclude_warning": bool(exclude_warning),
307
+ "exclude_retracted": bool(exclude_retracted),
308
+ "has_openalex_key": bool(str(cfg.get("openalex_api_key") or "").strip()),
309
+ "has_elsevier_key": bool(str(cfg.get("elsevier_api_key") or "").strip()),
310
+ "has_springer_key": bool(str(cfg.get("springer_meta_api_key") or "").strip()),
311
+ "has_s2_key": bool(str(cfg.get("semantic_scholar_api_key") or "").strip()),
312
+ "has_email": bool(email and email != "your-email@example.com"),
313
+ "scansci_available": scansci_avail,
314
+ "glossary": glossary_sig,
315
+ }
316
+
317
+
318
+ def search_papers_v2(
319
+ query: str = "",
320
+ limit: int = 10,
321
+ year_from: int | None = None,
322
+ year_to: int | None = None,
323
+ sort: str | None = None,
324
+ *,
325
+ author: str | None = None,
326
+ author_id: str | None = None,
327
+ max_cas_zone: int | None = None,
328
+ max_cas_sub_zone: int | None = None,
329
+ max_jcr_quartile: int | None = None,
330
+ exclude_warning: bool = False,
331
+ exclude_retracted: bool = False,
332
+ stats: dict[str, Any] | None = None,
333
+ expansions: list[dict[str, Any]] | None = None,
334
+ use_cache: bool = True,
335
+ ) -> list[dict[str, Any]]:
336
+ """Keyword search via ScanSci Find's discovery engine when available.
337
+
338
+ Other wordings of the query from the glossary (spelling, the user's terms,
339
+ element symbols) are searched too; ``expansions`` receives what was expanded.
340
+
341
+ Falls back transparently to the built-in three-source ``search_papers``
342
+ when the scansci-find CLI is missing or the discovery call fails. Author
343
+ search always stays on the built-in OpenAlex path.
344
+
345
+ Every result is labelled with its journal's partitions (when
346
+ litloop-journal-rank is installed) and a retraction flag. The journal
347
+ filters (max_cas_zone=2 keeps CAS major-category zones 1-2, max_cas_sub_zone=1
348
+ keeps journals in zone 1 of any of their subcategories, max_jcr_quartile=1 keeps Q1,
349
+ exclude_warning drops warning-listed journals) drop records whose journal
350
+ is unranked as well; ``stats`` receives how many were dropped and why.
351
+ """
352
+ from .. import cache as _cache
353
+
354
+ sort = _normalize_sort(sort)
355
+ cfg = load_config()
356
+ ttl_hours = float(cfg.get("search_cache_ttl_hours", 168))
357
+ can_cache = bool(use_cache and sort not in ("date", "publication_date") and ttl_hours > 0)
358
+ payload: dict[str, Any] | None = None
359
+ if can_cache:
360
+ payload = _search_cache_payload(
361
+ cfg,
362
+ query=query,
363
+ limit=limit,
364
+ year_from=year_from,
365
+ year_to=year_to,
366
+ sort=sort,
367
+ author=author,
368
+ author_id=author_id,
369
+ max_cas_zone=max_cas_zone,
370
+ max_cas_sub_zone=max_cas_sub_zone,
371
+ max_jcr_quartile=max_jcr_quartile,
372
+ exclude_warning=exclude_warning,
373
+ exclude_retracted=exclude_retracted,
374
+ )
375
+ cached = _cache.search_cache_get(payload, cfg)
376
+ if cached and isinstance(cached.get("results"), list):
377
+ if expansions is not None and isinstance(cached.get("expansions"), list):
378
+ expansions.extend(cached["expansions"])
379
+ if stats is not None:
380
+ if isinstance(cached.get("stats"), dict):
381
+ stats.update(cached["stats"])
382
+ stats["cached"] = True
383
+ return cached["results"]
384
+
385
+ filtering = bool(max_cas_zone or max_cas_sub_zone or max_jcr_quartile
386
+ or exclude_warning or exclude_retracted)
387
+ pool = limit * 3 if filtering else limit
388
+ errors: list[str] = []
389
+ local_expansions: list[dict[str, Any]] | None = [] if (expansions is not None or can_cache) else None
390
+ local_stats: dict[str, Any] | None = {} if (stats is not None or can_cache) else None
391
+ results = _collect(query, pool, limit, year_from, year_to, sort, author, author_id,
392
+ local_expansions, errors)
393
+ if local_stats is not None and errors:
394
+ local_stats["errors"] = errors
395
+ finalized = _finalize(results, limit, max_cas_zone=max_cas_zone, max_cas_sub_zone=max_cas_sub_zone,
396
+ max_jcr_quartile=max_jcr_quartile,
397
+ exclude_warning=exclude_warning, exclude_retracted=exclude_retracted, stats=local_stats)
398
+ if expansions is not None and local_expansions:
399
+ expansions.extend(local_expansions)
400
+ if stats is not None and local_stats:
401
+ stats.update(local_stats)
402
+ if can_cache and payload is not None and not errors and finalized:
403
+ try:
404
+ _cache.search_cache_set(
405
+ payload,
406
+ {"results": finalized, "expansions": local_expansions or [], "stats": local_stats or {}},
407
+ cfg,
408
+ )
409
+ except OSError as exc:
410
+ logger.warning("quick search: failed to write cache (%s)", exc)
411
+ return finalized
412
+
413
+
414
+ def _glossary_variants(query: str) -> tuple[list[str], list[dict[str, Any]]]:
415
+ """Other wordings of ``query`` from the glossary (``glossary.query_variants``:
416
+ the other spelling, the user's terms, element symbols and names)."""
417
+ from ..terms import glossary
418
+ try:
419
+ return glossary.query_variants(query, glossary.in_use())
420
+ except Exception as exc: # an unreadable glossary must not stop the search
421
+ logger.warning("search: glossary not read (%s)", exc)
422
+ return [], []
423
+
424
+
425
+ def _collect(query, pool, limit, year_from, year_to, sort, author, author_id,
426
+ expansions: list[dict[str, Any]] | None = None,
427
+ errors: list[str] | None = None) -> list[dict[str, Any]]:
428
+ if author_id or author:
429
+ return search_papers(
430
+ query, limit=pool, year_from=year_from, year_to=year_to, sort=sort,
431
+ author=author, author_id=author_id, errors=errors,
432
+ )
433
+ if not query:
434
+ return []
435
+ variants, used = _glossary_variants(query)
436
+ if expansions is not None:
437
+ expansions += used
438
+ if not variants:
439
+ return _tidy(_collect_one(query, pool, year_from, year_to, sort, errors), query, sort)
440
+ # Each wording is searched on its own; results are taken from each in turn,
441
+ # so papers that only write the other wording also reach the first page.
442
+ with ThreadPoolExecutor(max_workers=len(variants) + 1) as ex:
443
+ runs = list(ex.map(
444
+ lambda q: _collect_one(q, pool, year_from, year_to, sort, errors),
445
+ [query, *variants],
446
+ ))
447
+ logger.info("search: glossary wordings %s returned %s results", variants, [len(r) for r in runs])
448
+ return _tidy(_take_in_turns(runs), query, sort, variants)
449
+
450
+
451
+ def _take_in_turns(runs: list[list[dict[str, Any]]]) -> list[dict[str, Any]]:
452
+ """The first result of each run, then the second of each, ...; one record per DOI or title."""
453
+ out: list[dict[str, Any]] = []
454
+ seen: set[str] = set()
455
+ for i in range(max(map(len, runs), default=0)):
456
+ for run in runs:
457
+ if i < len(run):
458
+ r = run[i]
459
+ key = text.paper_key(r)
460
+ if key not in seen:
461
+ seen.add(key)
462
+ out.append(r)
463
+ return out
464
+
465
+
466
+ def _collect_one(query, pool, year_from, year_to, sort,
467
+ errors: list[str] | None = None) -> list[dict[str, Any]]:
468
+ """One wording of a keyword search, not yet tidied: the discovery engine, or the built-in search."""
469
+ try:
470
+ from .scansci_find import find_cli_available, search as discovery_search, to_legacy_results
471
+ if not find_cli_available():
472
+ return search_papers(query, limit=pool, year_from=year_from, year_to=year_to,
473
+ sort=sort, errors=errors)
474
+ with tempfile.TemporaryDirectory(prefix="scansci_find_") as tmp:
475
+ payload = discovery_search(
476
+ query, tmp, depth="quick", limit=max(pool * 2, 30),
477
+ year_from=year_from, year_to=year_to, sources=_quick_sources(),
478
+ )
479
+ results = to_legacy_results(payload.get("candidates") or [], limit=pool * 3)
480
+ logger.info("search: discovery engine returned %d results", len(results))
481
+ except Exception as exc: # noqa: BLE001 - degrade gracefully to 3-source search
482
+ logger.warning("search: discovery engine failed (%s), falling back to the built-in search", exc)
483
+ return search_papers(query, limit=pool, year_from=year_from, year_to=year_to, sort=sort)
484
+
485
+ if sort == "cited_by_count":
486
+ # The relevance pool holds only the top ~30 matches; the field's most
487
+ # cited papers are often outside it, so ask OpenAlex for those too.
488
+ seen = {str(r.get("doi") or "").lower() for r in results if r.get("doi")}
489
+ for extra in _search_openalex(query, max(pool * 2, 25), year_from, year_to, "cited_by_count"):
490
+ if extra["doi"].lower() not in seen:
491
+ seen.add(extra["doi"].lower())
492
+ extra["_matched"] = True # OpenAlex matched it in title/abstract
493
+ results.append(extra)
494
+ return results
495
+
496
+
497
+ def _tidy(results: list[dict[str, Any]], query: str, sort: str | None,
498
+ variants: list[str] | None = None) -> list[dict[str, Any]]:
499
+ """Drop junk titles, rank by how well each matches the query (or one of its
500
+ glossary ``variants``), merge duplicate titles. Both the discovery engine and
501
+ the built-in fallback go through it: the fallback used to skip it (code
502
+ review, 2026-10-18), so users without the engine got unranked,
503
+ undeduplicated results."""
504
+ results = _dedupe_titles(_match_tier(_drop_junk_titles(results), query, variants))
505
+ # Sorting by citations or date happens within a match tier, so a famous
506
+ # but weakly matching paper does not jump ahead of the on-topic ones.
507
+ if sort == "cited_by_count":
508
+ results.sort(key=lambda x: (x.get("_tier", 0), -(x.get("cited_by_count") or 0)))
509
+ elif sort == "publication_date":
510
+ results.sort(key=lambda x: (x.get("_tier", 0), -(x.get("year") or 0)))
511
+ for r in results:
512
+ r.pop("_tier", None)
513
+ return results
514
+
515
+
516
+ def _finalize(results: list[dict[str, Any]], limit: int, *, max_cas_zone=None, max_cas_sub_zone=None,
517
+ max_jcr_quartile=None,
518
+ exclude_warning=False, exclude_retracted=False,
519
+ stats: dict[str, Any] | None = None) -> list[dict[str, Any]]:
520
+ journal_filter = bool(max_cas_zone or max_cas_sub_zone or max_jcr_quartile or exclude_warning)
521
+ # Label only what can be shown, unless a filter needs the whole pool.
522
+ candidates = results if (journal_filter or exclude_retracted) else results[:limit]
523
+ _add_journal_rank(candidates)
524
+ if journal_filter:
525
+ candidates = _filter_by_journal(candidates, max_cas_zone, max_jcr_quartile,
526
+ exclude_warning, stats, max_cas_sub_zone)
527
+ _flag_retractions(candidates[: limit * 2] if exclude_retracted else candidates[:limit])
528
+ if exclude_retracted:
529
+ kept = [r for r in candidates if not r.get("retracted")]
530
+ if stats is not None:
531
+ stats["dropped_retracted"] = len(candidates) - len(kept)
532
+ candidates = kept
533
+ # A few spares: merging a preprint into its published version, which is
534
+ # only recognisable once both abstracts are in, must not shorten the list.
535
+ shown = candidates[: limit + 3]
536
+ # What the screener reads: clean abstracts, missing ones fetched
537
+ # (Semantic Scholar -> Scopus -> Crossref), short author lists.
538
+ from .abstracts import fill_abstracts, trim_authors
539
+ trim_authors(shown)
540
+ fill_abstracts(shown, load_config())
541
+ shown = _merge_preprints(shown, stats)
542
+ _note_unverified_venues(shown)
543
+ return shown[:limit]
544
+
545
+
546
+ _PREPRINT_DOI = re.compile(
547
+ r"^10\.(2139/ssrn|21203/rs\.|1101/|20944/preprints|48550/arxiv|31219/osf|26434/chemrxiv"
548
+ r"|22541/au|36227/techrxiv|31224/)", re.I)
549
+
550
+
551
+ def _is_preprint(r: dict[str, Any]) -> bool:
552
+ doi = str(r.get("doi") or "")
553
+ return bool(_PREPRINT_DOI.match(doi)) or (not doi and bool(r.get("arxiv_id")))
554
+
555
+
556
+ def _surname(r: dict[str, Any]) -> str:
557
+ authors = r.get("authors") or []
558
+ first = authors[0] if authors else ""
559
+ if isinstance(first, dict):
560
+ first = first.get("family") or first.get("name") or ""
561
+ parts = re.findall(r"[a-z]+", str(first).lower())
562
+ return parts[-1] if parts else ""
563
+
564
+
565
+ def _same_work(a: dict[str, Any], b: dict[str, Any]) -> bool:
566
+ """Same first author and near-identical abstracts (titles often change on publication)."""
567
+ if not _surname(a) or _surname(a) != _surname(b):
568
+ return False
569
+ wa = set(re.findall(r"[a-z0-9]{4,}", str(a.get("abstract") or "").lower()))
570
+ wb = set(re.findall(r"[a-z0-9]{4,}", str(b.get("abstract") or "").lower()))
571
+ if len(wa) < 20 or len(wb) < 20:
572
+ return False
573
+ return len(wa & wb) / len(wa | wb) >= 0.6
574
+
575
+
576
+ def _merge_preprints(results: list[dict[str, Any]], stats: dict[str, Any] | None = None) -> list[dict[str, Any]]:
577
+ """Fold a preprint into its published version: keep the journal record,
578
+ remember the preprint DOI (useful as an open copy), drop the duplicate."""
579
+ published = [r for r in results if not _is_preprint(r)]
580
+ out, merged = [], 0
581
+ for r in results:
582
+ if _is_preprint(r):
583
+ twin = next((p for p in published if _same_work(r, p)), None)
584
+ if twin is not None:
585
+ twin.setdefault("preprint_doi", r.get("doi") or r.get("arxiv_id"))
586
+ if not twin.get("oa_url") and r.get("oa_url"):
587
+ twin["preprint_url"] = r["oa_url"]
588
+ merged += 1
589
+ continue
590
+ out.append(r)
591
+ if merged and stats is not None:
592
+ stats["merged_preprints"] = merged
593
+ return out
594
+
595
+
596
+ def _note_unverified_venues(results: list[dict[str, Any]]) -> None:
597
+ """Without a DOI the venue comes from the index alone, and repositories
598
+ are sometimes filed under an unrelated journal; say so."""
599
+ for r in results:
600
+ if r.get("journal") and not r.get("doi"):
601
+ r["venue_unverified"] = True
602
+
603
+
604
+ _ABBREV_ONLY = re.compile(r"(?:[A-Z][A-Za-z]{0,9}\.\s*){2,}")
605
+
606
+
607
+ def _drop_junk_titles(results: list[dict[str, Any]]) -> list[dict[str, Any]]:
608
+ """Drop index records whose "title" is a journal abbreviation
609
+ ("Appl. Organomet. Chem.") or just the journal's name."""
610
+ kept = []
611
+ for r in results:
612
+ title = str(r.get("title") or "").strip()
613
+ journal = str(r.get("journal") or "").strip().lower()
614
+ if _ABBREV_ONLY.fullmatch(title) or (journal and title.lower() == journal):
615
+ continue
616
+ kept.append(r)
617
+ return kept
618
+
619
+
620
+ def _dedupe_titles(results: list[dict[str, Any]]) -> list[dict[str, Any]]:
621
+ """One record per paper: a preprint and its published version, or the same
622
+ paper from two sources under different DOIs/casing, share a title. The
623
+ first (best-ranked) wins; a DOI and a journal are borrowed if it lacks them."""
624
+ seen: dict[str, dict[str, Any]] = {}
625
+ out = []
626
+ for r in results:
627
+ key = text.title_key(r.get("title"))
628
+ if len(key) < 20: # too short to be sure
629
+ out.append(r)
630
+ continue
631
+ first = seen.get(key)
632
+ if first is None:
633
+ seen[key] = r
634
+ out.append(r)
635
+ continue
636
+ for field in ("doi", "journal", "issn", "abstract"):
637
+ if not first.get(field) and r.get(field):
638
+ first[field] = r[field]
639
+ first["cited_by_count"] = max(first.get("cited_by_count") or 0, r.get("cited_by_count") or 0)
640
+ return out
641
+
642
+
643
+ def _filter_by_journal(results, max_cas_zone, max_jcr_quartile, exclude_warning, stats,
644
+ max_cas_sub_zone=None):
645
+ kept, dropped = [], {"unranked": 0, "below_threshold": 0, "warning": 0}
646
+ for r in results:
647
+ if exclude_warning and r.get("journal_warning"):
648
+ dropped["warning"] += 1
649
+ continue
650
+ if max_cas_zone or max_cas_sub_zone or max_jcr_quartile:
651
+ cas, sub, q = r.get("cas_zone"), r.get("cas_sub_zone"), r.get("jcr_quartile")
652
+ if ((max_cas_zone and cas is None) or (max_cas_sub_zone and sub is None)
653
+ or (max_jcr_quartile and q is None)):
654
+ dropped["unranked"] += 1
655
+ continue
656
+ if ((max_cas_zone and cas > max_cas_zone) or (max_cas_sub_zone and sub > max_cas_sub_zone)
657
+ or (max_jcr_quartile and q > max_jcr_quartile)):
658
+ dropped["below_threshold"] += 1
659
+ continue
660
+ kept.append(r)
661
+ if stats is not None:
662
+ stats.update({f"dropped_{k}": v for k, v in dropped.items()})
663
+ return kept
664
+
665
+
666
+ def _add_journal_rank(results: list[dict[str, Any]]) -> list[dict[str, Any]]:
667
+ """Label each result with its journal's partitions when litloop-journal-rank
668
+ is installed (CAS / JCR / XR zones, warning lists, user catalogues)."""
669
+ try:
670
+ from litloop_journal_rank import lookup
671
+ except ImportError:
672
+ return results
673
+ try:
674
+ # Monthly background check for new ShowJCR tables (CAS / JCR / XR).
675
+ from litloop_journal_rank.showjcr_sync import ensure_fresh
676
+ ensure_fresh(proxy=str(load_config().get("host_proxy_url") or "") or None)
677
+ except Exception: # noqa: BLE001 - older tool version or no network
678
+ pass
679
+ for r in results:
680
+ if not (r.get("journal") or r.get("issn")):
681
+ continue
682
+ try:
683
+ info = lookup(journal=r.get("journal") or "", issn=r.get("issn") or None)
684
+ except Exception: # noqa: BLE001 - a ranking problem must not break search
685
+ continue
686
+ if not info.get("found"):
687
+ continue
688
+ r["journal_rank"] = info["label"]
689
+ if (info.get("cas") or {}).get("zone"):
690
+ r["cas_zone"] = info["cas"]["zone"]
691
+ sub_zones = [s["zone"] for s in (info.get("cas") or {}).get("subcategories", []) if s.get("zone")]
692
+ if sub_zones:
693
+ r["cas_sub_zone"] = min(sub_zones) # best subcategory zone
694
+ quartile = (info.get("jcr") or {}).get("best_quartile") or ""
695
+ if quartile[1:].isdigit():
696
+ r["jcr_quartile"] = int(quartile[1:])
697
+ if info.get("warnings") or (info.get("xr") or {}).get("under_review"):
698
+ r["journal_warning"] = True
699
+ return results
700
+
701
+
702
+ def _flag_retractions(results: list[dict[str, Any]]) -> None:
703
+ """Mark retracted papers and expressions of concern.
704
+
705
+ Prefers the local Retraction Watch copy kept by litloop-journal-rank
706
+ (`ljr retractions update`): offline, and it also knows expressions of
707
+ concern, corrections and the reasons. Without it, one batched OpenAlex
708
+ request (is_retracted only) for up to 50 DOIs; a failure leaves results
709
+ unflagged. Corrections are recorded in `notices` but not flagged.
710
+ """
711
+ dois = [str(r["doi"]).lower() for r in results if r.get("doi")][:50]
712
+ if not dois:
713
+ return
714
+ try:
715
+ from litloop_journal_rank.retractions import ensure_fresh, lookup_papers
716
+ # Missing or a week old: refresh in the background (GitLab is reached
717
+ # through the per-host proxy when one is configured).
718
+ ensure_fresh(proxy=str(load_config().get("host_proxy_url") or "") or None)
719
+ local = lookup_papers(dois)
720
+ except Exception: # noqa: BLE001 - tool missing or data unreadable
721
+ local = None
722
+ if local is not None:
723
+ for r in results:
724
+ info = local.get(str(r.get("doi") or "").lower())
725
+ if not info:
726
+ continue
727
+ if info["retracted"]:
728
+ r["retracted"] = True
729
+ if info["concern"]:
730
+ r["concern"] = True
731
+ r["notices"] = [{k: n[k] for k in ("nature", "date", "reasons")} for n in info["notices"]]
732
+ return
733
+ params: dict[str, Any] = {"filter": "doi:" + "|".join(dois), "select": "doi,is_retracted",
734
+ "per_page": 50}
735
+ key = str(load_config().get("openalex_api_key") or "").strip()
736
+ if key:
737
+ params["api_key"] = key
738
+ try:
739
+ resp = _plain_session().get("https://api.openalex.org/works", params=params,
740
+ timeout=_SEARCH_TIMEOUT)
741
+ works = resp.json().get("results", []) if resp.status_code == 200 else []
742
+ except Exception: # noqa: BLE001
743
+ return
744
+ retracted = {str(w.get("doi") or "").lower().replace("https://doi.org/", "")
745
+ for w in works if w.get("is_retracted")}
746
+ for r in results:
747
+ if str(r.get("doi") or "").lower() in retracted:
748
+ r["retracted"] = True
749
+
750
+ def _quick_sources() -> list[str]:
751
+ """Sources for an interactive search.
752
+
753
+ OpenAlex indexes what Crossref has, ranks it better and answers in a few
754
+ seconds; Crossref took 7-26 s and mostly added off-topic records, so it is
755
+ used only when OpenAlex has no API key. Semantic Scholar joins when it has
756
+ a key (the keyless pool answers 429 almost every time).
757
+ """
758
+ sources = ["openalex"] if str(load_config().get("openalex_api_key") or "").strip() else ["crossref"]
759
+ if os.environ.get("SEMANTIC_SCHOLAR_API_KEY"):
760
+ sources.append("semantic")
761
+ return sources
762
+
763
+
764
+ _D = re.escape(text.DASHES)
765
+ _QUERY_WORD = re.compile(rf"[A-Za-z0-9%]+(?:[{_D}][A-Za-z0-9%]+)*")
766
+
767
+
768
+ def _query_terms(query: str) -> list[str]:
769
+ """Content words of a query, normalised for a substring test.
770
+
771
+ Hyphenated concepts stay whole ("polymer derived"); chemical systems and
772
+ terms with digits are split into their parts, since papers write Al-Si-Mg
773
+ as Al-Mg-Si or AlSiMg. Words are made singular and long ones cut to a
774
+ crude stem (``text.singular``, ``text.stem``), so "ceramics" matches
775
+ "ceramic" and "infiltration" matches "infiltrated"."""
776
+ terms = []
777
+ for word in _QUERY_WORD.findall(query or ""):
778
+ if any(ch.isdigit() for ch in word) or text.is_chemical_system(word):
779
+ pieces = [p for p in re.split(rf"[{_D}]", word) if p]
780
+ else:
781
+ pieces = [word]
782
+ for piece in pieces:
783
+ words = text.plain(piece.strip("%")).split()
784
+ if not words or (len(words) == 1 and (words[0] in text.STOPWORDS or len(words[0]) < 2)):
785
+ continue
786
+ words[-1] = text.singular(words[-1])
787
+ terms.append(" ".join(words) if len(words) > 1 else text.stem(words[0]))
788
+ return list(dict.fromkeys(terms))
789
+
790
+
791
+ def _match_tier(results: list[dict[str, Any]], query: str,
792
+ variants: list[str] | None = None) -> list[dict[str, Any]]:
793
+ """Order records by how well they match the query, without dropping any.
794
+
795
+ Tier 0: the record names every hyphenated concept of the query and at
796
+ least 60% of its terms. Tier 1: the terms are there but a concept is
797
+ missing ("polymers ... derived" for "polymer-derived"), or there is no
798
+ real abstract and the title alone falls short. Tier 2: an abstract to
799
+ judge from, and fewer than 60% of the terms in it. Records keep their
800
+ relevance order inside a tier, so
801
+ weaker matches only surface when the strong ones run out.
802
+
803
+ Nothing is removed: we judge from a title and an often truncated
804
+ abstract, while the index matched the full record, and a deep search
805
+ loses more from a dropped target than from a few extra records to screen.
806
+ Queries in CJK scripts and records OpenAlex matched on title/abstract
807
+ (``_matched``) count as tier 0. With glossary ``variants`` (other wordings
808
+ of the query), a record gets the best tier any wording gives it.
809
+ """
810
+ cjk = re.search(r"[぀-ヿ㐀-鿿가-힯]", query or "")
811
+ specs = []
812
+ for q in [query or "", *(variants or [])]:
813
+ terms = [] if cjk else _query_terms(q)
814
+ need = max(1, math.ceil(0.6 * len(terms))) if len(terms) >= 2 else 0
815
+ specs.append((terms, need, set() if cjk else _query_compounds(q)))
816
+ tiered = []
817
+ for pos, r in enumerate(results):
818
+ tier = 0
819
+ if not r.pop("_matched", False):
820
+ words = f"{r.get('title') or ''} {r.get('abstract') or ''}"
821
+ plain, squashed = text.plain(words), text.squash(words)
822
+ judged_on_title = len(r.get("abstract") or "") < 300
823
+ tier = min(_tier(terms, need, compounds, plain, squashed, judged_on_title, bool(r.get("abstract")))
824
+ for terms, need, compounds in specs)
825
+ tiered.append((tier, pos, r))
826
+ tiered.sort(key=lambda x: (x[0], x[1]))
827
+ for tier, _, r in tiered:
828
+ r["_tier"] = tier
829
+ return [r for _, _, r in tiered]
830
+
831
+
832
+ def _tier(terms: list[str], need: int, compounds: set[str], plain: str, squashed: str,
833
+ judged_on_title: bool, has_abstract: bool) -> int:
834
+ """One record's tier against one wording of the query (see ``_match_tier``)."""
835
+ if need and sum(1 for t in terms if t in plain) < need:
836
+ # A title alone rarely carries every term: too little to call it off-topic.
837
+ return 1 if judged_on_title else 2
838
+ if compounds and has_abstract and not all(c in squashed for c in compounds):
839
+ return 1
840
+ return 0
841
+
842
+
843
+ def _query_compounds(query: str) -> set[str]:
844
+ """Hyphenated query terms that name one concept ("polymer-derived").
845
+
846
+ Chemical systems (Al-Si-Mg, Si-B-C-N) are left out because papers list
847
+ the elements in any order or run them together, and so are terms with
848
+ digits (3D-SiC, Al-12%Si), which are written in too many ways."""
849
+ out = set()
850
+ for c in re.findall(rf"[A-Za-z0-9]+(?:[{_D}][A-Za-z0-9]+)+", query or ""):
851
+ if any(ch.isdigit() for ch in c) or text.is_chemical_system(c):
852
+ continue
853
+ squashed = text.squash(c)
854
+ if len(squashed) >= 4:
855
+ out.add(squashed)
856
+ return out
857
+
858
+
859
+ _SORT_ALIASES = {
860
+ "cited": "cited_by_count", "citations": "cited_by_count", "citation": "cited_by_count",
861
+ "cited_by_count": "cited_by_count", "citation_count": "cited_by_count",
862
+ "date": "publication_date", "year": "publication_date", "newest": "publication_date",
863
+ "recent": "publication_date", "publication_date": "publication_date",
864
+ }
865
+
866
+
867
+ def _normalize_sort(sort: str | None) -> str | None:
868
+ """Accept the spellings agents and users actually type; None = relevance."""
869
+ if not sort:
870
+ return None
871
+ return _SORT_ALIASES.get(str(sort).strip().lower(), None)
872
+
873
+
874
+ def search_papers(
875
+ query: str = "",
876
+ limit: int = 10,
877
+ year_from: int | None = None,
878
+ year_to: int | None = None,
879
+ sort: str | None = None,
880
+ *,
881
+ author: str | None = None,
882
+ author_id: str | None = None,
883
+ errors: list[str] | None = None,
884
+ ) -> list[dict[str, Any]]:
885
+ """Search papers from OpenAlex + Semantic Scholar + Crossref in parallel.
886
+
887
+ When `author` or `author_id` is provided, searches by author instead of keyword.
888
+ - `author`: resolves author name → OpenAlex author ID → works
889
+ - `author_id`: directly searches works by OpenAlex author ID
890
+ """
891
+ # --- Author-based search (fast path) ---
892
+ if author_id or author:
893
+ matched_name = None
894
+ matched_works = 0
895
+ matched_cited = 0
896
+ if author and not author_id:
897
+ resolved, matched_name, matched_works, matched_cited = _lookup_author_id(author)
898
+ if not resolved:
899
+ return []
900
+ author_id = resolved
901
+
902
+ if author_id:
903
+ results = _search_openalex_by_author(
904
+ author_id, limit=limit,
905
+ year_from=year_from, year_to=year_to, sort=sort,
906
+ )
907
+ # Attach author match metadata to first result
908
+ if results and (matched_name or author):
909
+ results[0]["_author_match"] = {
910
+ "name": matched_name or author or "",
911
+ "id": author_id,
912
+ "works_count": matched_works,
913
+ "cited_by_count": matched_cited,
914
+ }
915
+ return results
916
+
917
+ # --- Keyword-based search (existing parallel path) ---
918
+ if not query:
919
+ return []
920
+
921
+ all_results: list[dict[str, Any]] = []
922
+ per_source = max(5, limit)
923
+
924
+ with ThreadPoolExecutor(max_workers=3) as pool:
925
+ futures = {
926
+ pool.submit(_search_openalex, query, per_source, year_from, year_to, sort, errors): "openalex",
927
+ pool.submit(_search_semantic_scholar, query, per_source, year_from, year_to, errors): "semantic_scholar",
928
+ pool.submit(_search_crossref, query, per_source, year_from, year_to, errors): "crossref",
929
+ }
930
+ for future in as_completed(futures, timeout=30):
931
+ try:
932
+ all_results.extend(future.result())
933
+ except Exception as exc:
934
+ _note(errors, f"{futures[future]} 出错:{type(exc).__name__}")
935
+
936
+ # Deduplicate by DOI (prefer entry with more info)
937
+ seen: dict[str, dict[str, Any]] = {}
938
+ for r in all_results:
939
+ doi = r.get("doi", "").lower()
940
+ if not doi:
941
+ continue
942
+ if doi not in seen:
943
+ seen[doi] = r
944
+ else:
945
+ existing = seen[doi]
946
+ # Merge: keep fields from whichever entry has more data
947
+ if not existing.get("abstract") and r.get("abstract"):
948
+ existing["abstract"] = r["abstract"]
949
+ if not existing.get("is_oa") and r.get("is_oa"):
950
+ existing["is_oa"] = True
951
+ existing["oa_url"] = r.get("oa_url", "")
952
+ if r.get("cited_by_count", 0) > existing.get("cited_by_count", 0):
953
+ existing["cited_by_count"] = r["cited_by_count"]
954
+ existing["source"] = existing.get("source", "") + "+" + r.get("source", "")
955
+
956
+ # Sort by relevance or citations
957
+ merged = list(seen.values())
958
+ if sort == "cited_by_count":
959
+ merged.sort(key=lambda x: x.get("cited_by_count", 0), reverse=True)
960
+ elif sort == "publication_date":
961
+ merged.sort(key=lambda x: x.get("year", 0), reverse=True)
962
+
963
+ return merged[:limit]
964
+
965
+
966
+ def _search_semantic_scholar(
967
+ query: str, limit: int = 10,
968
+ year_from: int | None = None, year_to: int | None = None,
969
+ errors: list[str] | None = None,
970
+ ) -> list[dict[str, Any]]:
971
+ import time
972
+ try:
973
+ session = _plain_session()
974
+ params: dict[str, Any] = {
975
+ "query": query,
976
+ "limit": limit,
977
+ "fields": "title,externalIds,authors,year,citationCount,abstract,isOpenAccess,openAccessPdf",
978
+ }
979
+ year_filter = []
980
+ if year_from:
981
+ year_filter.append(str(year_from))
982
+ if year_to:
983
+ year_filter.append(str(year_to))
984
+ if year_filter:
985
+ params["year"] = "-".join(year_filter) if len(year_filter) == 2 else year_filter[0]
986
+
987
+ for attempt in range(3):
988
+ resp = session.get(
989
+ "https://api.semanticscholar.org/graph/v1/paper/search",
990
+ params=params,
991
+ timeout=_SEARCH_TIMEOUT,
992
+ )
993
+ if resp.status_code == 200:
994
+ break
995
+ if resp.status_code == 429:
996
+ time.sleep(2 * (attempt + 1))
997
+ continue
998
+ _note(errors, f"Semantic Scholar {resp.status_code}")
999
+ return []
1000
+ else:
1001
+ _note(errors, "Semantic Scholar 限流(429)")
1002
+ return []
1003
+ data = resp.json()
1004
+ except Exception as exc:
1005
+ _note(errors, f"Semantic Scholar 连不上:{type(exc).__name__}")
1006
+ return []
1007
+
1008
+ results = []
1009
+ for paper in data.get("data", []):
1010
+ ext_ids = paper.get("externalIds") or {}
1011
+ doi = (ext_ids.get("DOI") or "").strip()
1012
+ if not doi:
1013
+ continue
1014
+ authors = [a.get("name", "") for a in (paper.get("authors") or [])[:5]]
1015
+ oa_info = paper.get("openAccessPdf") or {}
1016
+ results.append({
1017
+ "title": paper.get("title", ""),
1018
+ "doi": doi,
1019
+ "url": f"https://api.semanticscholar.org/DOI:{doi}",
1020
+ "authors": authors,
1021
+ "year": paper.get("year", ""),
1022
+ "cited_by_count": paper.get("citationCount", 0),
1023
+ "abstract": (paper.get("abstract") or "")[:500],
1024
+ "is_oa": paper.get("isOpenAccess", False),
1025
+ "oa_url": oa_info.get("url", ""),
1026
+ })
1027
+ return results
1028
+
1029
+
1030
+ def _search_crossref(
1031
+ query: str, limit: int = 10,
1032
+ year_from: int | None = None, year_to: int | None = None,
1033
+ errors: list[str] | None = None,
1034
+ ) -> list[dict[str, Any]]:
1035
+ try:
1036
+ session = _plain_session()
1037
+ params: dict[str, Any] = {
1038
+ "query": query,
1039
+ "rows": limit,
1040
+ "select": "DOI,title,author,published-print,is-referenced-by-count,abstract,link,container-title",
1041
+ }
1042
+ filters = []
1043
+ if year_from:
1044
+ filters.append(f"from-pub-date:{year_from}")
1045
+ if year_to:
1046
+ filters.append(f"until-pub-date:{year_to}")
1047
+ if filters:
1048
+ params["filter"] = ",".join(filters)
1049
+
1050
+ resp = session.get(
1051
+ "https://api.crossref.org/works",
1052
+ params=params,
1053
+ timeout=_SEARCH_TIMEOUT,
1054
+ )
1055
+ if resp.status_code != 200:
1056
+ _note(errors, f"Crossref {resp.status_code}")
1057
+ return []
1058
+ data = resp.json()
1059
+ except Exception as exc:
1060
+ _note(errors, f"Crossref 连不上:{type(exc).__name__}")
1061
+ return []
1062
+
1063
+ results = []
1064
+ for item in data.get("message", {}).get("items", []):
1065
+ doi = item.get("DOI", "")
1066
+ if not doi:
1067
+ continue
1068
+ titles = item.get("title", [])
1069
+ title = titles[0] if titles else ""
1070
+ authors = []
1071
+ for a in (item.get("author") or [])[:5]:
1072
+ name = " ".join(filter(None, [a.get("given"), a.get("family")]))
1073
+ if name:
1074
+ authors.append(name)
1075
+ # Year from published-print
1076
+ pub_date = item.get("published-print", {}).get("date-parts", [[]])
1077
+ year = pub_date[0][0] if pub_date and pub_date[0] else ""
1078
+ # OA links
1079
+ links = item.get("link", [])
1080
+ oa_url = ""
1081
+ for link in links:
1082
+ if link.get("content-type") == "application/pdf":
1083
+ oa_url = link.get("URL", "")
1084
+ break
1085
+ # Abstract (may contain HTML tags)
1086
+ abstract = (item.get("abstract") or "")[:500]
1087
+ if abstract:
1088
+ import re
1089
+ abstract = re.sub(r"<[^>]+>", "", abstract)
1090
+ results.append({
1091
+ "title": title,
1092
+ "doi": doi,
1093
+ "url": f"https://doi.org/{doi}",
1094
+ "authors": authors,
1095
+ "year": year,
1096
+ "cited_by_count": item.get("is-referenced-by-count", 0),
1097
+ "abstract": abstract,
1098
+ "is_oa": bool(oa_url),
1099
+ "oa_url": oa_url,
1100
+ })
1101
+ return results
1102
+