litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/search/deep.py ADDED
@@ -0,0 +1,1216 @@
1
+ """Deep search (stage 2): find the literature on a topic, not just the top hits.
2
+
3
+ The agent supplies a question, 4-8 English queries, 1-3 core terms and a few
4
+ seed papers. The plugin builds a candidate pool from two complementary
5
+ routes and hands it to the agent in ranked batches for screening:
6
+
7
+ * keyword route: every query's raw candidates (100 each, nothing cut);
8
+ * citation route: works that cite any of the seeds' references and mention a
9
+ core term in title/abstract, plus the seeds' own references and citing works.
10
+
11
+ On the acceptance sets neither route alone found most targets, together they
12
+ did (TODO.md, 2026-10-15). The first batch is ranked by
13
+
14
+ score = shared references with the seeds (anchored to the user's seeds)
15
+ + 2 x queries the record fully matches
16
+
17
+ From then on a small model trained on every rating so far ranks the rest
18
+ (continuous active learning, see ``_rerank``), and missing abstracts of the
19
+ top candidates are fetched before ranking. Papers the agent rates 高 join the
20
+ seeds, so later batches follow what the user actually wants (snowballing). A batch with almost nothing relevant, or
21
+ the screening cap, ends the search with a stated reason.
22
+
23
+ Ratings live in the screening session (``litloop_screening``); this module
24
+ keeps the pool in SQLite (``pool_items`` + ``sessions.pool_state_json``, with
25
+ automatic import of legacy ``<session>.pool.json``).
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import json
31
+ import math
32
+ import random
33
+ import re
34
+ import sqlite3
35
+ import threading
36
+ import time
37
+ from collections import Counter
38
+ from concurrent.futures import ThreadPoolExecutor
39
+ from typing import Any
40
+
41
+ from . import screening
42
+ from .. import db, store
43
+ from ..desktop import translate
44
+ from ..config import load_config
45
+ from ..identifiers import canonical_doi
46
+ from ..log import get_logger
47
+
48
+ logger = get_logger()
49
+ _run = threading.local() # the round being worked on in this thread, for progress reports
50
+
51
+
52
+ def _phase(name: str, total: int | None = None) -> None:
53
+ p = getattr(_run, "progress", None)
54
+ if p:
55
+ p.phase(name, total)
56
+
57
+
58
+ def _step(done: int, total: int | None = None) -> None:
59
+ p = getattr(_run, "progress", None)
60
+ if p:
61
+ p.step(done, total)
62
+
63
+ OPENALEX = "https://api.openalex.org/works"
64
+ _FIELDS = ("id,doi,title,publication_year,cited_by_count,referenced_works,"
65
+ "abstract_inverted_index,primary_location,authorships,type")
66
+ # Not research papers: never worth a reader's time. Book chapters, theses and
67
+ # conference papers stay; the ranking model learns where they belong.
68
+ NON_PAPER = {"dataset", "paratext", "erratum", "reference-entry", "book-review", "editorial",
69
+ "peer-review", "grant", "standard", "retraction", "supplementary-materials", "libguides"}
70
+ QUERY_WEIGHT = 2
71
+ EXACT_WEIGHT = 5.0 # a phrase from the exact pass sitting in the title (see the sweep in DECISIONS)
72
+ EXACT_MIN_COVER = 0.75 # ... and most of its words are there: all of a 2-4 word phrase
73
+ ABSTRACT_CHARS = 1200 # per paper in a batch; a batch of 20 stays under ~30 KB for the agent
74
+ PREFILL = 150 # fetch missing abstracts for this many top candidates before ranking
75
+ MIN_LABELS = (3, 5) # positives (高 + seeds), negatives (中/低/不相关) before the model takes over
76
+ PSEUDO_NEGATIVES = 100
77
+ TITLE_WEIGHT = 0.3 # extra weight of each title word in the ranking model
78
+ KNEE_RATIO = 6 # stop when 高 per batch falls below 1/6 of the best batch
79
+ WORKERS = 4 # parallel OpenAlex requests (each chunk of 50 ids is independent)
80
+ # OpenAlex budget guards (LitLoop's own choices, not official limits; the daily budget
81
+ # itself is in docs/API_LIMITS.md). A reference cited by thousands of on-topic papers
82
+ # (common in biomedicine) used to cost 50 credits for its 1000 most cited citers;
83
+ # one 2026-10-16 round spent 2184 credits this way.
84
+ SINGLE_REF_CAP = 200 # one reference over the cap: keep only its 200 most cited citers (one page)
85
+ ROUND_CREDITS = 800 # stop asking for more citers once a round has spent this many credits
86
+ RESERVE_CREDITS = 1000 # and leave this much of the daily budget for everything else
87
+
88
+
89
+ # --------------------------------------------------------------------------- OpenAlex
90
+
91
+ def _boolean_heavy(params: dict[str, Any]) -> bool:
92
+ """OpenAlex allows only 5 requests/s for searches with more than five boolean operators."""
93
+ text = " ".join(str(v) for v in params.values())
94
+ return len(re.findall(r"\b(?:OR|AND|NOT)\b", text)) > 5
95
+
96
+
97
+ def _get(params: dict[str, Any], *, tries: int = 6) -> dict[str, Any]:
98
+ """One OpenAlex request, paced and counted by ``quota``. A 429 is retried with
99
+ growing waits; a spent daily budget (remaining 0) is reported plainly."""
100
+ from .. import quota
101
+ from .quick import _plain_session
102
+ key = str(load_config().get("openalex_api_key") or "").strip()
103
+ if key:
104
+ params = {**params, "api_key": key}
105
+ lane = ("openalex_boolean" if _boolean_heavy(params) else "openalex") + ("" if key else "_nokey")
106
+ last: Exception | None = None
107
+ for attempt in range(tries):
108
+ wait = min(30.0, 1.5 * 2 ** attempt)
109
+ try:
110
+ with quota.slot(lane):
111
+ r = _plain_session().get(OPENALEX, params=params, timeout=60)
112
+ quota.record("openalex", r)
113
+ remaining = r.headers.get("X-RateLimit-Remaining")
114
+ if r.status_code == 200:
115
+ return r.json()
116
+ if r.status_code == 429 and remaining == "0":
117
+ hours = round(int(r.headers.get("X-RateLimit-Reset") or 0) / 3600, 1)
118
+ raise RuntimeError(f"OpenAlex daily budget used up ({r.headers.get('X-RateLimit-Limit')} "
119
+ f"credits); it resets in about {hours} h (midnight UTC)")
120
+ if r.status_code == 429:
121
+ wait = max(wait, float(r.headers.get("Retry-After") or 0))
122
+ last = RuntimeError(f"OpenAlex HTTP {r.status_code}")
123
+ except RuntimeError:
124
+ raise
125
+ except Exception as exc: # network hiccup: retry
126
+ last = exc
127
+ time.sleep(wait)
128
+ raise RuntimeError(f"OpenAlex request failed: {last}")
129
+
130
+
131
+ def _short(wid: str) -> str:
132
+ return str(wid or "").rsplit("/", 1)[-1]
133
+
134
+
135
+ def _record(w: dict[str, Any]) -> dict[str, Any]:
136
+ from .quick import _reconstruct_abstract
137
+ src = ((w.get("primary_location") or {}).get("source") or {})
138
+ return {
139
+ "openalex": _short(w.get("id")),
140
+ "doi": canonical_doi(w.get("doi")) or None,
141
+ "title": w.get("title") or "",
142
+ "year": w.get("publication_year"),
143
+ "journal": src.get("display_name") or "",
144
+ "issn": list(src.get("issn") or []),
145
+ "authors": [((a.get("author") or {}).get("display_name") or "") for a in (w.get("authorships") or [])[:3]],
146
+ "cited_by_count": w.get("cited_by_count") or 0,
147
+ "abstract": _reconstruct_abstract(w.get("abstract_inverted_index")),
148
+ "refs": [_short(r) for r in (w.get("referenced_works") or [])],
149
+ "type": w.get("type") or "",
150
+ }
151
+
152
+
153
+ def _year_filter(year_from: int | None, year_to: int | None) -> list[str]:
154
+ out = []
155
+ if year_from:
156
+ out.append(f"from_publication_date:{year_from}-01-01")
157
+ if year_to:
158
+ out.append(f"to_publication_date:{year_to}-12-31")
159
+ return out
160
+
161
+
162
+ def _parallel(fn, args: list) -> list:
163
+ """``[fn(a) for a in args]`` with WORKERS threads, results in order and concatenated."""
164
+ if len(args) <= 1:
165
+ return [x for a in args for x in fn(a)]
166
+ with ThreadPoolExecutor(max_workers=WORKERS) as ex:
167
+ return [x for part in ex.map(fn, args) for x in part]
168
+
169
+
170
+ def _chunks(values: list[str], n: int = 50) -> list[list[str]]:
171
+ return [values[i:i + n] for i in range(0, len(values), n)]
172
+
173
+
174
+ def _by_filter(field: str, values: list[str], extra: list[str] | None = None) -> list[dict[str, Any]]:
175
+ """Fetch works whose ``field`` is any of ``values`` (50 per request)."""
176
+ def one(chunk: list[str]) -> list[dict[str, Any]]:
177
+ flt = ",".join([f"{field}:" + "|".join(chunk)] + (extra or []))
178
+ return _get({"filter": flt, "per-page": 50, "select": _FIELDS}).get("results") or []
179
+ return [_record(w) for w in _parallel(one, _chunks(values))]
180
+
181
+
182
+ def _pages(flt: str, max_records: int, *, split_over: bool = False) -> list[dict[str, Any]] | None:
183
+ """Up to ``max_records`` works matching ``flt``, most cited first, so a cut drops the least cited.
184
+
185
+ With ``split_over``, return None as soon as the first page shows more than
186
+ ``max_records`` matches, so the caller can split the query instead; the
187
+ first page already carries the count, so no separate count request is
188
+ needed (each topic-search request costs 10 OpenAlex credits).
189
+ """
190
+ # Cursor paging past the first page needs an API key; without one, page numbers
191
+ # (allowed up to 10,000 results) do the same job.
192
+ keyed = bool(str(load_config().get("openalex_api_key") or "").strip())
193
+ out, cursor, page = [], "*", 1
194
+ while len(out) < max_records:
195
+ params = {"filter": flt, "per-page": 200, "select": _FIELDS, "sort": "cited_by_count:desc"}
196
+ params.update({"cursor": cursor} if keyed else {"page": page})
197
+ d = _get(params)
198
+ meta = d.get("meta") or {}
199
+ if split_over and page == 1 and (meta.get("count") or 0) > max_records:
200
+ return None
201
+ rows = d.get("results") or []
202
+ out += rows
203
+ page += 1
204
+ cursor = meta.get("next_cursor")
205
+ if not rows or (keyed and not cursor) or len(out) >= (meta.get("count") or 0):
206
+ break
207
+ return [_record(w) for w in out[:max_records]]
208
+
209
+
210
+ def _citing(work_ids: list[str], years: list[str], max_records: int = 400) -> list[dict[str, Any]]:
211
+ return _parallel(lambda chunk: _pages(",".join(["cites:" + "|".join(chunk)] + years), max_records),
212
+ _chunks(work_ids))
213
+
214
+
215
+ def _topic_search(term: str, glossary_alts: dict[str, list[str]] | None = None) -> str:
216
+ """Core term -> OpenAlex search text. Words must all appear; ``a|b`` means either.
217
+
218
+ Papers often write only the abbreviation ("Al/SiC"), so the agent writes
219
+ "Al|aluminum SiC"; British spellings are added here, giving
220
+ "(Al OR aluminum OR aluminium) AND SiC". The user's glossary adds its other
221
+ terms to a word it knows (``glossary_alts``, from ``_glossary_expansions``),
222
+ a phrase in quotes: "(PDC OR \"polymer-derived ceramic\") AND aerogel".
223
+ """
224
+ from ..terms.spelling import variants
225
+ groups = []
226
+ for w in term.replace(",", " ").split():
227
+ alts = [v for a in w.split("|") if a for v in variants(a)]
228
+ for a in w.split("|"):
229
+ for g in (glossary_alts or {}).get(a.lower(), []):
230
+ alts += [f'"{g}"'] if (" " in g or "-" in g) else variants(g)
231
+ if alts:
232
+ groups.append(list(dict.fromkeys(alts)))
233
+ if all(len(g) == 1 for g in groups):
234
+ return " ".join(g[0] for g in groups)
235
+ return " AND ".join(("(" + " OR ".join(g) + ")") if len(g) > 1 else g[0] for g in groups)
236
+
237
+
238
+ def _budget_ok(credits0: int) -> bool:
239
+ """True while this round may still spend OpenAlex credits (see ROUND_CREDITS, RESERVE_CREDITS)."""
240
+ from .. import quota
241
+ if quota.spent("openalex") - credits0 >= ROUND_CREDITS:
242
+ return False
243
+ remaining = (quota.summary().get("openalex") or {}).get("remaining")
244
+ return remaining is None or remaining > RESERVE_CREDITS
245
+
246
+
247
+ def _cites_refs_on_topic(ref_ids: list[str], core_terms: list[str], years: list[str],
248
+ max_records: int = 1000, credits0: int | None = None,
249
+ glossary_alts: dict[str, list[str]] | None = None,
250
+ ) -> tuple[list[dict[str, Any]], set[str]]:
251
+ """Works citing any of ``ref_ids`` whose title/abstract mentions every core term.
252
+
253
+ Each core term is one concept the paper must mention (``a|b`` = either);
254
+ the terms are joined into one search, so ["lanthanide|europium",
255
+ "drug|doxorubicin"] means rare earth AND drug. (Until 2026-10-16 each term
256
+ was a separate search and a paper matching any one was kept, which let
257
+ 27,000 drug-delivery papers without rare earths into one pool.)
258
+ ``ref_ids`` come most important first. Returns the works and the references
259
+ actually asked about: once the round's credit budget is spent, the rest
260
+ are left for a later round.
261
+ """
262
+ from .. import quota
263
+ credits0 = quota.spent("openalex") if credits0 is None else credits0
264
+ term = " ".join(t.strip() for t in core_terms or [] if t.strip())
265
+ jobs = [(term, chunk) for chunk in _chunks(ref_ids)]
266
+
267
+ def flt(term: str, chunk: list[str]) -> str:
268
+ topic = [f"title_and_abstract.search:{_topic_search(term, glossary_alts)}"] if term else []
269
+ return ",".join(["cites:" + "|".join(chunk)] + topic + years)
270
+
271
+ def one(job: tuple[str, list[str]]) -> list[tuple[bool, list[dict[str, Any]]]]:
272
+ term, chunk = job
273
+ single = len(chunk) == 1
274
+ got = _pages(flt(term, chunk), SINGLE_REF_CAP if single else max_records, split_over=not single)
275
+ return [(True, [])] if got is None else [(False, got)]
276
+
277
+ out: list[dict[str, Any]] = []
278
+ asked: set[str] = set()
279
+ _step(0, len(ref_ids))
280
+ try:
281
+ while jobs: # a query over the cap is split in halves; a few jobs at a time, budget permitting
282
+ if not _budget_ok(credits0):
283
+ break
284
+ wave, jobs = jobs[:WORKERS * 2], jobs[WORKERS * 2:]
285
+ again = []
286
+ for (term, chunk), (split, got) in zip(wave, _parallel(one, wave)):
287
+ if split:
288
+ half = len(chunk) // 2
289
+ again += [(term, chunk[:half]), (term, chunk[half:])]
290
+ else:
291
+ out += got
292
+ asked.update(chunk)
293
+ _step(len(asked))
294
+ jobs = again + jobs # finish the halves of important references first
295
+ except BaseException as exc:
296
+ if out or asked:
297
+ exc._partial_expand = (out, asked) # type: ignore[attr-defined]
298
+ raise
299
+ return out, asked
300
+
301
+
302
+ # --------------------------------------------------------------------------- pool
303
+
304
+ def _keyword_candidates(queries: list[str], year_from: int | None, year_to: int | None) -> dict[str, list[dict]]:
305
+ from .quick import _collect
306
+
307
+ def one(q: str) -> list[dict[str, Any]]:
308
+ try:
309
+ return _collect(q, 100, 100, year_from, year_to, None, None, None)
310
+ except Exception as exc:
311
+ logger.warning("deep search: query %r failed (%s)", q, exc)
312
+ return []
313
+
314
+ with ThreadPoolExecutor(max_workers=4) as ex:
315
+ return dict(zip(queries, ex.map(one, queries)))
316
+
317
+
318
+ _FILLER = frozenset("""
319
+ a an and as at by for from in into of on or the their to via with using used study studies
320
+ effect effects new based nano vitro vivo application applications
321
+ """.split())
322
+
323
+ _FUNCTIONS = ("drug", "delivery", "release", "loading", "imaging", "therapy", "antioxidant",
324
+ "cytotoxicity", "bioactivity", "carrier")
325
+
326
+
327
+ def _exact_terms(queries: list[str], core_terms: list[str]) -> list[str]:
328
+ """Two- and three-word phrases to AND together instead of ranking by overlap.
329
+
330
+ OpenAlex's ``search`` ranks by how many words match, so a long query buries
331
+ papers that use one different word: a query saying "cerium oxide" missed a
332
+ paper titled "cerium dioxide nanoparticles" among the first hundred. ANDing
333
+ two or three words shrinks the result set to a few hundred and puts it back
334
+ in view (2026-11-07: the two rare-earth targets the keyword route never
335
+ found sat at ranks 34 and 3 this way).
336
+ """
337
+ known = {a.lower() for term in core_terms for w in term.replace(",", " ").split()
338
+ for a in w.split("|") if a}
339
+ out: list[str] = []
340
+ for q in queries:
341
+ words = [w for w in re.split(r"[^0-9a-z\u4e00-\u9fff-]+", q.lower()) if w and w not in _FILLER]
342
+ if not words:
343
+ continue
344
+ out.append(" ".join(words[:3]))
345
+ element = next((w for w in words if w in known), "")
346
+ function = next((w for w in words if w in _FUNCTIONS), "")
347
+ if element and function and element != function:
348
+ out.append(f"{element} nanoparticle {function}")
349
+ seen: list[str] = []
350
+ for t in out:
351
+ if t and t not in seen:
352
+ seen.append(t)
353
+ return seen[:16]
354
+
355
+
356
+ def _exact_candidates(terms: list[str], year_from: int | None, year_to: int | None) -> dict[str, list[dict]]:
357
+ """Up to 60 most cited works whose title or abstract has every word of ``term``."""
358
+ years = _year_filter(year_from, year_to)
359
+
360
+ def one(term: str) -> list[dict[str, Any]]:
361
+ flt = ",".join([f"title_and_abstract.search:{term}"] + years)
362
+ try:
363
+ return _pages(flt, 60, split_over=False) or []
364
+ except Exception as exc:
365
+ logger.warning("deep search: exact query %r failed (%s)", term, exc)
366
+ return []
367
+
368
+ with ThreadPoolExecutor(max_workers=3) as ex:
369
+ return dict(zip(terms, ex.map(one, terms)))
370
+
371
+
372
+ def _exact_terms_of(rec: dict[str, Any]) -> list[str]:
373
+ """The and-style phrases that found this record (``_exact_terms`` wrote them into ``via``)."""
374
+ return [v[len("exact: "):] for v in (rec.get("via") or []) if v.startswith("exact: ")]
375
+
376
+
377
+ def _title_cover(rec: dict[str, Any], terms: list[str]) -> float:
378
+ """Best share of a phrase's words that the title itself carries (0 when it has none)."""
379
+ from ..terms.text import plain, singular
380
+ if not terms:
381
+ return 0.0
382
+ title = {singular(w) for w in plain(rec.get("title") or "").split()}
383
+ best = 0.0
384
+ for term in terms:
385
+ words = [singular(w) for w in plain(term).split() if w]
386
+ if words:
387
+ best = max(best, sum(1 for w in words if w in title) / len(words))
388
+ return best
389
+
390
+
391
+ def _key(rec: dict[str, Any]) -> str:
392
+ return rec.get("doi") or ("oa:" + rec["openalex"] if rec.get("openalex") else "")
393
+
394
+
395
+ def _add(pool: dict[str, dict], rec: dict[str, Any], via: str) -> None:
396
+ doi = rec.get("doi") or ""
397
+ if doi.startswith("10.1002/ange."): # Angewandte's German edition: same article as 10.1002/anie.*
398
+ rec = {**rec, "doi": "10.1002/anie." + doi[len("10.1002/ange."):]}
399
+ k = _key(rec)
400
+ if not k or not rec.get("title") or rec.get("type") in NON_PAPER:
401
+ return
402
+ have = pool.get(k)
403
+ if have is None:
404
+ pool[k] = {**rec, "via": [via]}
405
+ return
406
+ for field in ("abstract", "journal", "openalex", "refs", "year", "issn", "authors"):
407
+ if not have.get(field) and rec.get(field):
408
+ have[field] = rec[field]
409
+ if via != "metadata" and via not in have["via"]: # a metadata lookup is not a way of finding it
410
+ have["via"].append(via)
411
+
412
+
413
+ def _score(state: dict[str, Any]) -> None:
414
+ """Formula score: anchored coupling + QUERY_WEIGHT x fully matched queries.
415
+
416
+ References of the user's own seeds count fully; references that only
417
+ snowballed seeds cite count by how many seeds share them (1/3 each, at most
418
+ 1), so a seed set drifting toward a neighbouring topic cannot outvote the
419
+ original intent.
420
+ """
421
+ from .quick import _match_tier
422
+ pool = state["pool"]
423
+ seed_refs = set(state["seed_refs"])
424
+ orig_refs = {r for s in _orig_seeds(state) for r in (pool.get(s, {}).get("refs") or [])}
425
+ seed_df = Counter(r for s in state["seeds"] for r in set(pool.get(s, {}).get("refs") or []))
426
+ # Matching every query against every record is most of the cost; a record's
427
+ # hits only change with its text or the queries, so only new or changed ones are matched.
428
+ cache = _cache(state)["hits"]
429
+ queries = tuple(state["queries"])
430
+ # A query's glossary variants ("PDC aerogel" -> "polymer-derived ceramic aerogel")
431
+ # count as the query: a paper matching either has matched it once.
432
+ variants = state.get("query_variants") or {}
433
+ sig = (queries, tuple(tuple(variants.get(q) or ()) for q in queries))
434
+ stale = {k: r for k, r in pool.items() if cache.get(k, (None,))[0] != (sig, _text_sig(r))}
435
+ fresh = {k: 0 for k in stale}
436
+ for q in queries:
437
+ rows = [{"title": r["title"], "abstract": r.get("abstract") or "", "_k": k} for k, r in stale.items()]
438
+ hit = set()
439
+ for form in [q, *(variants.get(q) or [])]:
440
+ hit |= {row["_k"] for row in _match_tier(rows, form) if row["_tier"] == 0}
441
+ for k in hit:
442
+ fresh[k] += 1
443
+ for k, n in fresh.items():
444
+ cache[k] = ((sig, _text_sig(stale[k])), n)
445
+ hits = {k: cache[k][1] for k in pool}
446
+ for k, r in pool.items():
447
+ shared = [ref for ref in (r.get("refs") or []) if ref in seed_refs]
448
+ r["coupling"] = len(shared)
449
+ r["anchored"] = round(sum(1.0 if ref in orig_refs else min(1.0, seed_df[ref] / 3) for ref in shared), 2)
450
+ r["query_hits"] = hits[k]
451
+ cover = _title_cover(r, _exact_terms_of(r))
452
+ r["phrase"] = round(cover, 2)
453
+ r["score"] = (r["anchored"] + QUERY_WEIGHT * r["query_hits"]
454
+ + (EXACT_WEIGHT if cover >= EXACT_MIN_COVER - 1e-9 else 0.0))
455
+ r["rank"] = r["score"]
456
+
457
+
458
+ _CACHES: dict[str, dict[str, dict]] = {} # session -> per-record results that depend only on the record's text
459
+ _CACHES_LOCK = threading.Lock()
460
+
461
+
462
+ def _cache(state: dict[str, Any]) -> dict[str, dict]:
463
+ """In memory only: a restarted server simply recomputes.
464
+
465
+ Query hits (one number per paper) are kept for the latest 4 sessions; the
466
+ token counts (measured 2026-10-18: about 37 MB for 7,309 papers, 270 MB for
467
+ 52,668) only for the session in use.
468
+ """
469
+ session = str(state.get("session") or "")
470
+ with _CACHES_LOCK:
471
+ c = _CACHES.pop(session, None) or {"hits": {}, "tokens": {}}
472
+ for other in _CACHES.values():
473
+ other["tokens"] = {}
474
+ _CACHES[session] = c # most recent last
475
+ while len(_CACHES) > 4:
476
+ _CACHES.pop(next(iter(_CACHES)))
477
+ return c
478
+
479
+
480
+ _SESSION_LOCKS: dict[str, threading.Lock] = {}
481
+
482
+
483
+ def _session_lock(session: str) -> threading.Lock:
484
+ """One round at a time per session (the app's 继续检索 and the agent's next
485
+ could otherwise rank the same pool, and its caches, at once)."""
486
+ with _CACHES_LOCK:
487
+ if session not in _SESSION_LOCKS and len(_SESSION_LOCKS) >= 50:
488
+ for old in [s for s, lk in _SESSION_LOCKS.items() if not lk.locked()]:
489
+ del _SESSION_LOCKS[old] # idle sessions' locks: made again if needed
490
+ return _SESSION_LOCKS.setdefault(session, threading.Lock())
491
+
492
+
493
+ def _text_sig(r: dict[str, Any]) -> int:
494
+ return hash((r.get("title") or "", r.get("abstract") or ""))
495
+
496
+
497
+ def _orig_seeds(state: dict[str, Any]) -> list[str]:
498
+ return state.get("orig_seeds") or state["seeds"][:3]
499
+
500
+
501
+ # --------------------------------------------------------------------------- ranking model
502
+ #
503
+ # Continuous active learning, the usual way screening tools for systematic
504
+ # reviews rank: after every batch, train a small logistic regression on all
505
+ # ratings so far (高 and the seeds against 中/低/不相关, plus a random sample of
506
+ # unscreened papers standing in as negatives) and rank the rest by it.
507
+ # Negative ratings pull the ranking back when snowballing drifts. On the
508
+ # SiC/Al set, after the first 90 ratings, the 6 targets still unscreened
509
+ # ranked [18, 39, 68, 70, 98, 1196] by the old formula and [9, 19, 30, 38, 52,
510
+ # 56] by the model (TODO.md, 2026-10-15).
511
+
512
+ _WORD = re.compile(r"[a-z][a-z0-9]+")
513
+
514
+
515
+ def _tokens(rec: dict[str, Any]) -> Counter:
516
+ from ..terms.spelling import to_us
517
+ from ..terms.text import RANKING_STOPWORDS, stem
518
+ words = (rec.get("title") or "") + " " + (rec.get("abstract") or "")
519
+ return Counter(stem(w) for w in _WORD.findall(to_us(words)) if w not in RANKING_STOPWORDS)
520
+
521
+
522
+ def _features(state: dict[str, Any]) -> dict[str, dict[str, float]]:
523
+ pool = state["pool"]
524
+ cache = _cache(state)["tokens"]
525
+ docs, titles = {}, {}
526
+ for k, r in pool.items():
527
+ sig = _text_sig(r)
528
+ hit = cache.get(k)
529
+ if hit is None or hit[0] != sig:
530
+ hit = cache[k] = (sig, _tokens(r), list(_tokens({"title": r.get("title")})))
531
+ docs[k], titles[k] = hit[1], hit[2]
532
+ df = Counter(w for c in docs.values() for w in c)
533
+ n, nq = len(pool), max(1, len(state["queries"]))
534
+ out = {}
535
+ for k, c in docs.items():
536
+ v = {w: (1 + math.log(m)) * math.log(n / df[w]) for w, m in c.items() if df[w] >= 2}
537
+ norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
538
+ v = {w: x / norm for w, x in v.items()}
539
+ r = pool[k]
540
+ # Title words again, on their own: a title says what the paper is about
541
+ # better than the abstract, and is all there is when the abstract is missing.
542
+ v.update({"t:" + w: TITLE_WEIGHT for w in titles[k]})
543
+ v["_coupling"] = math.log1p(r.get("anchored", 0)) / 3
544
+ v["_queries"] = r.get("query_hits", 0) / nq
545
+ v["_no_abstract"] = 0.0 if r.get("abstract") else 1.0
546
+ out[k] = v
547
+ return out
548
+
549
+
550
+ def _train(examples: list[tuple[dict[str, float], int]], epochs: int = 60, lr: float = 0.5,
551
+ l2: float = 1e-3) -> Counter:
552
+ w: Counter = Counter()
553
+ for _ in range(epochs):
554
+ for v, y in examples:
555
+ z = sum(w[f] * x for f, x in v.items()) + w["_bias"]
556
+ g = 1 / (1 + math.exp(-max(-30.0, min(30.0, z)))) - y
557
+ for f, x in v.items():
558
+ w[f] -= lr * (g * x + l2 * w[f])
559
+ w["_bias"] -= lr * g
560
+ return w
561
+
562
+
563
+ def _rerank(state: dict[str, Any], rated: dict[str, str]) -> str:
564
+ """Set every record's ``rank``; returns "model" or "formula"."""
565
+ pool, seeds = state["pool"], set(state["seeds"])
566
+ pos = {k for k, r in rated.items() if r == "高" and k in pool} | {k for k in seeds if k in pool}
567
+ neg = [k for k, r in rated.items() if r != "高" and k in pool]
568
+ if len(pos) < MIN_LABELS[0] or len(neg) < MIN_LABELS[1]:
569
+ for r in pool.values():
570
+ r["rank"] = r.get("score", 0)
571
+ return "formula"
572
+ rng = random.Random(len(rated)) # same ratings, same ranking
573
+ unscreened = sorted(k for k in pool if k not in rated and k not in seeds)
574
+ pseudo = rng.sample(unscreened, min(PSEUDO_NEGATIVES, len(unscreened)))
575
+ feats = _features(state)
576
+ examples = [(feats[k], 1) for k in sorted(pos)] + [(feats[k], 0) for k in neg + pseudo]
577
+ rng.shuffle(examples)
578
+ w = _train(examples)
579
+ for k, r in pool.items():
580
+ r["rank"] = round(sum(w[f] * x for f, x in feats[k].items()) + w["_bias"], 4)
581
+ return "model"
582
+
583
+
584
+ def _prefill_abstracts(state: dict[str, Any], rated: dict[str, str]) -> int:
585
+ """Fetch abstracts for the top unscreened candidates that lack one, once each.
586
+
587
+ Ranking reads abstracts, so a paper without one used to sink and never
588
+ reach a batch, which is where its abstract would have been fetched.
589
+ """
590
+ return _prefill_fetch(_prefill_pick(state, rated))
591
+
592
+
593
+ def _prefill_pick(state: dict[str, Any], rated: dict[str, str]) -> list[dict[str, Any]]:
594
+ """The PREFILL best-ranked unscreened records that still lack an abstract."""
595
+ # Pick among the papers that lack one: they rank low for that very reason, so
596
+ # taking the overall top PREFILL would reach almost none of them.
597
+ todo = [r for k in _ranked(state) if k not in rated
598
+ for r in [state["pool"][k]] if not r.get("abstract") and not r.get("abstract_tried") and r.get("doi")]
599
+ return todo[:PREFILL]
600
+
601
+
602
+ def _prefill_fetch(todo: list[dict[str, Any]]) -> int:
603
+ """Fill abstracts into these records in place; returns how many were found."""
604
+ from .abstracts import fill_abstracts
605
+ if not todo:
606
+ return 0
607
+ _phase("abstracts", len(todo))
608
+ # Scopus answers about 3 papers a second at its allowed pace: budget for that.
609
+ fill_abstracts(todo, load_config(), budget_s=min(90.0, max(30.0, len(todo) / 3)), progress=_step)
610
+ for r in todo:
611
+ if r.get("abstract") or r.pop("abstract_decided", False):
612
+ r["abstract_tried"] = True # undecided ones (timeout, throttled) are asked again next round
613
+ return sum(1 for r in todo if r.get("abstract"))
614
+
615
+
616
+ def _pool_path(session: str):
617
+ return screening.screening_dir() / f"{session}.pool.json"
618
+
619
+
620
+ def has_pool(session: str) -> bool:
621
+ """Return True if a deep-search candidate pool exists for the session."""
622
+ return db.has_pool(session)
623
+
624
+
625
+ def _load(session: str) -> dict[str, Any]:
626
+ return db.load_pool_state(session)
627
+
628
+
629
+ def _save(state: dict[str, Any]) -> None:
630
+ db.save_pool_state(state)
631
+
632
+
633
+ def _expand(state: dict[str, Any], new_seed_keys: list[str]) -> int:
634
+ """Add citation-route candidates for new seeds; returns how many records were added."""
635
+ pool = state["pool"]
636
+ years = _year_filter(state.get("year_from"), state.get("year_to"))
637
+ from .. import quota
638
+ credits0 = quota.spent("openalex")
639
+ seeds = [pool[k] for k in new_seed_keys if k in pool]
640
+ # References shared by more seeds (the user's own seeds count double) are asked first,
641
+ # so a spent budget drops the least informative ones; they stay for a later round.
642
+ orig = set(_orig_seeds(state))
643
+ weight = Counter(r for k in set(state["seeds"]) | set(new_seed_keys) if k in pool
644
+ for r in set(pool[k].get("refs") or []) for _ in range(2 if k in orig else 1))
645
+ all_seeds = seeds + [pool[k] for k in state["seeds"] if k in pool]
646
+ new_refs = sorted({r for s in all_seeds for r in (s.get("refs") or [])} - set(state["queried_refs"]),
647
+ key=lambda r: (-weight[r], r))
648
+ before = len(pool)
649
+ asked: set[str] = set()
650
+ _phase("expand", len(new_refs))
651
+ try:
652
+ if new_refs:
653
+ for rec in _by_filter("openalex", new_refs, years):
654
+ _add(pool, rec, "seed_reference")
655
+ got, asked = _cites_refs_on_topic(new_refs, state["core_terms"], years, credits0=credits0,
656
+ glossary_alts=state.get("glossary_alts"))
657
+ for rec in got:
658
+ _add(pool, rec, "cites_seed_references")
659
+ seed_ids = [s["openalex"] for s in seeds if s.get("openalex")]
660
+ if seed_ids and _budget_ok(credits0):
661
+ for rec in _citing(seed_ids, years):
662
+ _add(pool, rec, "cites_seed")
663
+ except BaseException as exc:
664
+ partial = getattr(exc, "_partial_expand", None)
665
+ if partial:
666
+ partial_got, partial_asked = partial
667
+ for rec in partial_got:
668
+ _add(pool, rec, "cites_seed_references")
669
+ asked |= partial_asked
670
+ state["deferred_refs"] = len(new_refs) - len(asked)
671
+ state["queried_refs"] = sorted(set(state["queried_refs"]) | asked)
672
+ state["seed_refs"] = sorted(set(state["seed_refs"]) | {r for s in seeds for r in (s.get("refs") or [])})
673
+ if state.get("session"):
674
+ _save(state)
675
+ raise
676
+ deferred = len(new_refs) - len(asked)
677
+ state["deferred_refs"] = deferred
678
+ if deferred:
679
+ logger.info("deep search: OpenAlex budget for this round used; %d references left for later", deferred)
680
+ state["queried_refs"] = sorted(set(state["queried_refs"]) | asked)
681
+ state["seed_refs"] = sorted(set(state["seed_refs"]) | {r for s in seeds for r in (s.get("refs") or [])})
682
+ for k in new_seed_keys:
683
+ if k not in state["seeds"]:
684
+ state["seeds"].append(k)
685
+ return len(pool) - before
686
+
687
+
688
+ # --------------------------------------------------------------------------- public
689
+
690
+ def _core_term_notes(core_terms: list[str]) -> list[str]:
691
+ """Warnings about core terms that would make the citation route far too broad."""
692
+ from ..terms.glossary import AMBIGUOUS_SYMBOLS
693
+ alts = {a.lower() for term in core_terms for w in term.replace(",", " ").split() for a in w.split("|") if a}
694
+ loose = sorted(alts & AMBIGUOUS_SYMBOLS)
695
+ if not loose:
696
+ return []
697
+ return ["core_terms contain element symbols that are also common words (" + ", ".join(loose)
698
+ + "; search ignores case): write the element's full name instead"]
699
+
700
+
701
+ def start(question: str, queries: list[str], seeds: list[str], core_terms: list[str], *,
702
+ criteria: list[str] | None = None, must_find: list[str] | None = None,
703
+ year_from: int | None = None, year_to: int | None = None,
704
+ batch_size: int = 20, cap: int = 200) -> dict[str, Any]:
705
+ """Build the pool and return the first batch to screen."""
706
+ if not queries:
707
+ raise ValueError("give 4-8 English queries")
708
+ if not seeds:
709
+ raise ValueError("give 1-3 seed DOIs (papers the user already knows are on topic)")
710
+ t0 = time.time()
711
+ session = screening.save(question, [], criteria=criteria, query=" | ".join(queries), mode="deep",
712
+ search_params={"core_terms": core_terms, "seeds": seeds,
713
+ "year_from": year_from, "year_to": year_to})["session"]
714
+ _run.progress = store.Progress(session, "start")
715
+ try:
716
+ return _start(session, t0, question, queries, seeds, core_terms, must_find, year_from, year_to,
717
+ batch_size, cap)
718
+ except BaseException as exc:
719
+ _run.progress.finish("failed", error=str(exc) or type(exc).__name__)
720
+ raise
721
+ finally:
722
+ _run.progress = None
723
+
724
+
725
+ def _start(session: str, t0: float, question: str, queries: list[str], seeds: list[str],
726
+ core_terms: list[str], must_find: list[str] | None, year_from: int | None, year_to: int | None,
727
+ batch_size: int, cap: int) -> dict[str, Any]:
728
+ query_variants, glossary_alts, glossary_used = _glossary_expansions(queries, core_terms)
729
+ all_queries = queries + [v for q in queries for v in query_variants.get(q, [])]
730
+ years = _year_filter(year_from, year_to)
731
+ pool: dict[str, dict] = {}
732
+ seed_recs = _by_filter("doi", [canonical_doi(s) for s in seeds])
733
+ for rec in seed_recs:
734
+ _add(pool, rec, "seed")
735
+ # Off by default: measured 2026-11-07 on the rare-earth set, it puts 8 of the 9 targets
736
+ # into the pool but the ranking buries them again (the read window found 1 instead of 2).
737
+ # See docs/DECISIONS.md; keep it for experiments until the scorer can tell the hits apart.
738
+ exact_terms = _exact_terms(queries, core_terms) if load_config().get("deep_exact_queries") else []
739
+ _phase("keywords", len(all_queries) + len(exact_terms))
740
+ if exact_terms:
741
+ for term, rows in _exact_candidates(exact_terms, year_from, year_to).items():
742
+ for rec in rows:
743
+ _add(pool, rec, f"exact: {term}")
744
+ for q, rows in _keyword_candidates(all_queries, year_from, year_to).items():
745
+ for r in rows:
746
+ _add(pool, {"openalex": None, "doi": canonical_doi(r.get("doi")) or None,
747
+ "title": r.get("title") or "", "year": r.get("year"), "journal": r.get("journal") or "",
748
+ "issn": r.get("issn") or [], "authors": r.get("authors") or [],
749
+ "cited_by_count": r.get("cited_by_count") or 0, "abstract": r.get("abstract") or "",
750
+ "refs": None}, f"query: {q}")
751
+ # Keyword hits come without reference lists; fetch them so coupling can count.
752
+ missing = [k for k, r in pool.items() if r.get("refs") is None and r.get("doi")]
753
+ for rec in _by_filter("doi", missing):
754
+ _add(pool, rec, "metadata")
755
+ state = {"session": session, "question": question, "queries": queries, "core_terms": core_terms,
756
+ "orig_seeds": [_key(r) for r in seed_recs],
757
+ "year_from": year_from, "year_to": year_to, "batch_size": batch_size, "cap": cap,
758
+ "must_find": [canonical_doi(d) for d in (must_find or [])], "seeds": [], "seed_refs": [],
759
+ "queried_refs": [], "served": [], "rounds": [], "pool": pool,
760
+ "query_variants": query_variants, "glossary_alts": glossary_alts, "glossary_used": glossary_used}
761
+ _save(state)
762
+ _expand(state, [_key(r) for r in seed_recs])
763
+ _score(state)
764
+ state["rounds"].append({"time": time.strftime("%Y-%m-%d %H:%M:%S"), "event": "start",
765
+ "pool": len(pool), "seconds": round(time.time() - t0, 1)})
766
+ _save(state)
767
+ out = _next_batch(state)
768
+ notes = _core_term_notes(core_terms)
769
+ if notes:
770
+ out["core_term_notes"] = notes
771
+ if glossary_used:
772
+ out["glossary"] = glossary_used
773
+ _run.progress.finish()
774
+ return out
775
+
776
+
777
+ def _glossary_expansions(queries: list[str], core_terms: list[str],
778
+ ) -> tuple[dict[str, list[str]], dict[str, list[str]], list[dict[str, Any]]]:
779
+ """What the glossary (``glossary.py``: the user's entries and the built-in
780
+ elements) adds to this search: other wordings of each query, other terms for
781
+ the words of the core terms, and a report of both for the user. Core-term
782
+ words are expanded only when there are at least two of them, so another
783
+ concept keeps each expansion on topic."""
784
+ from ..terms import glossary
785
+ try:
786
+ use = glossary.in_use()
787
+ except Exception as exc: # an unreadable glossary must not stop the search
788
+ logger.warning("deep search: glossary not read (%s)", exc)
789
+ return {}, {}, []
790
+ variants: dict[str, list[str]] = {}
791
+ used: list[dict[str, Any]] = []
792
+ for q in queries:
793
+ v, u = glossary.query_variants(q, use)
794
+ if v:
795
+ variants[q] = v
796
+ used += u
797
+ alts: dict[str, list[str]] = {}
798
+ words = [a for term in core_terms for w in term.replace(",", " ").split() for a in w.split("|") if a]
799
+ if len(words) >= 2:
800
+ for a in words:
801
+ more, kinds = glossary.alternatives(a, use)
802
+ if more and a.lower() not in alts:
803
+ alts[a.lower()] = more
804
+ used.append({"term": a, "with": more, "core_term": True, **({"builtin": kinds[0]} if kinds else {})})
805
+ return variants, alts, used
806
+
807
+
808
+ def _glossary_suggestions(state: dict[str, Any], rated: dict[str, str]) -> list[dict[str, Any]]:
809
+ """Abbreviations the papers read so far define ("polymer-derived ceramics
810
+ (PDCs)"), added to the glossary as suggestions for the user to confirm."""
811
+ from ..terms import glossary
812
+ pool = state["pool"]
813
+ keys = set(state["seeds"]) | {k for k, r in rated.items() if r in ("高", "中")}
814
+ texts = [pool[k].get("abstract") or "" for k in keys if k in pool]
815
+ try:
816
+ return [{k: e.get(k) for k in ("id", "terms", "note")} for e in glossary.suggest_from(texts)]
817
+ except Exception as exc: # a suggestion is a convenience; the search goes on without it
818
+ logger.warning("deep search: no glossary suggestions (%s)", exc)
819
+ return []
820
+
821
+
822
+ def next_batch(session: str, force: bool = False) -> dict[str, Any]:
823
+ """Fold the latest ratings back in (高 papers become seeds) and serve the next batch.
824
+
825
+ force: serve a batch even when stopping is suggested (the user chose to go on;
826
+ the app's 继续检索 button).
827
+ """
828
+ _run.progress = store.Progress(session, "next")
829
+ try:
830
+ with _session_lock(session):
831
+ out = _next_round(session, force)
832
+ except BaseException as exc:
833
+ _run.progress.finish("failed", error=str(exc) or type(exc).__name__)
834
+ raise
835
+ finally:
836
+ p, _run.progress = _run.progress, None
837
+ p.finish()
838
+ return out
839
+
840
+
841
+ def _next_round(session: str, force: bool = False) -> dict[str, Any]:
842
+ from .. import quota
843
+ t0 = time.time()
844
+ credits0 = quota.spent("openalex")
845
+ state = _load(session)
846
+ rated = _ratings(state)
847
+ orig_missing = [k for k in _orig_seeds(state) if k in state["pool"] and k not in state["seeds"]]
848
+ new_seeds = list(dict.fromkeys(orig_missing + [
849
+ k for k, rel in rated.items() if rel == "高" and k in state["pool"] and k not in state["seeds"]
850
+ ]))
851
+ # Abstracts for the best-ranked papers without one, fetched while the pool is
852
+ # expanded (2026-10-20: the two ran one after the other, 12-25 s + ~50 s a
853
+ # round). Picked from last round's ranking, so papers this expansion adds
854
+ # get theirs next round.
855
+ todo = _prefill_pick(state, rated) if force or not _stop(state, rated)["stop_suggested"] else []
856
+ added, found = 0, 0
857
+ with ThreadPoolExecutor(max_workers=1) as ex:
858
+ filling = ex.submit(_prefill_fetch, todo) if todo else None
859
+ if new_seeds or state.get("deferred_refs"):
860
+ added = _expand(state, new_seeds)
861
+ t_exp = time.time()
862
+ if filling is not None:
863
+ if not filling.done():
864
+ _phase("abstracts", len(todo))
865
+ found = filling.result()
866
+ if added or found or not state.get("rounds"):
867
+ _score(state)
868
+ t1 = time.time()
869
+ rnd = {"time": time.strftime("%Y-%m-%d %H:%M:%S"), "event": "next", "new_seeds": len(new_seeds),
870
+ "pool_added": added, "expand_seconds": round(t_exp - t0, 1),
871
+ "abstracts_asked": len(todo), "abstracts_found": found,
872
+ "abstracts_wait_seconds": round(t1 - t_exp, 1)} # beyond the expansion
873
+ state["rounds"].append(rnd)
874
+ _save(state)
875
+ out = _next_batch(state, rated, force, prefilled=True)
876
+ suggested = _glossary_suggestions(state, rated)
877
+ if suggested:
878
+ out["glossary_suggestions"] = suggested
879
+ rnd["rank_seconds"] = round(time.time() - t1, 1) # abstracts, model, journal ranks
880
+ rnd["openalex_credits"] = quota.spent("openalex") - credits0
881
+ rnd["api_budget"] = {k: {f: v.get(f) for f in ("remaining", "limit", "resets_at")}
882
+ for k, v in quota.summary().items()}
883
+ _save(state)
884
+ return out
885
+
886
+
887
+ def rate(session: str, ratings: list[dict[str, Any]]) -> dict[str, Any]:
888
+ """Save the agent's ratings for the latest batch by number: [{"n": 3, "relevance": "高", "reason": "..."}].
889
+
890
+ Title, DOI, year and journal come from the batch, so the agent writes only
891
+ its judgement. Nothing is saved if any number or relevance is invalid.
892
+ """
893
+ state = _load(session)
894
+ batch = state.get("last_batch") or []
895
+ if not batch:
896
+ raise ValueError("no batch to rate yet; call action='start' or 'next' first")
897
+ items, bad = [], []
898
+ for r in ratings or []:
899
+ try:
900
+ n = int(r.get("n"))
901
+ except (TypeError, ValueError):
902
+ n = 0
903
+ rel = str(r.get("relevance") or "").strip()
904
+ if not 1 <= n <= len(batch) or rel not in screening.RELEVANCE:
905
+ bad.append(r)
906
+ continue
907
+ b = batch[n - 1]
908
+ items.append({**{f: b.get(f) for f in _BATCH_FIELDS},
909
+ "relevance": rel, "reason": str(r.get("reason") or rel), "abstract_zh": r.get("abstract_zh")})
910
+ if bad:
911
+ raise ValueError(f"nothing saved: invalid entries {bad[:3]}; n is 1-{len(batch)}, "
912
+ f"relevance one of {'/'.join(screening.RELEVANCE)}")
913
+ saved = screening.save("", items, session=session, mode="deep")
914
+ done = {int(r["n"]) for r in ratings}
915
+ out = {"session": session, "saved": len(items), "total": saved["total"], "by_relevance": saved["by_relevance"],
916
+ "not_rated": [i for i in range(1, len(batch) + 1) if i not in done],
917
+ "next_step": "action='next'" if len(done) == len(batch) else "rate the papers in not_rated, then action='next'"}
918
+ left = screening.untranslated(session)
919
+ if left: # the app shows these abstracts in Chinese
920
+ out["untranslated"] = left
921
+ out["next_step"] = ("translate their abstracts into Chinese: litloop_screening(action='translate', session=..., "
922
+ "items=[{doi, abstract_zh}]); then " + out["next_step"])
923
+ return out
924
+
925
+
926
+ def status(session: str) -> dict[str, Any]:
927
+ """Counts, coverage, the stop rule, and the latest batch's papers nobody has rated:
928
+ the app can serve a batch itself (继续检索), so the agent may come back to one."""
929
+ state = _load(session)
930
+ rated = _ratings(state)
931
+ order = _ranked(state)
932
+ agent = _agent_rated(state)
933
+ unrated = [{"n": n, **{f: b.get(f) for f in _BATCH_FIELDS}}
934
+ for n, b in enumerate(state.get("last_batch") or [], 1) if _batch_key(b) not in agent]
935
+ stop = _stop(state, rated)
936
+ remaining = len([k for k in order if k not in rated])
937
+ batches = len([b for b in state.get("served") or [] if b])
938
+ deferred_refs = int(state.get("deferred_refs") or 0)
939
+ prog = store.read_progress(session)
940
+ progress_info: dict[str, Any] | None = None
941
+ if prog:
942
+ progress_info = {}
943
+ for k in ("event", "state", "phase", "stopped_phase", "error", "updated"):
944
+ v = prog.get(k)
945
+ if k == "phase" and isinstance(v, dict):
946
+ v = v.get("name")
947
+ if v is not None:
948
+ progress_info[k] = v
949
+ resumable = bool(
950
+ unrated
951
+ or (not stop["stop_suggested"] and (remaining > 0 or deferred_refs > 0 or not state.get("served")))
952
+ or (prog and prog.get("state") in ("failed", "interrupted"))
953
+ )
954
+ out = {"session": session, "pool": len(state["pool"]), "screened": len(rated),
955
+ "batches": batches, "remaining": remaining, "deferred_refs": deferred_refs,
956
+ "unrated_count": len(unrated), "progress": progress_info, "resumable": resumable,
957
+ "by_relevance": {r: sum(1 for v in rated.values() if v == r) for r in screening.RELEVANCE},
958
+ "seeds": len(state["seeds"]), "coverage": _coverage(state, rated, order), **stop}
959
+ if unrated:
960
+ out["unrated"] = unrated
961
+ out["next_step"] = ("rate these papers of the latest batch by their n (action='rate'; "
962
+ "a rating the user already gave in the app stays), then action='next'")
963
+ elif resumable:
964
+ out["next_step"] = "resume from where this session stopped: call action='next'"
965
+ return out
966
+
967
+
968
+ def _agent_rated(state: dict[str, Any]) -> set[str]:
969
+ """Screening keys of the papers the agent itself rated (the app's own ratings have no reason)."""
970
+ return {screening._key(it) for it in screening.load(state["session"])["items"] if it.get("reason")}
971
+
972
+
973
+ def _batch_key(b: dict[str, Any]) -> str:
974
+ return screening._key({"doi": b.get("doi"), "title": b.get("title")})
975
+
976
+
977
+ def batch_paper(session: str, doi: str | None, title: str | None) -> dict[str, Any] | None:
978
+ """A paper of the latest batch, as the batch showed it (for a rating the user gives first)."""
979
+ return db.get_batch_paper(session, doi, title)
980
+
981
+
982
+ # --------------------------------------------------------------------------- helpers
983
+
984
+ def _ratings(state: dict[str, Any]) -> dict[str, str]:
985
+ """Ratings keyed like the pool. Papers without a DOI are matched back by title, the way screening keys them."""
986
+ by_title = {screening._key({"title": r.get("title")}): k for k, r in state["pool"].items() if not r.get("doi")}
987
+ out = {}
988
+ for it in screening.load(state["session"])["items"]:
989
+ key = screening._key(it)
990
+ key = by_title.get(key) if key.startswith("title:") else key
991
+ if key:
992
+ out[key] = screening._rating(it) # the user's own rating (from the app) wins
993
+ return out
994
+
995
+
996
+ def _ranked(state: dict[str, Any]) -> list[str]:
997
+ seeds = set(state["seeds"])
998
+ pool = state["pool"]
999
+ return sorted((k for k in pool if k not in seeds),
1000
+ key=lambda k: (-pool[k].get("rank", pool[k].get("score", 0)), -(pool[k].get("cited_by_count") or 0)))
1001
+
1002
+
1003
+ def _remaining_estimate(state: dict[str, Any], rated: dict[str, str]) -> dict[str, Any] | None:
1004
+ """Roughly how many 高 papers are still unread, from the yield curve.
1005
+
1006
+ The model's scores are not calibrated probabilities (it is trained against a
1007
+ random sample of unread papers), so the estimate comes from what the ratings
1008
+ show instead: fit a geometric decay to the share of 高 per batch (least
1009
+ squares on log share, recent batches weighted more) and sum the batches
1010
+ still ahead, as far as the pool goes. Needs at least three rated batches;
1011
+ a flat or rising curve has no finite tail and gives no total.
1012
+ """
1013
+ batches = [b for b in state["served"] if b]
1014
+ rates, sizes = [], []
1015
+ for b in batches:
1016
+ judged = [k for k in b if k in rated]
1017
+ if len(judged) < max(3, len(b) // 2):
1018
+ continue
1019
+ rates.append(sum(1 for k in judged if rated[k] == "高") / len(judged))
1020
+ sizes.append(len(judged))
1021
+ if len(rates) < 3:
1022
+ return None
1023
+ size = state.get("batch_size") or len(batches[-1])
1024
+ xs = list(range(len(rates)))
1025
+ ys = [math.log(r + 0.02) for r in rates] # 0.02: a batch without 高 is not log(0)
1026
+ ws = [1 + i for i in xs]
1027
+ sw = sum(ws)
1028
+ mx = sum(w * x for w, x in zip(ws, xs)) / sw
1029
+ my = sum(w * y for w, y in zip(ws, ys)) / sw
1030
+ var = sum(w * (x - mx) ** 2 for w, x in zip(ws, xs))
1031
+ slope = sum(w * (x - mx) * (y - my) for w, x, y in zip(ws, xs, ys)) / var if var else 0.0
1032
+ q = math.exp(slope)
1033
+ last = max(0.0, math.exp(my + slope * (xs[-1] - mx)) - 0.02)
1034
+ seeds = set(state.get("seeds") or [])
1035
+ unread = sum(1 for k in state.get("pool") or {} if k not in rated and k not in seeds)
1036
+ out = {"next_batch": round(last * q * size, 1), "batches": len(rates), "decay": round(q, 2)}
1037
+ if q < 0.95 and unread:
1038
+ ahead = min(unread // max(1, size), 50)
1039
+ out["remaining"] = round(sum(last * q ** j * size for j in range(1, ahead + 1)), 1)
1040
+ return out
1041
+
1042
+
1043
+ def _stop(state: dict[str, Any], rated: dict[str, str]) -> dict[str, Any]:
1044
+ """Suggest stopping at the cap, or once the yield of 高 has fallen off.
1045
+
1046
+ Knee rule, as in screening for systematic reviews: when the share of 高 in
1047
+ the latest batch is below 1/KNEE_RATIO of the best earlier batch, the
1048
+ relevant papers have mostly been read. Counting 中 too never fired: the
1049
+ neighbouring topics that come up late keep it high.
1050
+ """
1051
+ est = _remaining_estimate(state, rated)
1052
+ if len(rated) >= state["cap"]:
1053
+ return _with_estimate({"stop_suggested": True, "stop_rule": "cap",
1054
+ "stop_reason": f"已评分 {len(rated)} 篇,达到上限 {state['cap']} 篇"}, est)
1055
+ batches = [b for b in state["served"] if b]
1056
+ last = batches[-1] if batches else []
1057
+ judged = [rated[k] for k in last if k in rated]
1058
+ if len(batches) < 2 or not judged or len(judged) < len(last) // 2:
1059
+ return _with_estimate({"stop_suggested": False, "stop_reason": ""}, est)
1060
+ high = sum(1 for r in judged if r == "高")
1061
+ rates = [sum(1 for k in b if rated.get(k) == "高") / max(1, sum(1 for k in b if k in rated)) for b in batches[:-1]]
1062
+ best = max(rates)
1063
+ if best > 0 and high / len(judged) <= best / KNEE_RATIO:
1064
+ return _with_estimate({"stop_suggested": True, "stop_rule": "knee",
1065
+ "stop_reason": (f"本批 {len(judged)} 篇中只有 {high} 篇评为高,不到此前最好一批"
1066
+ f"({best:.0%})的 1/{KNEE_RATIO},相关文献大多已读到")}, est)
1067
+ useful = sum(1 for r in judged if r in ("高", "中"))
1068
+ if useful <= max(1, len(judged) // 15):
1069
+ return _with_estimate({"stop_suggested": True, "stop_rule": "few_useful",
1070
+ "stop_reason": f"本批 {len(judged)} 篇中只有 {useful} 篇评为高或中,新增已经很少"}, est)
1071
+ return _with_estimate({"stop_suggested": False, "stop_reason": ""}, est)
1072
+
1073
+
1074
+ def _stop_rule(stop: dict[str, Any]) -> str:
1075
+ """Which rule suggested stopping ("" when none): ``stop_rule`` as _stop sets it."""
1076
+ return str(stop.get("stop_rule") or "") if stop.get("stop_suggested") else ""
1077
+
1078
+
1079
+ def _log_stop(state: dict[str, Any], rated: dict[str, str], stop: dict[str, Any]) -> None:
1080
+ """Keep what the stop rule and the estimate said at each point, to calibrate them later.
1081
+
1082
+ ``scripts/stop_calibration.py`` compares each entry with how many 高 papers
1083
+ were still found after it. One entry per number of rated papers.
1084
+ """
1085
+ log = state.setdefault("stop_checks", [])
1086
+ if log and log[-1].get("screened") == len(rated):
1087
+ log.pop() # asked again without new ratings: keep the latest answer
1088
+ est = stop.get("remaining_estimate") or {}
1089
+ log.append({"time": time.strftime("%Y-%m-%d %H:%M:%S"), "screened": len(rated),
1090
+ "high": sum(1 for r in rated.values() if r == "高"), "rule": _stop_rule(stop),
1091
+ "estimate": est.get("remaining"), "next_batch": est.get("next_batch"), "decay": est.get("decay")})
1092
+
1093
+
1094
+ def _with_estimate(stop: dict[str, Any], est: dict[str, Any] | None) -> dict[str, Any]:
1095
+ """Attach the estimate; when stopping is suggested, also say it in the reason."""
1096
+ if est is None:
1097
+ return stop
1098
+ stop["remaining_estimate"] = est
1099
+ if stop["stop_suggested"] and "remaining" in est:
1100
+ stop["stop_reason"] += (f"。按每批高相关比例的下降趋势估算,继续读约还能找到 "
1101
+ f"{est['remaining']:.0f} 篇高相关文献(下一批约 {est['next_batch']:.0f} 篇)")
1102
+ return stop
1103
+
1104
+
1105
+ def _coverage(state: dict[str, Any], rated: dict[str, str], order: list[str]) -> list[dict[str, Any]]:
1106
+ pos = {k: i for i, k in enumerate(order, 1)}
1107
+ out = []
1108
+ for doi in state.get("must_find") or []:
1109
+ if doi in rated:
1110
+ out.append({"doi": doi, "status": "screened", "relevance": rated[doi]})
1111
+ elif doi in state["seeds"]:
1112
+ out.append({"doi": doi, "status": "seed"})
1113
+ elif doi in pos:
1114
+ out.append({"doi": doi, "status": "in_pool", "rank": pos[doi]})
1115
+ else:
1116
+ out.append({"doi": doi, "status": "not_found"})
1117
+ return out
1118
+
1119
+
1120
+ # What a rated paper keeps in the screening record, so the app can show it
1121
+ # (abstract and authors included: the app never reads the pool).
1122
+ _BATCH_FIELDS = ("doi", "title", "year", "journal", "journal_rank", "retracted", "concern",
1123
+ "authors", "cited_by_count", "abstract", "abstract_missing")
1124
+
1125
+
1126
+ def _write_summary(state: dict[str, Any], rated: dict[str, str], stop: dict[str, Any],
1127
+ coverage: list[dict[str, Any]], notes: list[str], remaining: int,
1128
+ batch: list[dict[str, Any]]) -> None:
1129
+ """A small overview of the session for the app, which never reads the (huge) pool."""
1130
+ batches = []
1131
+ for i, keys in enumerate(b for b in state["served"] if b):
1132
+ counts = {r: sum(1 for k in keys if rated.get(k) == r) for r in screening.RELEVANCE}
1133
+ batches.append({"batch": i + 1, "size": len(keys), "rated": sum(counts.values()), **counts})
1134
+ rounds = [{f: r.get(f) for f in ("time", "event", "new_seeds", "pool_added", "expand_seconds",
1135
+ "rank_seconds", "openalex_credits", "seconds") if r.get(f) is not None}
1136
+ for r in state["rounds"]]
1137
+ summary = {"session": state["session"], "question": state["question"], "mode": "deep",
1138
+ "updated": time.strftime("%Y-%m-%d %H:%M:%S"), "pool": len(state["pool"]),
1139
+ "screened": len(rated), "remaining": remaining, "ranked_by": state.get("ranked_by", "formula"),
1140
+ "seeds": len(state["seeds"]), "batches": batches, "rounds": rounds,
1141
+ "stop_suggested": stop.get("stop_suggested", False), "stop_reason": stop.get("stop_reason", ""),
1142
+ "remaining_estimate": stop.get("remaining_estimate"),
1143
+ "coverage": coverage, "api_warnings": notes, "core_terms": state.get("core_terms", []),
1144
+ "glossary": state.get("glossary_used", []),
1145
+ "queries": state.get("queries", []), "current_batch": batch}
1146
+ try:
1147
+ db.save_summary(state["session"], summary)
1148
+ except (OSError, sqlite3.Error) as exc:
1149
+ logger.warning("deep search: could not write the session summary (%s)", exc)
1150
+
1151
+
1152
+ def _next_batch(state: dict[str, Any], rated: dict[str, str] | None = None, force: bool = False,
1153
+ prefilled: bool = False) -> dict[str, Any]:
1154
+ from .abstracts import clean_abstract, fill_abstracts
1155
+ from .quick import _add_journal_rank, _flag_retractions
1156
+ rated = rated if rated is not None else _ratings(state)
1157
+ stop = _stop(state, rated)
1158
+ _log_stop(state, rated, stop)
1159
+ go_on = force or not stop["stop_suggested"]
1160
+ if go_on:
1161
+ if not prefilled and _prefill_abstracts(state, rated):
1162
+ _score(state)
1163
+ _phase("rank")
1164
+ state["ranked_by"] = _rerank(state, rated)
1165
+ served_before = {k for b in state["served"] for k in b}
1166
+ order = [k for k in _ranked(state) if k not in rated]
1167
+ fresh = [k for k in order if k not in served_before]
1168
+ # Unrated papers from earlier batches come back first: the agent may have skipped some.
1169
+ pending = [k for k in order if k in served_before]
1170
+ picked = (pending + fresh)[: state["batch_size"]] if go_on else []
1171
+ items = [dict(state["pool"][k]) for k in picked]
1172
+ if items:
1173
+ _phase("batch", len(items))
1174
+ fill_abstracts([it for it in items if not it.get("abstract")], load_config(), progress=_step)
1175
+ for k, it in zip(picked, items): # keep what was found, so ranking can use it too
1176
+ if it.get("abstract") and not state["pool"][k].get("abstract"):
1177
+ state["pool"][k]["abstract"] = it["abstract"]
1178
+ state["pool"][k]["abstract_source"] = it.get("abstract_source")
1179
+ _add_journal_rank(items)
1180
+ _flag_retractions(items)
1181
+ batch = []
1182
+ for n, (k, it) in enumerate(zip(picked, items), 1):
1183
+ batch.append({
1184
+ "n": n, "doi": it.get("doi"), "title": it["title"], "year": it.get("year"), "journal": it.get("journal"),
1185
+ "journal_rank": it.get("journal_rank"), "authors": (it.get("authors") or [])[:3],
1186
+ "cited_by_count": it.get("cited_by_count"),
1187
+ "abstract": clean_abstract(it.get("abstract") or "")[:ABSTRACT_CHARS],
1188
+ "abstract_missing": not it.get("abstract"),
1189
+ "retracted": it.get("retracted") or None, "concern": it.get("concern") or None,
1190
+ "why": {"shared_refs_with_seeds": it.get("coupling", 0), "queries_matched": it.get("query_hits", 0),
1191
+ "found_via": it.get("via", [])[:2]},
1192
+ })
1193
+ if picked:
1194
+ state["served"].append([k for k in picked if k not in served_before])
1195
+ # What action="rate" needs to turn "n=3 高" back into a screening item.
1196
+ state["last_batch"] = [{"key": k, **{f: b.get(f) for f in _BATCH_FIELDS}} for k, b in zip(picked, batch)]
1197
+ _save(state)
1198
+ from .. import quota
1199
+ notes = quota.warnings()
1200
+ if not str(load_config().get("openalex_api_key") or "").strip():
1201
+ notes.insert(0, "no OpenAlex API key: the free daily budget without one is about 1,000 credits, "
1202
+ "which one deep search can use up; add a free key (openalex_api_key)")
1203
+ if state.get("deferred_refs"):
1204
+ notes.append(f"openalex: this round's credit budget was used; {state['deferred_refs']} seed references "
1205
+ "were not followed yet and are tried again in later rounds")
1206
+ coverage = _coverage(state, rated, _ranked(state))
1207
+ _write_summary(state, rated, stop, coverage, notes, remaining=len(order) - len(picked), batch=batch)
1208
+ return {"session": state["session"], "pool": len(state["pool"]), "screened": len(rated),
1209
+ "api_warnings": notes,
1210
+ "remaining": len(order) - len(picked), "ranked_by": state.get("ranked_by", "formula"), "batch": batch,
1211
+ "coverage": coverage, **stop,
1212
+ "next_step": ("rate every paper: action='rate', session=..., ratings=[{n, relevance, reason}"
1213
+ + (f", abstract_zh (Chinese abstract) for {'/'.join(screening.translate_wanted())}"
1214
+ if screening.translate_wanted() and not translate.by_api() else "")
1215
+ + "], then action='next'") if batch else
1216
+ "stop: show the user the screening session and let them choose"}