litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
litloop/search/deep.py
ADDED
|
@@ -0,0 +1,1216 @@
|
|
|
1
|
+
"""Deep search (stage 2): find the literature on a topic, not just the top hits.
|
|
2
|
+
|
|
3
|
+
The agent supplies a question, 4-8 English queries, 1-3 core terms and a few
|
|
4
|
+
seed papers. The plugin builds a candidate pool from two complementary
|
|
5
|
+
routes and hands it to the agent in ranked batches for screening:
|
|
6
|
+
|
|
7
|
+
* keyword route: every query's raw candidates (100 each, nothing cut);
|
|
8
|
+
* citation route: works that cite any of the seeds' references and mention a
|
|
9
|
+
core term in title/abstract, plus the seeds' own references and citing works.
|
|
10
|
+
|
|
11
|
+
On the acceptance sets neither route alone found most targets, together they
|
|
12
|
+
did (TODO.md, 2026-10-15). The first batch is ranked by
|
|
13
|
+
|
|
14
|
+
score = shared references with the seeds (anchored to the user's seeds)
|
|
15
|
+
+ 2 x queries the record fully matches
|
|
16
|
+
|
|
17
|
+
From then on a small model trained on every rating so far ranks the rest
|
|
18
|
+
(continuous active learning, see ``_rerank``), and missing abstracts of the
|
|
19
|
+
top candidates are fetched before ranking. Papers the agent rates 高 join the
|
|
20
|
+
seeds, so later batches follow what the user actually wants (snowballing). A batch with almost nothing relevant, or
|
|
21
|
+
the screening cap, ends the search with a stated reason.
|
|
22
|
+
|
|
23
|
+
Ratings live in the screening session (``litloop_screening``); this module
|
|
24
|
+
keeps the pool in SQLite (``pool_items`` + ``sessions.pool_state_json``, with
|
|
25
|
+
automatic import of legacy ``<session>.pool.json``).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import math
|
|
32
|
+
import random
|
|
33
|
+
import re
|
|
34
|
+
import sqlite3
|
|
35
|
+
import threading
|
|
36
|
+
import time
|
|
37
|
+
from collections import Counter
|
|
38
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
from . import screening
|
|
42
|
+
from .. import db, store
|
|
43
|
+
from ..desktop import translate
|
|
44
|
+
from ..config import load_config
|
|
45
|
+
from ..identifiers import canonical_doi
|
|
46
|
+
from ..log import get_logger
|
|
47
|
+
|
|
48
|
+
logger = get_logger()
|
|
49
|
+
_run = threading.local() # the round being worked on in this thread, for progress reports
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _phase(name: str, total: int | None = None) -> None:
|
|
53
|
+
p = getattr(_run, "progress", None)
|
|
54
|
+
if p:
|
|
55
|
+
p.phase(name, total)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _step(done: int, total: int | None = None) -> None:
|
|
59
|
+
p = getattr(_run, "progress", None)
|
|
60
|
+
if p:
|
|
61
|
+
p.step(done, total)
|
|
62
|
+
|
|
63
|
+
OPENALEX = "https://api.openalex.org/works"
|
|
64
|
+
_FIELDS = ("id,doi,title,publication_year,cited_by_count,referenced_works,"
|
|
65
|
+
"abstract_inverted_index,primary_location,authorships,type")
|
|
66
|
+
# Not research papers: never worth a reader's time. Book chapters, theses and
|
|
67
|
+
# conference papers stay; the ranking model learns where they belong.
|
|
68
|
+
NON_PAPER = {"dataset", "paratext", "erratum", "reference-entry", "book-review", "editorial",
|
|
69
|
+
"peer-review", "grant", "standard", "retraction", "supplementary-materials", "libguides"}
|
|
70
|
+
QUERY_WEIGHT = 2
|
|
71
|
+
EXACT_WEIGHT = 5.0 # a phrase from the exact pass sitting in the title (see the sweep in DECISIONS)
|
|
72
|
+
EXACT_MIN_COVER = 0.75 # ... and most of its words are there: all of a 2-4 word phrase
|
|
73
|
+
ABSTRACT_CHARS = 1200 # per paper in a batch; a batch of 20 stays under ~30 KB for the agent
|
|
74
|
+
PREFILL = 150 # fetch missing abstracts for this many top candidates before ranking
|
|
75
|
+
MIN_LABELS = (3, 5) # positives (高 + seeds), negatives (中/低/不相关) before the model takes over
|
|
76
|
+
PSEUDO_NEGATIVES = 100
|
|
77
|
+
TITLE_WEIGHT = 0.3 # extra weight of each title word in the ranking model
|
|
78
|
+
KNEE_RATIO = 6 # stop when 高 per batch falls below 1/6 of the best batch
|
|
79
|
+
WORKERS = 4 # parallel OpenAlex requests (each chunk of 50 ids is independent)
|
|
80
|
+
# OpenAlex budget guards (LitLoop's own choices, not official limits; the daily budget
|
|
81
|
+
# itself is in docs/API_LIMITS.md). A reference cited by thousands of on-topic papers
|
|
82
|
+
# (common in biomedicine) used to cost 50 credits for its 1000 most cited citers;
|
|
83
|
+
# one 2026-10-16 round spent 2184 credits this way.
|
|
84
|
+
SINGLE_REF_CAP = 200 # one reference over the cap: keep only its 200 most cited citers (one page)
|
|
85
|
+
ROUND_CREDITS = 800 # stop asking for more citers once a round has spent this many credits
|
|
86
|
+
RESERVE_CREDITS = 1000 # and leave this much of the daily budget for everything else
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# --------------------------------------------------------------------------- OpenAlex
|
|
90
|
+
|
|
91
|
+
def _boolean_heavy(params: dict[str, Any]) -> bool:
|
|
92
|
+
"""OpenAlex allows only 5 requests/s for searches with more than five boolean operators."""
|
|
93
|
+
text = " ".join(str(v) for v in params.values())
|
|
94
|
+
return len(re.findall(r"\b(?:OR|AND|NOT)\b", text)) > 5
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _get(params: dict[str, Any], *, tries: int = 6) -> dict[str, Any]:
|
|
98
|
+
"""One OpenAlex request, paced and counted by ``quota``. A 429 is retried with
|
|
99
|
+
growing waits; a spent daily budget (remaining 0) is reported plainly."""
|
|
100
|
+
from .. import quota
|
|
101
|
+
from .quick import _plain_session
|
|
102
|
+
key = str(load_config().get("openalex_api_key") or "").strip()
|
|
103
|
+
if key:
|
|
104
|
+
params = {**params, "api_key": key}
|
|
105
|
+
lane = ("openalex_boolean" if _boolean_heavy(params) else "openalex") + ("" if key else "_nokey")
|
|
106
|
+
last: Exception | None = None
|
|
107
|
+
for attempt in range(tries):
|
|
108
|
+
wait = min(30.0, 1.5 * 2 ** attempt)
|
|
109
|
+
try:
|
|
110
|
+
with quota.slot(lane):
|
|
111
|
+
r = _plain_session().get(OPENALEX, params=params, timeout=60)
|
|
112
|
+
quota.record("openalex", r)
|
|
113
|
+
remaining = r.headers.get("X-RateLimit-Remaining")
|
|
114
|
+
if r.status_code == 200:
|
|
115
|
+
return r.json()
|
|
116
|
+
if r.status_code == 429 and remaining == "0":
|
|
117
|
+
hours = round(int(r.headers.get("X-RateLimit-Reset") or 0) / 3600, 1)
|
|
118
|
+
raise RuntimeError(f"OpenAlex daily budget used up ({r.headers.get('X-RateLimit-Limit')} "
|
|
119
|
+
f"credits); it resets in about {hours} h (midnight UTC)")
|
|
120
|
+
if r.status_code == 429:
|
|
121
|
+
wait = max(wait, float(r.headers.get("Retry-After") or 0))
|
|
122
|
+
last = RuntimeError(f"OpenAlex HTTP {r.status_code}")
|
|
123
|
+
except RuntimeError:
|
|
124
|
+
raise
|
|
125
|
+
except Exception as exc: # network hiccup: retry
|
|
126
|
+
last = exc
|
|
127
|
+
time.sleep(wait)
|
|
128
|
+
raise RuntimeError(f"OpenAlex request failed: {last}")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _short(wid: str) -> str:
|
|
132
|
+
return str(wid or "").rsplit("/", 1)[-1]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _record(w: dict[str, Any]) -> dict[str, Any]:
|
|
136
|
+
from .quick import _reconstruct_abstract
|
|
137
|
+
src = ((w.get("primary_location") or {}).get("source") or {})
|
|
138
|
+
return {
|
|
139
|
+
"openalex": _short(w.get("id")),
|
|
140
|
+
"doi": canonical_doi(w.get("doi")) or None,
|
|
141
|
+
"title": w.get("title") or "",
|
|
142
|
+
"year": w.get("publication_year"),
|
|
143
|
+
"journal": src.get("display_name") or "",
|
|
144
|
+
"issn": list(src.get("issn") or []),
|
|
145
|
+
"authors": [((a.get("author") or {}).get("display_name") or "") for a in (w.get("authorships") or [])[:3]],
|
|
146
|
+
"cited_by_count": w.get("cited_by_count") or 0,
|
|
147
|
+
"abstract": _reconstruct_abstract(w.get("abstract_inverted_index")),
|
|
148
|
+
"refs": [_short(r) for r in (w.get("referenced_works") or [])],
|
|
149
|
+
"type": w.get("type") or "",
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _year_filter(year_from: int | None, year_to: int | None) -> list[str]:
|
|
154
|
+
out = []
|
|
155
|
+
if year_from:
|
|
156
|
+
out.append(f"from_publication_date:{year_from}-01-01")
|
|
157
|
+
if year_to:
|
|
158
|
+
out.append(f"to_publication_date:{year_to}-12-31")
|
|
159
|
+
return out
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _parallel(fn, args: list) -> list:
|
|
163
|
+
"""``[fn(a) for a in args]`` with WORKERS threads, results in order and concatenated."""
|
|
164
|
+
if len(args) <= 1:
|
|
165
|
+
return [x for a in args for x in fn(a)]
|
|
166
|
+
with ThreadPoolExecutor(max_workers=WORKERS) as ex:
|
|
167
|
+
return [x for part in ex.map(fn, args) for x in part]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _chunks(values: list[str], n: int = 50) -> list[list[str]]:
|
|
171
|
+
return [values[i:i + n] for i in range(0, len(values), n)]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _by_filter(field: str, values: list[str], extra: list[str] | None = None) -> list[dict[str, Any]]:
|
|
175
|
+
"""Fetch works whose ``field`` is any of ``values`` (50 per request)."""
|
|
176
|
+
def one(chunk: list[str]) -> list[dict[str, Any]]:
|
|
177
|
+
flt = ",".join([f"{field}:" + "|".join(chunk)] + (extra or []))
|
|
178
|
+
return _get({"filter": flt, "per-page": 50, "select": _FIELDS}).get("results") or []
|
|
179
|
+
return [_record(w) for w in _parallel(one, _chunks(values))]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _pages(flt: str, max_records: int, *, split_over: bool = False) -> list[dict[str, Any]] | None:
|
|
183
|
+
"""Up to ``max_records`` works matching ``flt``, most cited first, so a cut drops the least cited.
|
|
184
|
+
|
|
185
|
+
With ``split_over``, return None as soon as the first page shows more than
|
|
186
|
+
``max_records`` matches, so the caller can split the query instead; the
|
|
187
|
+
first page already carries the count, so no separate count request is
|
|
188
|
+
needed (each topic-search request costs 10 OpenAlex credits).
|
|
189
|
+
"""
|
|
190
|
+
# Cursor paging past the first page needs an API key; without one, page numbers
|
|
191
|
+
# (allowed up to 10,000 results) do the same job.
|
|
192
|
+
keyed = bool(str(load_config().get("openalex_api_key") or "").strip())
|
|
193
|
+
out, cursor, page = [], "*", 1
|
|
194
|
+
while len(out) < max_records:
|
|
195
|
+
params = {"filter": flt, "per-page": 200, "select": _FIELDS, "sort": "cited_by_count:desc"}
|
|
196
|
+
params.update({"cursor": cursor} if keyed else {"page": page})
|
|
197
|
+
d = _get(params)
|
|
198
|
+
meta = d.get("meta") or {}
|
|
199
|
+
if split_over and page == 1 and (meta.get("count") or 0) > max_records:
|
|
200
|
+
return None
|
|
201
|
+
rows = d.get("results") or []
|
|
202
|
+
out += rows
|
|
203
|
+
page += 1
|
|
204
|
+
cursor = meta.get("next_cursor")
|
|
205
|
+
if not rows or (keyed and not cursor) or len(out) >= (meta.get("count") or 0):
|
|
206
|
+
break
|
|
207
|
+
return [_record(w) for w in out[:max_records]]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _citing(work_ids: list[str], years: list[str], max_records: int = 400) -> list[dict[str, Any]]:
|
|
211
|
+
return _parallel(lambda chunk: _pages(",".join(["cites:" + "|".join(chunk)] + years), max_records),
|
|
212
|
+
_chunks(work_ids))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _topic_search(term: str, glossary_alts: dict[str, list[str]] | None = None) -> str:
|
|
216
|
+
"""Core term -> OpenAlex search text. Words must all appear; ``a|b`` means either.
|
|
217
|
+
|
|
218
|
+
Papers often write only the abbreviation ("Al/SiC"), so the agent writes
|
|
219
|
+
"Al|aluminum SiC"; British spellings are added here, giving
|
|
220
|
+
"(Al OR aluminum OR aluminium) AND SiC". The user's glossary adds its other
|
|
221
|
+
terms to a word it knows (``glossary_alts``, from ``_glossary_expansions``),
|
|
222
|
+
a phrase in quotes: "(PDC OR \"polymer-derived ceramic\") AND aerogel".
|
|
223
|
+
"""
|
|
224
|
+
from ..terms.spelling import variants
|
|
225
|
+
groups = []
|
|
226
|
+
for w in term.replace(",", " ").split():
|
|
227
|
+
alts = [v for a in w.split("|") if a for v in variants(a)]
|
|
228
|
+
for a in w.split("|"):
|
|
229
|
+
for g in (glossary_alts or {}).get(a.lower(), []):
|
|
230
|
+
alts += [f'"{g}"'] if (" " in g or "-" in g) else variants(g)
|
|
231
|
+
if alts:
|
|
232
|
+
groups.append(list(dict.fromkeys(alts)))
|
|
233
|
+
if all(len(g) == 1 for g in groups):
|
|
234
|
+
return " ".join(g[0] for g in groups)
|
|
235
|
+
return " AND ".join(("(" + " OR ".join(g) + ")") if len(g) > 1 else g[0] for g in groups)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _budget_ok(credits0: int) -> bool:
|
|
239
|
+
"""True while this round may still spend OpenAlex credits (see ROUND_CREDITS, RESERVE_CREDITS)."""
|
|
240
|
+
from .. import quota
|
|
241
|
+
if quota.spent("openalex") - credits0 >= ROUND_CREDITS:
|
|
242
|
+
return False
|
|
243
|
+
remaining = (quota.summary().get("openalex") or {}).get("remaining")
|
|
244
|
+
return remaining is None or remaining > RESERVE_CREDITS
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _cites_refs_on_topic(ref_ids: list[str], core_terms: list[str], years: list[str],
|
|
248
|
+
max_records: int = 1000, credits0: int | None = None,
|
|
249
|
+
glossary_alts: dict[str, list[str]] | None = None,
|
|
250
|
+
) -> tuple[list[dict[str, Any]], set[str]]:
|
|
251
|
+
"""Works citing any of ``ref_ids`` whose title/abstract mentions every core term.
|
|
252
|
+
|
|
253
|
+
Each core term is one concept the paper must mention (``a|b`` = either);
|
|
254
|
+
the terms are joined into one search, so ["lanthanide|europium",
|
|
255
|
+
"drug|doxorubicin"] means rare earth AND drug. (Until 2026-10-16 each term
|
|
256
|
+
was a separate search and a paper matching any one was kept, which let
|
|
257
|
+
27,000 drug-delivery papers without rare earths into one pool.)
|
|
258
|
+
``ref_ids`` come most important first. Returns the works and the references
|
|
259
|
+
actually asked about: once the round's credit budget is spent, the rest
|
|
260
|
+
are left for a later round.
|
|
261
|
+
"""
|
|
262
|
+
from .. import quota
|
|
263
|
+
credits0 = quota.spent("openalex") if credits0 is None else credits0
|
|
264
|
+
term = " ".join(t.strip() for t in core_terms or [] if t.strip())
|
|
265
|
+
jobs = [(term, chunk) for chunk in _chunks(ref_ids)]
|
|
266
|
+
|
|
267
|
+
def flt(term: str, chunk: list[str]) -> str:
|
|
268
|
+
topic = [f"title_and_abstract.search:{_topic_search(term, glossary_alts)}"] if term else []
|
|
269
|
+
return ",".join(["cites:" + "|".join(chunk)] + topic + years)
|
|
270
|
+
|
|
271
|
+
def one(job: tuple[str, list[str]]) -> list[tuple[bool, list[dict[str, Any]]]]:
|
|
272
|
+
term, chunk = job
|
|
273
|
+
single = len(chunk) == 1
|
|
274
|
+
got = _pages(flt(term, chunk), SINGLE_REF_CAP if single else max_records, split_over=not single)
|
|
275
|
+
return [(True, [])] if got is None else [(False, got)]
|
|
276
|
+
|
|
277
|
+
out: list[dict[str, Any]] = []
|
|
278
|
+
asked: set[str] = set()
|
|
279
|
+
_step(0, len(ref_ids))
|
|
280
|
+
try:
|
|
281
|
+
while jobs: # a query over the cap is split in halves; a few jobs at a time, budget permitting
|
|
282
|
+
if not _budget_ok(credits0):
|
|
283
|
+
break
|
|
284
|
+
wave, jobs = jobs[:WORKERS * 2], jobs[WORKERS * 2:]
|
|
285
|
+
again = []
|
|
286
|
+
for (term, chunk), (split, got) in zip(wave, _parallel(one, wave)):
|
|
287
|
+
if split:
|
|
288
|
+
half = len(chunk) // 2
|
|
289
|
+
again += [(term, chunk[:half]), (term, chunk[half:])]
|
|
290
|
+
else:
|
|
291
|
+
out += got
|
|
292
|
+
asked.update(chunk)
|
|
293
|
+
_step(len(asked))
|
|
294
|
+
jobs = again + jobs # finish the halves of important references first
|
|
295
|
+
except BaseException as exc:
|
|
296
|
+
if out or asked:
|
|
297
|
+
exc._partial_expand = (out, asked) # type: ignore[attr-defined]
|
|
298
|
+
raise
|
|
299
|
+
return out, asked
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# --------------------------------------------------------------------------- pool
|
|
303
|
+
|
|
304
|
+
def _keyword_candidates(queries: list[str], year_from: int | None, year_to: int | None) -> dict[str, list[dict]]:
|
|
305
|
+
from .quick import _collect
|
|
306
|
+
|
|
307
|
+
def one(q: str) -> list[dict[str, Any]]:
|
|
308
|
+
try:
|
|
309
|
+
return _collect(q, 100, 100, year_from, year_to, None, None, None)
|
|
310
|
+
except Exception as exc:
|
|
311
|
+
logger.warning("deep search: query %r failed (%s)", q, exc)
|
|
312
|
+
return []
|
|
313
|
+
|
|
314
|
+
with ThreadPoolExecutor(max_workers=4) as ex:
|
|
315
|
+
return dict(zip(queries, ex.map(one, queries)))
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
_FILLER = frozenset("""
|
|
319
|
+
a an and as at by for from in into of on or the their to via with using used study studies
|
|
320
|
+
effect effects new based nano vitro vivo application applications
|
|
321
|
+
""".split())
|
|
322
|
+
|
|
323
|
+
_FUNCTIONS = ("drug", "delivery", "release", "loading", "imaging", "therapy", "antioxidant",
|
|
324
|
+
"cytotoxicity", "bioactivity", "carrier")
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _exact_terms(queries: list[str], core_terms: list[str]) -> list[str]:
|
|
328
|
+
"""Two- and three-word phrases to AND together instead of ranking by overlap.
|
|
329
|
+
|
|
330
|
+
OpenAlex's ``search`` ranks by how many words match, so a long query buries
|
|
331
|
+
papers that use one different word: a query saying "cerium oxide" missed a
|
|
332
|
+
paper titled "cerium dioxide nanoparticles" among the first hundred. ANDing
|
|
333
|
+
two or three words shrinks the result set to a few hundred and puts it back
|
|
334
|
+
in view (2026-11-07: the two rare-earth targets the keyword route never
|
|
335
|
+
found sat at ranks 34 and 3 this way).
|
|
336
|
+
"""
|
|
337
|
+
known = {a.lower() for term in core_terms for w in term.replace(",", " ").split()
|
|
338
|
+
for a in w.split("|") if a}
|
|
339
|
+
out: list[str] = []
|
|
340
|
+
for q in queries:
|
|
341
|
+
words = [w for w in re.split(r"[^0-9a-z\u4e00-\u9fff-]+", q.lower()) if w and w not in _FILLER]
|
|
342
|
+
if not words:
|
|
343
|
+
continue
|
|
344
|
+
out.append(" ".join(words[:3]))
|
|
345
|
+
element = next((w for w in words if w in known), "")
|
|
346
|
+
function = next((w for w in words if w in _FUNCTIONS), "")
|
|
347
|
+
if element and function and element != function:
|
|
348
|
+
out.append(f"{element} nanoparticle {function}")
|
|
349
|
+
seen: list[str] = []
|
|
350
|
+
for t in out:
|
|
351
|
+
if t and t not in seen:
|
|
352
|
+
seen.append(t)
|
|
353
|
+
return seen[:16]
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _exact_candidates(terms: list[str], year_from: int | None, year_to: int | None) -> dict[str, list[dict]]:
|
|
357
|
+
"""Up to 60 most cited works whose title or abstract has every word of ``term``."""
|
|
358
|
+
years = _year_filter(year_from, year_to)
|
|
359
|
+
|
|
360
|
+
def one(term: str) -> list[dict[str, Any]]:
|
|
361
|
+
flt = ",".join([f"title_and_abstract.search:{term}"] + years)
|
|
362
|
+
try:
|
|
363
|
+
return _pages(flt, 60, split_over=False) or []
|
|
364
|
+
except Exception as exc:
|
|
365
|
+
logger.warning("deep search: exact query %r failed (%s)", term, exc)
|
|
366
|
+
return []
|
|
367
|
+
|
|
368
|
+
with ThreadPoolExecutor(max_workers=3) as ex:
|
|
369
|
+
return dict(zip(terms, ex.map(one, terms)))
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _exact_terms_of(rec: dict[str, Any]) -> list[str]:
|
|
373
|
+
"""The and-style phrases that found this record (``_exact_terms`` wrote them into ``via``)."""
|
|
374
|
+
return [v[len("exact: "):] for v in (rec.get("via") or []) if v.startswith("exact: ")]
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _title_cover(rec: dict[str, Any], terms: list[str]) -> float:
|
|
378
|
+
"""Best share of a phrase's words that the title itself carries (0 when it has none)."""
|
|
379
|
+
from ..terms.text import plain, singular
|
|
380
|
+
if not terms:
|
|
381
|
+
return 0.0
|
|
382
|
+
title = {singular(w) for w in plain(rec.get("title") or "").split()}
|
|
383
|
+
best = 0.0
|
|
384
|
+
for term in terms:
|
|
385
|
+
words = [singular(w) for w in plain(term).split() if w]
|
|
386
|
+
if words:
|
|
387
|
+
best = max(best, sum(1 for w in words if w in title) / len(words))
|
|
388
|
+
return best
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _key(rec: dict[str, Any]) -> str:
|
|
392
|
+
return rec.get("doi") or ("oa:" + rec["openalex"] if rec.get("openalex") else "")
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _add(pool: dict[str, dict], rec: dict[str, Any], via: str) -> None:
|
|
396
|
+
doi = rec.get("doi") or ""
|
|
397
|
+
if doi.startswith("10.1002/ange."): # Angewandte's German edition: same article as 10.1002/anie.*
|
|
398
|
+
rec = {**rec, "doi": "10.1002/anie." + doi[len("10.1002/ange."):]}
|
|
399
|
+
k = _key(rec)
|
|
400
|
+
if not k or not rec.get("title") or rec.get("type") in NON_PAPER:
|
|
401
|
+
return
|
|
402
|
+
have = pool.get(k)
|
|
403
|
+
if have is None:
|
|
404
|
+
pool[k] = {**rec, "via": [via]}
|
|
405
|
+
return
|
|
406
|
+
for field in ("abstract", "journal", "openalex", "refs", "year", "issn", "authors"):
|
|
407
|
+
if not have.get(field) and rec.get(field):
|
|
408
|
+
have[field] = rec[field]
|
|
409
|
+
if via != "metadata" and via not in have["via"]: # a metadata lookup is not a way of finding it
|
|
410
|
+
have["via"].append(via)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _score(state: dict[str, Any]) -> None:
|
|
414
|
+
"""Formula score: anchored coupling + QUERY_WEIGHT x fully matched queries.
|
|
415
|
+
|
|
416
|
+
References of the user's own seeds count fully; references that only
|
|
417
|
+
snowballed seeds cite count by how many seeds share them (1/3 each, at most
|
|
418
|
+
1), so a seed set drifting toward a neighbouring topic cannot outvote the
|
|
419
|
+
original intent.
|
|
420
|
+
"""
|
|
421
|
+
from .quick import _match_tier
|
|
422
|
+
pool = state["pool"]
|
|
423
|
+
seed_refs = set(state["seed_refs"])
|
|
424
|
+
orig_refs = {r for s in _orig_seeds(state) for r in (pool.get(s, {}).get("refs") or [])}
|
|
425
|
+
seed_df = Counter(r for s in state["seeds"] for r in set(pool.get(s, {}).get("refs") or []))
|
|
426
|
+
# Matching every query against every record is most of the cost; a record's
|
|
427
|
+
# hits only change with its text or the queries, so only new or changed ones are matched.
|
|
428
|
+
cache = _cache(state)["hits"]
|
|
429
|
+
queries = tuple(state["queries"])
|
|
430
|
+
# A query's glossary variants ("PDC aerogel" -> "polymer-derived ceramic aerogel")
|
|
431
|
+
# count as the query: a paper matching either has matched it once.
|
|
432
|
+
variants = state.get("query_variants") or {}
|
|
433
|
+
sig = (queries, tuple(tuple(variants.get(q) or ()) for q in queries))
|
|
434
|
+
stale = {k: r for k, r in pool.items() if cache.get(k, (None,))[0] != (sig, _text_sig(r))}
|
|
435
|
+
fresh = {k: 0 for k in stale}
|
|
436
|
+
for q in queries:
|
|
437
|
+
rows = [{"title": r["title"], "abstract": r.get("abstract") or "", "_k": k} for k, r in stale.items()]
|
|
438
|
+
hit = set()
|
|
439
|
+
for form in [q, *(variants.get(q) or [])]:
|
|
440
|
+
hit |= {row["_k"] for row in _match_tier(rows, form) if row["_tier"] == 0}
|
|
441
|
+
for k in hit:
|
|
442
|
+
fresh[k] += 1
|
|
443
|
+
for k, n in fresh.items():
|
|
444
|
+
cache[k] = ((sig, _text_sig(stale[k])), n)
|
|
445
|
+
hits = {k: cache[k][1] for k in pool}
|
|
446
|
+
for k, r in pool.items():
|
|
447
|
+
shared = [ref for ref in (r.get("refs") or []) if ref in seed_refs]
|
|
448
|
+
r["coupling"] = len(shared)
|
|
449
|
+
r["anchored"] = round(sum(1.0 if ref in orig_refs else min(1.0, seed_df[ref] / 3) for ref in shared), 2)
|
|
450
|
+
r["query_hits"] = hits[k]
|
|
451
|
+
cover = _title_cover(r, _exact_terms_of(r))
|
|
452
|
+
r["phrase"] = round(cover, 2)
|
|
453
|
+
r["score"] = (r["anchored"] + QUERY_WEIGHT * r["query_hits"]
|
|
454
|
+
+ (EXACT_WEIGHT if cover >= EXACT_MIN_COVER - 1e-9 else 0.0))
|
|
455
|
+
r["rank"] = r["score"]
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
_CACHES: dict[str, dict[str, dict]] = {} # session -> per-record results that depend only on the record's text
|
|
459
|
+
_CACHES_LOCK = threading.Lock()
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _cache(state: dict[str, Any]) -> dict[str, dict]:
|
|
463
|
+
"""In memory only: a restarted server simply recomputes.
|
|
464
|
+
|
|
465
|
+
Query hits (one number per paper) are kept for the latest 4 sessions; the
|
|
466
|
+
token counts (measured 2026-10-18: about 37 MB for 7,309 papers, 270 MB for
|
|
467
|
+
52,668) only for the session in use.
|
|
468
|
+
"""
|
|
469
|
+
session = str(state.get("session") or "")
|
|
470
|
+
with _CACHES_LOCK:
|
|
471
|
+
c = _CACHES.pop(session, None) or {"hits": {}, "tokens": {}}
|
|
472
|
+
for other in _CACHES.values():
|
|
473
|
+
other["tokens"] = {}
|
|
474
|
+
_CACHES[session] = c # most recent last
|
|
475
|
+
while len(_CACHES) > 4:
|
|
476
|
+
_CACHES.pop(next(iter(_CACHES)))
|
|
477
|
+
return c
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
_SESSION_LOCKS: dict[str, threading.Lock] = {}
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _session_lock(session: str) -> threading.Lock:
|
|
484
|
+
"""One round at a time per session (the app's 继续检索 and the agent's next
|
|
485
|
+
could otherwise rank the same pool, and its caches, at once)."""
|
|
486
|
+
with _CACHES_LOCK:
|
|
487
|
+
if session not in _SESSION_LOCKS and len(_SESSION_LOCKS) >= 50:
|
|
488
|
+
for old in [s for s, lk in _SESSION_LOCKS.items() if not lk.locked()]:
|
|
489
|
+
del _SESSION_LOCKS[old] # idle sessions' locks: made again if needed
|
|
490
|
+
return _SESSION_LOCKS.setdefault(session, threading.Lock())
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _text_sig(r: dict[str, Any]) -> int:
|
|
494
|
+
return hash((r.get("title") or "", r.get("abstract") or ""))
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _orig_seeds(state: dict[str, Any]) -> list[str]:
|
|
498
|
+
return state.get("orig_seeds") or state["seeds"][:3]
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
# --------------------------------------------------------------------------- ranking model
|
|
502
|
+
#
|
|
503
|
+
# Continuous active learning, the usual way screening tools for systematic
|
|
504
|
+
# reviews rank: after every batch, train a small logistic regression on all
|
|
505
|
+
# ratings so far (高 and the seeds against 中/低/不相关, plus a random sample of
|
|
506
|
+
# unscreened papers standing in as negatives) and rank the rest by it.
|
|
507
|
+
# Negative ratings pull the ranking back when snowballing drifts. On the
|
|
508
|
+
# SiC/Al set, after the first 90 ratings, the 6 targets still unscreened
|
|
509
|
+
# ranked [18, 39, 68, 70, 98, 1196] by the old formula and [9, 19, 30, 38, 52,
|
|
510
|
+
# 56] by the model (TODO.md, 2026-10-15).
|
|
511
|
+
|
|
512
|
+
_WORD = re.compile(r"[a-z][a-z0-9]+")
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _tokens(rec: dict[str, Any]) -> Counter:
|
|
516
|
+
from ..terms.spelling import to_us
|
|
517
|
+
from ..terms.text import RANKING_STOPWORDS, stem
|
|
518
|
+
words = (rec.get("title") or "") + " " + (rec.get("abstract") or "")
|
|
519
|
+
return Counter(stem(w) for w in _WORD.findall(to_us(words)) if w not in RANKING_STOPWORDS)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _features(state: dict[str, Any]) -> dict[str, dict[str, float]]:
|
|
523
|
+
pool = state["pool"]
|
|
524
|
+
cache = _cache(state)["tokens"]
|
|
525
|
+
docs, titles = {}, {}
|
|
526
|
+
for k, r in pool.items():
|
|
527
|
+
sig = _text_sig(r)
|
|
528
|
+
hit = cache.get(k)
|
|
529
|
+
if hit is None or hit[0] != sig:
|
|
530
|
+
hit = cache[k] = (sig, _tokens(r), list(_tokens({"title": r.get("title")})))
|
|
531
|
+
docs[k], titles[k] = hit[1], hit[2]
|
|
532
|
+
df = Counter(w for c in docs.values() for w in c)
|
|
533
|
+
n, nq = len(pool), max(1, len(state["queries"]))
|
|
534
|
+
out = {}
|
|
535
|
+
for k, c in docs.items():
|
|
536
|
+
v = {w: (1 + math.log(m)) * math.log(n / df[w]) for w, m in c.items() if df[w] >= 2}
|
|
537
|
+
norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
|
|
538
|
+
v = {w: x / norm for w, x in v.items()}
|
|
539
|
+
r = pool[k]
|
|
540
|
+
# Title words again, on their own: a title says what the paper is about
|
|
541
|
+
# better than the abstract, and is all there is when the abstract is missing.
|
|
542
|
+
v.update({"t:" + w: TITLE_WEIGHT for w in titles[k]})
|
|
543
|
+
v["_coupling"] = math.log1p(r.get("anchored", 0)) / 3
|
|
544
|
+
v["_queries"] = r.get("query_hits", 0) / nq
|
|
545
|
+
v["_no_abstract"] = 0.0 if r.get("abstract") else 1.0
|
|
546
|
+
out[k] = v
|
|
547
|
+
return out
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _train(examples: list[tuple[dict[str, float], int]], epochs: int = 60, lr: float = 0.5,
|
|
551
|
+
l2: float = 1e-3) -> Counter:
|
|
552
|
+
w: Counter = Counter()
|
|
553
|
+
for _ in range(epochs):
|
|
554
|
+
for v, y in examples:
|
|
555
|
+
z = sum(w[f] * x for f, x in v.items()) + w["_bias"]
|
|
556
|
+
g = 1 / (1 + math.exp(-max(-30.0, min(30.0, z)))) - y
|
|
557
|
+
for f, x in v.items():
|
|
558
|
+
w[f] -= lr * (g * x + l2 * w[f])
|
|
559
|
+
w["_bias"] -= lr * g
|
|
560
|
+
return w
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def _rerank(state: dict[str, Any], rated: dict[str, str]) -> str:
|
|
564
|
+
"""Set every record's ``rank``; returns "model" or "formula"."""
|
|
565
|
+
pool, seeds = state["pool"], set(state["seeds"])
|
|
566
|
+
pos = {k for k, r in rated.items() if r == "高" and k in pool} | {k for k in seeds if k in pool}
|
|
567
|
+
neg = [k for k, r in rated.items() if r != "高" and k in pool]
|
|
568
|
+
if len(pos) < MIN_LABELS[0] or len(neg) < MIN_LABELS[1]:
|
|
569
|
+
for r in pool.values():
|
|
570
|
+
r["rank"] = r.get("score", 0)
|
|
571
|
+
return "formula"
|
|
572
|
+
rng = random.Random(len(rated)) # same ratings, same ranking
|
|
573
|
+
unscreened = sorted(k for k in pool if k not in rated and k not in seeds)
|
|
574
|
+
pseudo = rng.sample(unscreened, min(PSEUDO_NEGATIVES, len(unscreened)))
|
|
575
|
+
feats = _features(state)
|
|
576
|
+
examples = [(feats[k], 1) for k in sorted(pos)] + [(feats[k], 0) for k in neg + pseudo]
|
|
577
|
+
rng.shuffle(examples)
|
|
578
|
+
w = _train(examples)
|
|
579
|
+
for k, r in pool.items():
|
|
580
|
+
r["rank"] = round(sum(w[f] * x for f, x in feats[k].items()) + w["_bias"], 4)
|
|
581
|
+
return "model"
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _prefill_abstracts(state: dict[str, Any], rated: dict[str, str]) -> int:
|
|
585
|
+
"""Fetch abstracts for the top unscreened candidates that lack one, once each.
|
|
586
|
+
|
|
587
|
+
Ranking reads abstracts, so a paper without one used to sink and never
|
|
588
|
+
reach a batch, which is where its abstract would have been fetched.
|
|
589
|
+
"""
|
|
590
|
+
return _prefill_fetch(_prefill_pick(state, rated))
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _prefill_pick(state: dict[str, Any], rated: dict[str, str]) -> list[dict[str, Any]]:
|
|
594
|
+
"""The PREFILL best-ranked unscreened records that still lack an abstract."""
|
|
595
|
+
# Pick among the papers that lack one: they rank low for that very reason, so
|
|
596
|
+
# taking the overall top PREFILL would reach almost none of them.
|
|
597
|
+
todo = [r for k in _ranked(state) if k not in rated
|
|
598
|
+
for r in [state["pool"][k]] if not r.get("abstract") and not r.get("abstract_tried") and r.get("doi")]
|
|
599
|
+
return todo[:PREFILL]
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _prefill_fetch(todo: list[dict[str, Any]]) -> int:
|
|
603
|
+
"""Fill abstracts into these records in place; returns how many were found."""
|
|
604
|
+
from .abstracts import fill_abstracts
|
|
605
|
+
if not todo:
|
|
606
|
+
return 0
|
|
607
|
+
_phase("abstracts", len(todo))
|
|
608
|
+
# Scopus answers about 3 papers a second at its allowed pace: budget for that.
|
|
609
|
+
fill_abstracts(todo, load_config(), budget_s=min(90.0, max(30.0, len(todo) / 3)), progress=_step)
|
|
610
|
+
for r in todo:
|
|
611
|
+
if r.get("abstract") or r.pop("abstract_decided", False):
|
|
612
|
+
r["abstract_tried"] = True # undecided ones (timeout, throttled) are asked again next round
|
|
613
|
+
return sum(1 for r in todo if r.get("abstract"))
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _pool_path(session: str):
|
|
617
|
+
return screening.screening_dir() / f"{session}.pool.json"
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def has_pool(session: str) -> bool:
|
|
621
|
+
"""Return True if a deep-search candidate pool exists for the session."""
|
|
622
|
+
return db.has_pool(session)
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def _load(session: str) -> dict[str, Any]:
|
|
626
|
+
return db.load_pool_state(session)
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def _save(state: dict[str, Any]) -> None:
|
|
630
|
+
db.save_pool_state(state)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _expand(state: dict[str, Any], new_seed_keys: list[str]) -> int:
|
|
634
|
+
"""Add citation-route candidates for new seeds; returns how many records were added."""
|
|
635
|
+
pool = state["pool"]
|
|
636
|
+
years = _year_filter(state.get("year_from"), state.get("year_to"))
|
|
637
|
+
from .. import quota
|
|
638
|
+
credits0 = quota.spent("openalex")
|
|
639
|
+
seeds = [pool[k] for k in new_seed_keys if k in pool]
|
|
640
|
+
# References shared by more seeds (the user's own seeds count double) are asked first,
|
|
641
|
+
# so a spent budget drops the least informative ones; they stay for a later round.
|
|
642
|
+
orig = set(_orig_seeds(state))
|
|
643
|
+
weight = Counter(r for k in set(state["seeds"]) | set(new_seed_keys) if k in pool
|
|
644
|
+
for r in set(pool[k].get("refs") or []) for _ in range(2 if k in orig else 1))
|
|
645
|
+
all_seeds = seeds + [pool[k] for k in state["seeds"] if k in pool]
|
|
646
|
+
new_refs = sorted({r for s in all_seeds for r in (s.get("refs") or [])} - set(state["queried_refs"]),
|
|
647
|
+
key=lambda r: (-weight[r], r))
|
|
648
|
+
before = len(pool)
|
|
649
|
+
asked: set[str] = set()
|
|
650
|
+
_phase("expand", len(new_refs))
|
|
651
|
+
try:
|
|
652
|
+
if new_refs:
|
|
653
|
+
for rec in _by_filter("openalex", new_refs, years):
|
|
654
|
+
_add(pool, rec, "seed_reference")
|
|
655
|
+
got, asked = _cites_refs_on_topic(new_refs, state["core_terms"], years, credits0=credits0,
|
|
656
|
+
glossary_alts=state.get("glossary_alts"))
|
|
657
|
+
for rec in got:
|
|
658
|
+
_add(pool, rec, "cites_seed_references")
|
|
659
|
+
seed_ids = [s["openalex"] for s in seeds if s.get("openalex")]
|
|
660
|
+
if seed_ids and _budget_ok(credits0):
|
|
661
|
+
for rec in _citing(seed_ids, years):
|
|
662
|
+
_add(pool, rec, "cites_seed")
|
|
663
|
+
except BaseException as exc:
|
|
664
|
+
partial = getattr(exc, "_partial_expand", None)
|
|
665
|
+
if partial:
|
|
666
|
+
partial_got, partial_asked = partial
|
|
667
|
+
for rec in partial_got:
|
|
668
|
+
_add(pool, rec, "cites_seed_references")
|
|
669
|
+
asked |= partial_asked
|
|
670
|
+
state["deferred_refs"] = len(new_refs) - len(asked)
|
|
671
|
+
state["queried_refs"] = sorted(set(state["queried_refs"]) | asked)
|
|
672
|
+
state["seed_refs"] = sorted(set(state["seed_refs"]) | {r for s in seeds for r in (s.get("refs") or [])})
|
|
673
|
+
if state.get("session"):
|
|
674
|
+
_save(state)
|
|
675
|
+
raise
|
|
676
|
+
deferred = len(new_refs) - len(asked)
|
|
677
|
+
state["deferred_refs"] = deferred
|
|
678
|
+
if deferred:
|
|
679
|
+
logger.info("deep search: OpenAlex budget for this round used; %d references left for later", deferred)
|
|
680
|
+
state["queried_refs"] = sorted(set(state["queried_refs"]) | asked)
|
|
681
|
+
state["seed_refs"] = sorted(set(state["seed_refs"]) | {r for s in seeds for r in (s.get("refs") or [])})
|
|
682
|
+
for k in new_seed_keys:
|
|
683
|
+
if k not in state["seeds"]:
|
|
684
|
+
state["seeds"].append(k)
|
|
685
|
+
return len(pool) - before
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
# --------------------------------------------------------------------------- public
|
|
689
|
+
|
|
690
|
+
def _core_term_notes(core_terms: list[str]) -> list[str]:
|
|
691
|
+
"""Warnings about core terms that would make the citation route far too broad."""
|
|
692
|
+
from ..terms.glossary import AMBIGUOUS_SYMBOLS
|
|
693
|
+
alts = {a.lower() for term in core_terms for w in term.replace(",", " ").split() for a in w.split("|") if a}
|
|
694
|
+
loose = sorted(alts & AMBIGUOUS_SYMBOLS)
|
|
695
|
+
if not loose:
|
|
696
|
+
return []
|
|
697
|
+
return ["core_terms contain element symbols that are also common words (" + ", ".join(loose)
|
|
698
|
+
+ "; search ignores case): write the element's full name instead"]
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def start(question: str, queries: list[str], seeds: list[str], core_terms: list[str], *,
|
|
702
|
+
criteria: list[str] | None = None, must_find: list[str] | None = None,
|
|
703
|
+
year_from: int | None = None, year_to: int | None = None,
|
|
704
|
+
batch_size: int = 20, cap: int = 200) -> dict[str, Any]:
|
|
705
|
+
"""Build the pool and return the first batch to screen."""
|
|
706
|
+
if not queries:
|
|
707
|
+
raise ValueError("give 4-8 English queries")
|
|
708
|
+
if not seeds:
|
|
709
|
+
raise ValueError("give 1-3 seed DOIs (papers the user already knows are on topic)")
|
|
710
|
+
t0 = time.time()
|
|
711
|
+
session = screening.save(question, [], criteria=criteria, query=" | ".join(queries), mode="deep",
|
|
712
|
+
search_params={"core_terms": core_terms, "seeds": seeds,
|
|
713
|
+
"year_from": year_from, "year_to": year_to})["session"]
|
|
714
|
+
_run.progress = store.Progress(session, "start")
|
|
715
|
+
try:
|
|
716
|
+
return _start(session, t0, question, queries, seeds, core_terms, must_find, year_from, year_to,
|
|
717
|
+
batch_size, cap)
|
|
718
|
+
except BaseException as exc:
|
|
719
|
+
_run.progress.finish("failed", error=str(exc) or type(exc).__name__)
|
|
720
|
+
raise
|
|
721
|
+
finally:
|
|
722
|
+
_run.progress = None
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def _start(session: str, t0: float, question: str, queries: list[str], seeds: list[str],
|
|
726
|
+
core_terms: list[str], must_find: list[str] | None, year_from: int | None, year_to: int | None,
|
|
727
|
+
batch_size: int, cap: int) -> dict[str, Any]:
|
|
728
|
+
query_variants, glossary_alts, glossary_used = _glossary_expansions(queries, core_terms)
|
|
729
|
+
all_queries = queries + [v for q in queries for v in query_variants.get(q, [])]
|
|
730
|
+
years = _year_filter(year_from, year_to)
|
|
731
|
+
pool: dict[str, dict] = {}
|
|
732
|
+
seed_recs = _by_filter("doi", [canonical_doi(s) for s in seeds])
|
|
733
|
+
for rec in seed_recs:
|
|
734
|
+
_add(pool, rec, "seed")
|
|
735
|
+
# Off by default: measured 2026-11-07 on the rare-earth set, it puts 8 of the 9 targets
|
|
736
|
+
# into the pool but the ranking buries them again (the read window found 1 instead of 2).
|
|
737
|
+
# See docs/DECISIONS.md; keep it for experiments until the scorer can tell the hits apart.
|
|
738
|
+
exact_terms = _exact_terms(queries, core_terms) if load_config().get("deep_exact_queries") else []
|
|
739
|
+
_phase("keywords", len(all_queries) + len(exact_terms))
|
|
740
|
+
if exact_terms:
|
|
741
|
+
for term, rows in _exact_candidates(exact_terms, year_from, year_to).items():
|
|
742
|
+
for rec in rows:
|
|
743
|
+
_add(pool, rec, f"exact: {term}")
|
|
744
|
+
for q, rows in _keyword_candidates(all_queries, year_from, year_to).items():
|
|
745
|
+
for r in rows:
|
|
746
|
+
_add(pool, {"openalex": None, "doi": canonical_doi(r.get("doi")) or None,
|
|
747
|
+
"title": r.get("title") or "", "year": r.get("year"), "journal": r.get("journal") or "",
|
|
748
|
+
"issn": r.get("issn") or [], "authors": r.get("authors") or [],
|
|
749
|
+
"cited_by_count": r.get("cited_by_count") or 0, "abstract": r.get("abstract") or "",
|
|
750
|
+
"refs": None}, f"query: {q}")
|
|
751
|
+
# Keyword hits come without reference lists; fetch them so coupling can count.
|
|
752
|
+
missing = [k for k, r in pool.items() if r.get("refs") is None and r.get("doi")]
|
|
753
|
+
for rec in _by_filter("doi", missing):
|
|
754
|
+
_add(pool, rec, "metadata")
|
|
755
|
+
state = {"session": session, "question": question, "queries": queries, "core_terms": core_terms,
|
|
756
|
+
"orig_seeds": [_key(r) for r in seed_recs],
|
|
757
|
+
"year_from": year_from, "year_to": year_to, "batch_size": batch_size, "cap": cap,
|
|
758
|
+
"must_find": [canonical_doi(d) for d in (must_find or [])], "seeds": [], "seed_refs": [],
|
|
759
|
+
"queried_refs": [], "served": [], "rounds": [], "pool": pool,
|
|
760
|
+
"query_variants": query_variants, "glossary_alts": glossary_alts, "glossary_used": glossary_used}
|
|
761
|
+
_save(state)
|
|
762
|
+
_expand(state, [_key(r) for r in seed_recs])
|
|
763
|
+
_score(state)
|
|
764
|
+
state["rounds"].append({"time": time.strftime("%Y-%m-%d %H:%M:%S"), "event": "start",
|
|
765
|
+
"pool": len(pool), "seconds": round(time.time() - t0, 1)})
|
|
766
|
+
_save(state)
|
|
767
|
+
out = _next_batch(state)
|
|
768
|
+
notes = _core_term_notes(core_terms)
|
|
769
|
+
if notes:
|
|
770
|
+
out["core_term_notes"] = notes
|
|
771
|
+
if glossary_used:
|
|
772
|
+
out["glossary"] = glossary_used
|
|
773
|
+
_run.progress.finish()
|
|
774
|
+
return out
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _glossary_expansions(queries: list[str], core_terms: list[str],
|
|
778
|
+
) -> tuple[dict[str, list[str]], dict[str, list[str]], list[dict[str, Any]]]:
|
|
779
|
+
"""What the glossary (``glossary.py``: the user's entries and the built-in
|
|
780
|
+
elements) adds to this search: other wordings of each query, other terms for
|
|
781
|
+
the words of the core terms, and a report of both for the user. Core-term
|
|
782
|
+
words are expanded only when there are at least two of them, so another
|
|
783
|
+
concept keeps each expansion on topic."""
|
|
784
|
+
from ..terms import glossary
|
|
785
|
+
try:
|
|
786
|
+
use = glossary.in_use()
|
|
787
|
+
except Exception as exc: # an unreadable glossary must not stop the search
|
|
788
|
+
logger.warning("deep search: glossary not read (%s)", exc)
|
|
789
|
+
return {}, {}, []
|
|
790
|
+
variants: dict[str, list[str]] = {}
|
|
791
|
+
used: list[dict[str, Any]] = []
|
|
792
|
+
for q in queries:
|
|
793
|
+
v, u = glossary.query_variants(q, use)
|
|
794
|
+
if v:
|
|
795
|
+
variants[q] = v
|
|
796
|
+
used += u
|
|
797
|
+
alts: dict[str, list[str]] = {}
|
|
798
|
+
words = [a for term in core_terms for w in term.replace(",", " ").split() for a in w.split("|") if a]
|
|
799
|
+
if len(words) >= 2:
|
|
800
|
+
for a in words:
|
|
801
|
+
more, kinds = glossary.alternatives(a, use)
|
|
802
|
+
if more and a.lower() not in alts:
|
|
803
|
+
alts[a.lower()] = more
|
|
804
|
+
used.append({"term": a, "with": more, "core_term": True, **({"builtin": kinds[0]} if kinds else {})})
|
|
805
|
+
return variants, alts, used
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _glossary_suggestions(state: dict[str, Any], rated: dict[str, str]) -> list[dict[str, Any]]:
|
|
809
|
+
"""Abbreviations the papers read so far define ("polymer-derived ceramics
|
|
810
|
+
(PDCs)"), added to the glossary as suggestions for the user to confirm."""
|
|
811
|
+
from ..terms import glossary
|
|
812
|
+
pool = state["pool"]
|
|
813
|
+
keys = set(state["seeds"]) | {k for k, r in rated.items() if r in ("高", "中")}
|
|
814
|
+
texts = [pool[k].get("abstract") or "" for k in keys if k in pool]
|
|
815
|
+
try:
|
|
816
|
+
return [{k: e.get(k) for k in ("id", "terms", "note")} for e in glossary.suggest_from(texts)]
|
|
817
|
+
except Exception as exc: # a suggestion is a convenience; the search goes on without it
|
|
818
|
+
logger.warning("deep search: no glossary suggestions (%s)", exc)
|
|
819
|
+
return []
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def next_batch(session: str, force: bool = False) -> dict[str, Any]:
|
|
823
|
+
"""Fold the latest ratings back in (高 papers become seeds) and serve the next batch.
|
|
824
|
+
|
|
825
|
+
force: serve a batch even when stopping is suggested (the user chose to go on;
|
|
826
|
+
the app's 继续检索 button).
|
|
827
|
+
"""
|
|
828
|
+
_run.progress = store.Progress(session, "next")
|
|
829
|
+
try:
|
|
830
|
+
with _session_lock(session):
|
|
831
|
+
out = _next_round(session, force)
|
|
832
|
+
except BaseException as exc:
|
|
833
|
+
_run.progress.finish("failed", error=str(exc) or type(exc).__name__)
|
|
834
|
+
raise
|
|
835
|
+
finally:
|
|
836
|
+
p, _run.progress = _run.progress, None
|
|
837
|
+
p.finish()
|
|
838
|
+
return out
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def _next_round(session: str, force: bool = False) -> dict[str, Any]:
|
|
842
|
+
from .. import quota
|
|
843
|
+
t0 = time.time()
|
|
844
|
+
credits0 = quota.spent("openalex")
|
|
845
|
+
state = _load(session)
|
|
846
|
+
rated = _ratings(state)
|
|
847
|
+
orig_missing = [k for k in _orig_seeds(state) if k in state["pool"] and k not in state["seeds"]]
|
|
848
|
+
new_seeds = list(dict.fromkeys(orig_missing + [
|
|
849
|
+
k for k, rel in rated.items() if rel == "高" and k in state["pool"] and k not in state["seeds"]
|
|
850
|
+
]))
|
|
851
|
+
# Abstracts for the best-ranked papers without one, fetched while the pool is
|
|
852
|
+
# expanded (2026-10-20: the two ran one after the other, 12-25 s + ~50 s a
|
|
853
|
+
# round). Picked from last round's ranking, so papers this expansion adds
|
|
854
|
+
# get theirs next round.
|
|
855
|
+
todo = _prefill_pick(state, rated) if force or not _stop(state, rated)["stop_suggested"] else []
|
|
856
|
+
added, found = 0, 0
|
|
857
|
+
with ThreadPoolExecutor(max_workers=1) as ex:
|
|
858
|
+
filling = ex.submit(_prefill_fetch, todo) if todo else None
|
|
859
|
+
if new_seeds or state.get("deferred_refs"):
|
|
860
|
+
added = _expand(state, new_seeds)
|
|
861
|
+
t_exp = time.time()
|
|
862
|
+
if filling is not None:
|
|
863
|
+
if not filling.done():
|
|
864
|
+
_phase("abstracts", len(todo))
|
|
865
|
+
found = filling.result()
|
|
866
|
+
if added or found or not state.get("rounds"):
|
|
867
|
+
_score(state)
|
|
868
|
+
t1 = time.time()
|
|
869
|
+
rnd = {"time": time.strftime("%Y-%m-%d %H:%M:%S"), "event": "next", "new_seeds": len(new_seeds),
|
|
870
|
+
"pool_added": added, "expand_seconds": round(t_exp - t0, 1),
|
|
871
|
+
"abstracts_asked": len(todo), "abstracts_found": found,
|
|
872
|
+
"abstracts_wait_seconds": round(t1 - t_exp, 1)} # beyond the expansion
|
|
873
|
+
state["rounds"].append(rnd)
|
|
874
|
+
_save(state)
|
|
875
|
+
out = _next_batch(state, rated, force, prefilled=True)
|
|
876
|
+
suggested = _glossary_suggestions(state, rated)
|
|
877
|
+
if suggested:
|
|
878
|
+
out["glossary_suggestions"] = suggested
|
|
879
|
+
rnd["rank_seconds"] = round(time.time() - t1, 1) # abstracts, model, journal ranks
|
|
880
|
+
rnd["openalex_credits"] = quota.spent("openalex") - credits0
|
|
881
|
+
rnd["api_budget"] = {k: {f: v.get(f) for f in ("remaining", "limit", "resets_at")}
|
|
882
|
+
for k, v in quota.summary().items()}
|
|
883
|
+
_save(state)
|
|
884
|
+
return out
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def rate(session: str, ratings: list[dict[str, Any]]) -> dict[str, Any]:
|
|
888
|
+
"""Save the agent's ratings for the latest batch by number: [{"n": 3, "relevance": "高", "reason": "..."}].
|
|
889
|
+
|
|
890
|
+
Title, DOI, year and journal come from the batch, so the agent writes only
|
|
891
|
+
its judgement. Nothing is saved if any number or relevance is invalid.
|
|
892
|
+
"""
|
|
893
|
+
state = _load(session)
|
|
894
|
+
batch = state.get("last_batch") or []
|
|
895
|
+
if not batch:
|
|
896
|
+
raise ValueError("no batch to rate yet; call action='start' or 'next' first")
|
|
897
|
+
items, bad = [], []
|
|
898
|
+
for r in ratings or []:
|
|
899
|
+
try:
|
|
900
|
+
n = int(r.get("n"))
|
|
901
|
+
except (TypeError, ValueError):
|
|
902
|
+
n = 0
|
|
903
|
+
rel = str(r.get("relevance") or "").strip()
|
|
904
|
+
if not 1 <= n <= len(batch) or rel not in screening.RELEVANCE:
|
|
905
|
+
bad.append(r)
|
|
906
|
+
continue
|
|
907
|
+
b = batch[n - 1]
|
|
908
|
+
items.append({**{f: b.get(f) for f in _BATCH_FIELDS},
|
|
909
|
+
"relevance": rel, "reason": str(r.get("reason") or rel), "abstract_zh": r.get("abstract_zh")})
|
|
910
|
+
if bad:
|
|
911
|
+
raise ValueError(f"nothing saved: invalid entries {bad[:3]}; n is 1-{len(batch)}, "
|
|
912
|
+
f"relevance one of {'/'.join(screening.RELEVANCE)}")
|
|
913
|
+
saved = screening.save("", items, session=session, mode="deep")
|
|
914
|
+
done = {int(r["n"]) for r in ratings}
|
|
915
|
+
out = {"session": session, "saved": len(items), "total": saved["total"], "by_relevance": saved["by_relevance"],
|
|
916
|
+
"not_rated": [i for i in range(1, len(batch) + 1) if i not in done],
|
|
917
|
+
"next_step": "action='next'" if len(done) == len(batch) else "rate the papers in not_rated, then action='next'"}
|
|
918
|
+
left = screening.untranslated(session)
|
|
919
|
+
if left: # the app shows these abstracts in Chinese
|
|
920
|
+
out["untranslated"] = left
|
|
921
|
+
out["next_step"] = ("translate their abstracts into Chinese: litloop_screening(action='translate', session=..., "
|
|
922
|
+
"items=[{doi, abstract_zh}]); then " + out["next_step"])
|
|
923
|
+
return out
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def status(session: str) -> dict[str, Any]:
|
|
927
|
+
"""Counts, coverage, the stop rule, and the latest batch's papers nobody has rated:
|
|
928
|
+
the app can serve a batch itself (继续检索), so the agent may come back to one."""
|
|
929
|
+
state = _load(session)
|
|
930
|
+
rated = _ratings(state)
|
|
931
|
+
order = _ranked(state)
|
|
932
|
+
agent = _agent_rated(state)
|
|
933
|
+
unrated = [{"n": n, **{f: b.get(f) for f in _BATCH_FIELDS}}
|
|
934
|
+
for n, b in enumerate(state.get("last_batch") or [], 1) if _batch_key(b) not in agent]
|
|
935
|
+
stop = _stop(state, rated)
|
|
936
|
+
remaining = len([k for k in order if k not in rated])
|
|
937
|
+
batches = len([b for b in state.get("served") or [] if b])
|
|
938
|
+
deferred_refs = int(state.get("deferred_refs") or 0)
|
|
939
|
+
prog = store.read_progress(session)
|
|
940
|
+
progress_info: dict[str, Any] | None = None
|
|
941
|
+
if prog:
|
|
942
|
+
progress_info = {}
|
|
943
|
+
for k in ("event", "state", "phase", "stopped_phase", "error", "updated"):
|
|
944
|
+
v = prog.get(k)
|
|
945
|
+
if k == "phase" and isinstance(v, dict):
|
|
946
|
+
v = v.get("name")
|
|
947
|
+
if v is not None:
|
|
948
|
+
progress_info[k] = v
|
|
949
|
+
resumable = bool(
|
|
950
|
+
unrated
|
|
951
|
+
or (not stop["stop_suggested"] and (remaining > 0 or deferred_refs > 0 or not state.get("served")))
|
|
952
|
+
or (prog and prog.get("state") in ("failed", "interrupted"))
|
|
953
|
+
)
|
|
954
|
+
out = {"session": session, "pool": len(state["pool"]), "screened": len(rated),
|
|
955
|
+
"batches": batches, "remaining": remaining, "deferred_refs": deferred_refs,
|
|
956
|
+
"unrated_count": len(unrated), "progress": progress_info, "resumable": resumable,
|
|
957
|
+
"by_relevance": {r: sum(1 for v in rated.values() if v == r) for r in screening.RELEVANCE},
|
|
958
|
+
"seeds": len(state["seeds"]), "coverage": _coverage(state, rated, order), **stop}
|
|
959
|
+
if unrated:
|
|
960
|
+
out["unrated"] = unrated
|
|
961
|
+
out["next_step"] = ("rate these papers of the latest batch by their n (action='rate'; "
|
|
962
|
+
"a rating the user already gave in the app stays), then action='next'")
|
|
963
|
+
elif resumable:
|
|
964
|
+
out["next_step"] = "resume from where this session stopped: call action='next'"
|
|
965
|
+
return out
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _agent_rated(state: dict[str, Any]) -> set[str]:
|
|
969
|
+
"""Screening keys of the papers the agent itself rated (the app's own ratings have no reason)."""
|
|
970
|
+
return {screening._key(it) for it in screening.load(state["session"])["items"] if it.get("reason")}
|
|
971
|
+
|
|
972
|
+
|
|
973
|
+
def _batch_key(b: dict[str, Any]) -> str:
|
|
974
|
+
return screening._key({"doi": b.get("doi"), "title": b.get("title")})
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
def batch_paper(session: str, doi: str | None, title: str | None) -> dict[str, Any] | None:
|
|
978
|
+
"""A paper of the latest batch, as the batch showed it (for a rating the user gives first)."""
|
|
979
|
+
return db.get_batch_paper(session, doi, title)
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
# --------------------------------------------------------------------------- helpers
|
|
983
|
+
|
|
984
|
+
def _ratings(state: dict[str, Any]) -> dict[str, str]:
|
|
985
|
+
"""Ratings keyed like the pool. Papers without a DOI are matched back by title, the way screening keys them."""
|
|
986
|
+
by_title = {screening._key({"title": r.get("title")}): k for k, r in state["pool"].items() if not r.get("doi")}
|
|
987
|
+
out = {}
|
|
988
|
+
for it in screening.load(state["session"])["items"]:
|
|
989
|
+
key = screening._key(it)
|
|
990
|
+
key = by_title.get(key) if key.startswith("title:") else key
|
|
991
|
+
if key:
|
|
992
|
+
out[key] = screening._rating(it) # the user's own rating (from the app) wins
|
|
993
|
+
return out
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
def _ranked(state: dict[str, Any]) -> list[str]:
|
|
997
|
+
seeds = set(state["seeds"])
|
|
998
|
+
pool = state["pool"]
|
|
999
|
+
return sorted((k for k in pool if k not in seeds),
|
|
1000
|
+
key=lambda k: (-pool[k].get("rank", pool[k].get("score", 0)), -(pool[k].get("cited_by_count") or 0)))
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def _remaining_estimate(state: dict[str, Any], rated: dict[str, str]) -> dict[str, Any] | None:
|
|
1004
|
+
"""Roughly how many 高 papers are still unread, from the yield curve.
|
|
1005
|
+
|
|
1006
|
+
The model's scores are not calibrated probabilities (it is trained against a
|
|
1007
|
+
random sample of unread papers), so the estimate comes from what the ratings
|
|
1008
|
+
show instead: fit a geometric decay to the share of 高 per batch (least
|
|
1009
|
+
squares on log share, recent batches weighted more) and sum the batches
|
|
1010
|
+
still ahead, as far as the pool goes. Needs at least three rated batches;
|
|
1011
|
+
a flat or rising curve has no finite tail and gives no total.
|
|
1012
|
+
"""
|
|
1013
|
+
batches = [b for b in state["served"] if b]
|
|
1014
|
+
rates, sizes = [], []
|
|
1015
|
+
for b in batches:
|
|
1016
|
+
judged = [k for k in b if k in rated]
|
|
1017
|
+
if len(judged) < max(3, len(b) // 2):
|
|
1018
|
+
continue
|
|
1019
|
+
rates.append(sum(1 for k in judged if rated[k] == "高") / len(judged))
|
|
1020
|
+
sizes.append(len(judged))
|
|
1021
|
+
if len(rates) < 3:
|
|
1022
|
+
return None
|
|
1023
|
+
size = state.get("batch_size") or len(batches[-1])
|
|
1024
|
+
xs = list(range(len(rates)))
|
|
1025
|
+
ys = [math.log(r + 0.02) for r in rates] # 0.02: a batch without 高 is not log(0)
|
|
1026
|
+
ws = [1 + i for i in xs]
|
|
1027
|
+
sw = sum(ws)
|
|
1028
|
+
mx = sum(w * x for w, x in zip(ws, xs)) / sw
|
|
1029
|
+
my = sum(w * y for w, y in zip(ws, ys)) / sw
|
|
1030
|
+
var = sum(w * (x - mx) ** 2 for w, x in zip(ws, xs))
|
|
1031
|
+
slope = sum(w * (x - mx) * (y - my) for w, x, y in zip(ws, xs, ys)) / var if var else 0.0
|
|
1032
|
+
q = math.exp(slope)
|
|
1033
|
+
last = max(0.0, math.exp(my + slope * (xs[-1] - mx)) - 0.02)
|
|
1034
|
+
seeds = set(state.get("seeds") or [])
|
|
1035
|
+
unread = sum(1 for k in state.get("pool") or {} if k not in rated and k not in seeds)
|
|
1036
|
+
out = {"next_batch": round(last * q * size, 1), "batches": len(rates), "decay": round(q, 2)}
|
|
1037
|
+
if q < 0.95 and unread:
|
|
1038
|
+
ahead = min(unread // max(1, size), 50)
|
|
1039
|
+
out["remaining"] = round(sum(last * q ** j * size for j in range(1, ahead + 1)), 1)
|
|
1040
|
+
return out
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def _stop(state: dict[str, Any], rated: dict[str, str]) -> dict[str, Any]:
|
|
1044
|
+
"""Suggest stopping at the cap, or once the yield of 高 has fallen off.
|
|
1045
|
+
|
|
1046
|
+
Knee rule, as in screening for systematic reviews: when the share of 高 in
|
|
1047
|
+
the latest batch is below 1/KNEE_RATIO of the best earlier batch, the
|
|
1048
|
+
relevant papers have mostly been read. Counting 中 too never fired: the
|
|
1049
|
+
neighbouring topics that come up late keep it high.
|
|
1050
|
+
"""
|
|
1051
|
+
est = _remaining_estimate(state, rated)
|
|
1052
|
+
if len(rated) >= state["cap"]:
|
|
1053
|
+
return _with_estimate({"stop_suggested": True, "stop_rule": "cap",
|
|
1054
|
+
"stop_reason": f"已评分 {len(rated)} 篇,达到上限 {state['cap']} 篇"}, est)
|
|
1055
|
+
batches = [b for b in state["served"] if b]
|
|
1056
|
+
last = batches[-1] if batches else []
|
|
1057
|
+
judged = [rated[k] for k in last if k in rated]
|
|
1058
|
+
if len(batches) < 2 or not judged or len(judged) < len(last) // 2:
|
|
1059
|
+
return _with_estimate({"stop_suggested": False, "stop_reason": ""}, est)
|
|
1060
|
+
high = sum(1 for r in judged if r == "高")
|
|
1061
|
+
rates = [sum(1 for k in b if rated.get(k) == "高") / max(1, sum(1 for k in b if k in rated)) for b in batches[:-1]]
|
|
1062
|
+
best = max(rates)
|
|
1063
|
+
if best > 0 and high / len(judged) <= best / KNEE_RATIO:
|
|
1064
|
+
return _with_estimate({"stop_suggested": True, "stop_rule": "knee",
|
|
1065
|
+
"stop_reason": (f"本批 {len(judged)} 篇中只有 {high} 篇评为高,不到此前最好一批"
|
|
1066
|
+
f"({best:.0%})的 1/{KNEE_RATIO},相关文献大多已读到")}, est)
|
|
1067
|
+
useful = sum(1 for r in judged if r in ("高", "中"))
|
|
1068
|
+
if useful <= max(1, len(judged) // 15):
|
|
1069
|
+
return _with_estimate({"stop_suggested": True, "stop_rule": "few_useful",
|
|
1070
|
+
"stop_reason": f"本批 {len(judged)} 篇中只有 {useful} 篇评为高或中,新增已经很少"}, est)
|
|
1071
|
+
return _with_estimate({"stop_suggested": False, "stop_reason": ""}, est)
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
def _stop_rule(stop: dict[str, Any]) -> str:
|
|
1075
|
+
"""Which rule suggested stopping ("" when none): ``stop_rule`` as _stop sets it."""
|
|
1076
|
+
return str(stop.get("stop_rule") or "") if stop.get("stop_suggested") else ""
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def _log_stop(state: dict[str, Any], rated: dict[str, str], stop: dict[str, Any]) -> None:
|
|
1080
|
+
"""Keep what the stop rule and the estimate said at each point, to calibrate them later.
|
|
1081
|
+
|
|
1082
|
+
``scripts/stop_calibration.py`` compares each entry with how many 高 papers
|
|
1083
|
+
were still found after it. One entry per number of rated papers.
|
|
1084
|
+
"""
|
|
1085
|
+
log = state.setdefault("stop_checks", [])
|
|
1086
|
+
if log and log[-1].get("screened") == len(rated):
|
|
1087
|
+
log.pop() # asked again without new ratings: keep the latest answer
|
|
1088
|
+
est = stop.get("remaining_estimate") or {}
|
|
1089
|
+
log.append({"time": time.strftime("%Y-%m-%d %H:%M:%S"), "screened": len(rated),
|
|
1090
|
+
"high": sum(1 for r in rated.values() if r == "高"), "rule": _stop_rule(stop),
|
|
1091
|
+
"estimate": est.get("remaining"), "next_batch": est.get("next_batch"), "decay": est.get("decay")})
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _with_estimate(stop: dict[str, Any], est: dict[str, Any] | None) -> dict[str, Any]:
|
|
1095
|
+
"""Attach the estimate; when stopping is suggested, also say it in the reason."""
|
|
1096
|
+
if est is None:
|
|
1097
|
+
return stop
|
|
1098
|
+
stop["remaining_estimate"] = est
|
|
1099
|
+
if stop["stop_suggested"] and "remaining" in est:
|
|
1100
|
+
stop["stop_reason"] += (f"。按每批高相关比例的下降趋势估算,继续读约还能找到 "
|
|
1101
|
+
f"{est['remaining']:.0f} 篇高相关文献(下一批约 {est['next_batch']:.0f} 篇)")
|
|
1102
|
+
return stop
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
def _coverage(state: dict[str, Any], rated: dict[str, str], order: list[str]) -> list[dict[str, Any]]:
|
|
1106
|
+
pos = {k: i for i, k in enumerate(order, 1)}
|
|
1107
|
+
out = []
|
|
1108
|
+
for doi in state.get("must_find") or []:
|
|
1109
|
+
if doi in rated:
|
|
1110
|
+
out.append({"doi": doi, "status": "screened", "relevance": rated[doi]})
|
|
1111
|
+
elif doi in state["seeds"]:
|
|
1112
|
+
out.append({"doi": doi, "status": "seed"})
|
|
1113
|
+
elif doi in pos:
|
|
1114
|
+
out.append({"doi": doi, "status": "in_pool", "rank": pos[doi]})
|
|
1115
|
+
else:
|
|
1116
|
+
out.append({"doi": doi, "status": "not_found"})
|
|
1117
|
+
return out
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
# What a rated paper keeps in the screening record, so the app can show it
|
|
1121
|
+
# (abstract and authors included: the app never reads the pool).
|
|
1122
|
+
_BATCH_FIELDS = ("doi", "title", "year", "journal", "journal_rank", "retracted", "concern",
|
|
1123
|
+
"authors", "cited_by_count", "abstract", "abstract_missing")
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def _write_summary(state: dict[str, Any], rated: dict[str, str], stop: dict[str, Any],
|
|
1127
|
+
coverage: list[dict[str, Any]], notes: list[str], remaining: int,
|
|
1128
|
+
batch: list[dict[str, Any]]) -> None:
|
|
1129
|
+
"""A small overview of the session for the app, which never reads the (huge) pool."""
|
|
1130
|
+
batches = []
|
|
1131
|
+
for i, keys in enumerate(b for b in state["served"] if b):
|
|
1132
|
+
counts = {r: sum(1 for k in keys if rated.get(k) == r) for r in screening.RELEVANCE}
|
|
1133
|
+
batches.append({"batch": i + 1, "size": len(keys), "rated": sum(counts.values()), **counts})
|
|
1134
|
+
rounds = [{f: r.get(f) for f in ("time", "event", "new_seeds", "pool_added", "expand_seconds",
|
|
1135
|
+
"rank_seconds", "openalex_credits", "seconds") if r.get(f) is not None}
|
|
1136
|
+
for r in state["rounds"]]
|
|
1137
|
+
summary = {"session": state["session"], "question": state["question"], "mode": "deep",
|
|
1138
|
+
"updated": time.strftime("%Y-%m-%d %H:%M:%S"), "pool": len(state["pool"]),
|
|
1139
|
+
"screened": len(rated), "remaining": remaining, "ranked_by": state.get("ranked_by", "formula"),
|
|
1140
|
+
"seeds": len(state["seeds"]), "batches": batches, "rounds": rounds,
|
|
1141
|
+
"stop_suggested": stop.get("stop_suggested", False), "stop_reason": stop.get("stop_reason", ""),
|
|
1142
|
+
"remaining_estimate": stop.get("remaining_estimate"),
|
|
1143
|
+
"coverage": coverage, "api_warnings": notes, "core_terms": state.get("core_terms", []),
|
|
1144
|
+
"glossary": state.get("glossary_used", []),
|
|
1145
|
+
"queries": state.get("queries", []), "current_batch": batch}
|
|
1146
|
+
try:
|
|
1147
|
+
db.save_summary(state["session"], summary)
|
|
1148
|
+
except (OSError, sqlite3.Error) as exc:
|
|
1149
|
+
logger.warning("deep search: could not write the session summary (%s)", exc)
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def _next_batch(state: dict[str, Any], rated: dict[str, str] | None = None, force: bool = False,
|
|
1153
|
+
prefilled: bool = False) -> dict[str, Any]:
|
|
1154
|
+
from .abstracts import clean_abstract, fill_abstracts
|
|
1155
|
+
from .quick import _add_journal_rank, _flag_retractions
|
|
1156
|
+
rated = rated if rated is not None else _ratings(state)
|
|
1157
|
+
stop = _stop(state, rated)
|
|
1158
|
+
_log_stop(state, rated, stop)
|
|
1159
|
+
go_on = force or not stop["stop_suggested"]
|
|
1160
|
+
if go_on:
|
|
1161
|
+
if not prefilled and _prefill_abstracts(state, rated):
|
|
1162
|
+
_score(state)
|
|
1163
|
+
_phase("rank")
|
|
1164
|
+
state["ranked_by"] = _rerank(state, rated)
|
|
1165
|
+
served_before = {k for b in state["served"] for k in b}
|
|
1166
|
+
order = [k for k in _ranked(state) if k not in rated]
|
|
1167
|
+
fresh = [k for k in order if k not in served_before]
|
|
1168
|
+
# Unrated papers from earlier batches come back first: the agent may have skipped some.
|
|
1169
|
+
pending = [k for k in order if k in served_before]
|
|
1170
|
+
picked = (pending + fresh)[: state["batch_size"]] if go_on else []
|
|
1171
|
+
items = [dict(state["pool"][k]) for k in picked]
|
|
1172
|
+
if items:
|
|
1173
|
+
_phase("batch", len(items))
|
|
1174
|
+
fill_abstracts([it for it in items if not it.get("abstract")], load_config(), progress=_step)
|
|
1175
|
+
for k, it in zip(picked, items): # keep what was found, so ranking can use it too
|
|
1176
|
+
if it.get("abstract") and not state["pool"][k].get("abstract"):
|
|
1177
|
+
state["pool"][k]["abstract"] = it["abstract"]
|
|
1178
|
+
state["pool"][k]["abstract_source"] = it.get("abstract_source")
|
|
1179
|
+
_add_journal_rank(items)
|
|
1180
|
+
_flag_retractions(items)
|
|
1181
|
+
batch = []
|
|
1182
|
+
for n, (k, it) in enumerate(zip(picked, items), 1):
|
|
1183
|
+
batch.append({
|
|
1184
|
+
"n": n, "doi": it.get("doi"), "title": it["title"], "year": it.get("year"), "journal": it.get("journal"),
|
|
1185
|
+
"journal_rank": it.get("journal_rank"), "authors": (it.get("authors") or [])[:3],
|
|
1186
|
+
"cited_by_count": it.get("cited_by_count"),
|
|
1187
|
+
"abstract": clean_abstract(it.get("abstract") or "")[:ABSTRACT_CHARS],
|
|
1188
|
+
"abstract_missing": not it.get("abstract"),
|
|
1189
|
+
"retracted": it.get("retracted") or None, "concern": it.get("concern") or None,
|
|
1190
|
+
"why": {"shared_refs_with_seeds": it.get("coupling", 0), "queries_matched": it.get("query_hits", 0),
|
|
1191
|
+
"found_via": it.get("via", [])[:2]},
|
|
1192
|
+
})
|
|
1193
|
+
if picked:
|
|
1194
|
+
state["served"].append([k for k in picked if k not in served_before])
|
|
1195
|
+
# What action="rate" needs to turn "n=3 高" back into a screening item.
|
|
1196
|
+
state["last_batch"] = [{"key": k, **{f: b.get(f) for f in _BATCH_FIELDS}} for k, b in zip(picked, batch)]
|
|
1197
|
+
_save(state)
|
|
1198
|
+
from .. import quota
|
|
1199
|
+
notes = quota.warnings()
|
|
1200
|
+
if not str(load_config().get("openalex_api_key") or "").strip():
|
|
1201
|
+
notes.insert(0, "no OpenAlex API key: the free daily budget without one is about 1,000 credits, "
|
|
1202
|
+
"which one deep search can use up; add a free key (openalex_api_key)")
|
|
1203
|
+
if state.get("deferred_refs"):
|
|
1204
|
+
notes.append(f"openalex: this round's credit budget was used; {state['deferred_refs']} seed references "
|
|
1205
|
+
"were not followed yet and are tried again in later rounds")
|
|
1206
|
+
coverage = _coverage(state, rated, _ranked(state))
|
|
1207
|
+
_write_summary(state, rated, stop, coverage, notes, remaining=len(order) - len(picked), batch=batch)
|
|
1208
|
+
return {"session": state["session"], "pool": len(state["pool"]), "screened": len(rated),
|
|
1209
|
+
"api_warnings": notes,
|
|
1210
|
+
"remaining": len(order) - len(picked), "ranked_by": state.get("ranked_by", "formula"), "batch": batch,
|
|
1211
|
+
"coverage": coverage, **stop,
|
|
1212
|
+
"next_step": ("rate every paper: action='rate', session=..., ratings=[{n, relevance, reason}"
|
|
1213
|
+
+ (f", abstract_zh (Chinese abstract) for {'/'.join(screening.translate_wanted())}"
|
|
1214
|
+
if screening.translate_wanted() and not translate.by_api() else "")
|
|
1215
|
+
+ "], then action='next'") if batch else
|
|
1216
|
+
"stop: show the user the screening session and let them choose"}
|