litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""Abstracts for search results: clean what we have, fetch what is missing.
|
|
2
|
+
|
|
3
|
+
Fetch order, free sources first (measured on 45 materials-science papers,
|
|
4
|
+
29 of them Elsevier):
|
|
5
|
+
|
|
6
|
+
1. OpenAlex - already in the search results; holds almost no Elsevier abstracts.
|
|
7
|
+
2. Semantic Scholar batch endpoint - one keyless request, about 2/3 coverage.
|
|
8
|
+
3. Scopus Abstract Retrieval API - with ``elsevier_api_key``; covered all 45
|
|
9
|
+
(all publishers, not only Elsevier). Weekly quota, so only for what is
|
|
10
|
+
still missing; Elsevier DOIs go first since 1-2 rarely have them.
|
|
11
|
+
4. Crossref - publisher-deposited abstracts (Wiley, Springer, MDPI...), for
|
|
12
|
+
users without an Elsevier key.
|
|
13
|
+
|
|
14
|
+
Whatever is still missing is marked ``abstract_missing`` so the screener
|
|
15
|
+
knows it judged from the title alone.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import html
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import threading
|
|
24
|
+
import time
|
|
25
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
26
|
+
from concurrent.futures import TimeoutError as FuturesTimeout
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
import requests
|
|
30
|
+
|
|
31
|
+
from .. import quota
|
|
32
|
+
from ..log import get_logger
|
|
33
|
+
|
|
34
|
+
log = get_logger()
|
|
35
|
+
|
|
36
|
+
_MAX_CHARS = 4000 # longer abstracts keep head and tail (conclusions sit at the end)
|
|
37
|
+
_HEAD, _TAIL = 2600, 1200
|
|
38
|
+
|
|
39
|
+
_TAG = re.compile(r"<[^>]+>")
|
|
40
|
+
_LEADING = re.compile(r"^\s*(?:graphical\s+abstract|abstract|summary|highlights)\s*[:.\-]?\s*", re.I)
|
|
41
|
+
_TRAILING = [
|
|
42
|
+
re.compile(r"\s*(?:Copyright\s*)?(?:©|\(c\)|Copyright)\s*\d{4}.*$", re.I | re.S),
|
|
43
|
+
re.compile(r"\s*This article is protected by copyright\..*$", re.I | re.S),
|
|
44
|
+
re.compile(r"\s*All rights reserved\.?\s*$", re.I),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def clean_abstract(text: str) -> str:
|
|
49
|
+
"""Plain text: no JATS/HTML tags, entities, boilerplate or copyright line."""
|
|
50
|
+
if not text:
|
|
51
|
+
return ""
|
|
52
|
+
t = str(text)
|
|
53
|
+
# Sub/superscripts: Crossref's pretty-printed JATS puts whitespace around
|
|
54
|
+
# them ("A\n <jats:sub>2</jats:sub>\n B ..."). Attach the index to its
|
|
55
|
+
# symbol, and the next element symbol of a formula to the index.
|
|
56
|
+
t = re.sub(r"\s*<(?:jats:)?(?:sub|sup|inf)\b[^>]*>\s*(.*?)\s*</(?:jats:)?(?:sub|sup|inf)>",
|
|
57
|
+
r"\1", t, flags=re.S | re.I)
|
|
58
|
+
t = re.sub(r"(?<=[A-Z][a-z]\d)\s+(?=[A-Z][a-z]?\d)|(?<=[A-Z]\d)\s+(?=[A-Z][a-z]?\d)", "", t)
|
|
59
|
+
t = html.unescape(t)
|
|
60
|
+
t = _TAG.sub(" ", t)
|
|
61
|
+
t = re.sub(r"\s+", " ", t).strip()
|
|
62
|
+
for _ in range(2):
|
|
63
|
+
t = _LEADING.sub("", t)
|
|
64
|
+
for pat in _TRAILING:
|
|
65
|
+
t = pat.sub("", t)
|
|
66
|
+
t = t.strip()
|
|
67
|
+
if len(t) > _MAX_CHARS:
|
|
68
|
+
t = t[:_HEAD].rstrip() + " … " + t[-_TAIL:].lstrip()
|
|
69
|
+
return t
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _session() -> requests.Session:
|
|
73
|
+
from ..network import user_agent
|
|
74
|
+
|
|
75
|
+
s = requests.Session()
|
|
76
|
+
s.headers["User-Agent"] = user_agent("(https://github.com/dysprosium231/litloop-paper-agent)")
|
|
77
|
+
return s
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
S2_BATCH = 500
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _semantic_scholar(dois: list[str], timeout: float) -> dict[str, str]:
|
|
84
|
+
if not dois:
|
|
85
|
+
return {}
|
|
86
|
+
headers = {}
|
|
87
|
+
if os.environ.get("SEMANTIC_SCHOLAR_API_KEY"):
|
|
88
|
+
headers["x-api-key"] = os.environ["SEMANTIC_SCHOLAR_API_KEY"]
|
|
89
|
+
out: dict[str, str] = {}
|
|
90
|
+
for i in range(0, len(dois), S2_BATCH): # the batch endpoint takes at most 500 ids
|
|
91
|
+
chunk = dois[i:i + S2_BATCH]
|
|
92
|
+
try:
|
|
93
|
+
with quota.slot("semantic_scholar"):
|
|
94
|
+
r = _session().post("https://api.semanticscholar.org/graph/v1/paper/batch",
|
|
95
|
+
params={"fields": "abstract"}, json={"ids": [f"DOI:{d}" for d in chunk]},
|
|
96
|
+
headers=headers, timeout=timeout)
|
|
97
|
+
quota.record("semantic_scholar", r)
|
|
98
|
+
if r.status_code != 200:
|
|
99
|
+
log.debug(f"abstracts: Semantic Scholar batch HTTP {r.status_code}")
|
|
100
|
+
break
|
|
101
|
+
rows = r.json()
|
|
102
|
+
except (requests.RequestException, ValueError) as e:
|
|
103
|
+
log.debug(f"abstracts: Semantic Scholar batch failed: {e}")
|
|
104
|
+
break
|
|
105
|
+
if not isinstance(rows, list):
|
|
106
|
+
log.debug("abstracts: Semantic Scholar batch returned no list")
|
|
107
|
+
break
|
|
108
|
+
out.update({d: x["abstract"] for d, x in zip(chunk, rows) if isinstance(x, dict) and x.get("abstract")})
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class _Scopus:
|
|
113
|
+
"""Scopus Abstract Retrieval: 9 requests/s and 10,000 per week per key
|
|
114
|
+
(dev.elsevier.com/api_key_settings.html). Parallelism and pacing come from
|
|
115
|
+
``quota``; a 429 carrying ``X-ELS-Status: QUOTA_EXCEEDED`` means the weekly
|
|
116
|
+
quota is spent, any other 429 is the per-second throttle."""
|
|
117
|
+
|
|
118
|
+
def __init__(self, key: str):
|
|
119
|
+
self.key = key
|
|
120
|
+
self.exhausted = False # weekly quota spent: stop asking this run
|
|
121
|
+
|
|
122
|
+
def get(self, doi: str, timeout: float) -> str | None:
|
|
123
|
+
"""The abstract, "" if Scopus has none, None if it could not be asked."""
|
|
124
|
+
for attempt in range(3):
|
|
125
|
+
if self.exhausted:
|
|
126
|
+
return None
|
|
127
|
+
try:
|
|
128
|
+
with quota.slot("scopus"):
|
|
129
|
+
r = _session().get(f"https://api.elsevier.com/content/abstract/doi/{doi}",
|
|
130
|
+
headers={"X-ELS-APIKey": self.key, "Accept": "application/json"},
|
|
131
|
+
timeout=timeout)
|
|
132
|
+
except requests.RequestException:
|
|
133
|
+
return None
|
|
134
|
+
quota.record("scopus", r)
|
|
135
|
+
if r.status_code == 429:
|
|
136
|
+
if "QUOTA_EXCEEDED" in str(r.headers.get("X-ELS-Status") or ""):
|
|
137
|
+
self.exhausted = True
|
|
138
|
+
log.warning("abstracts: Scopus weekly quota exhausted")
|
|
139
|
+
return None
|
|
140
|
+
time.sleep(1.0 + attempt) # throttled: slow down and retry
|
|
141
|
+
continue
|
|
142
|
+
if r.status_code in (400, 404):
|
|
143
|
+
return "" # not in Scopus
|
|
144
|
+
if r.status_code != 200:
|
|
145
|
+
return None
|
|
146
|
+
try:
|
|
147
|
+
core = (r.json().get("abstracts-retrieval-response") or {}).get("coredata") or {}
|
|
148
|
+
except ValueError:
|
|
149
|
+
return None
|
|
150
|
+
return str(core.get("dc:description") or "")
|
|
151
|
+
return None # still throttled
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _crossref(doi: str, timeout: float, email: str = "") -> str | None:
|
|
155
|
+
"""One Crossref record. With a contact email the request joins the polite
|
|
156
|
+
pool (3 parallel requests instead of 1)."""
|
|
157
|
+
provider = "crossref_polite" if email else "crossref"
|
|
158
|
+
try:
|
|
159
|
+
with quota.slot(provider):
|
|
160
|
+
r = _session().get(f"https://api.crossref.org/works/{doi}",
|
|
161
|
+
params={"mailto": email} if email else None, timeout=timeout)
|
|
162
|
+
quota.record("crossref", r)
|
|
163
|
+
if r.status_code == 404:
|
|
164
|
+
return ""
|
|
165
|
+
return str(r.json().get("message", {}).get("abstract") or "") if r.status_code == 200 else None
|
|
166
|
+
except (requests.RequestException, ValueError):
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
ELSEVIER_PREFIX = "10.1016/"
|
|
171
|
+
# DOI prefixes of Springer Nature (Springer, Nature, BMC, Kluwer, Palgrave, EPJ).
|
|
172
|
+
SPRINGER_PREFIXES = ("10.1007/", "10.1038/", "10.1186/", "10.1023/", "10.1057/", "10.1140/")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _springer(doi: str, timeout: float, key: str, daily_limit: int) -> str | None:
|
|
176
|
+
"""Abstract from the Springer Nature Meta API (subscription articles too).
|
|
177
|
+
Abstracts are licensed for personal, non-commercial use only."""
|
|
178
|
+
if quota.summary().get("springer", {}).get("requests_today", 0) >= daily_limit:
|
|
179
|
+
return None # the plan's daily allowance is used up
|
|
180
|
+
try:
|
|
181
|
+
with quota.slot("springer"):
|
|
182
|
+
r = _session().get("https://api.springernature.com/meta/v2/json",
|
|
183
|
+
params={"q": f"doi:{doi}", "api_key": key}, timeout=timeout)
|
|
184
|
+
quota.record("springer", r)
|
|
185
|
+
if r.status_code != 200:
|
|
186
|
+
return None
|
|
187
|
+
records = r.json().get("records") or []
|
|
188
|
+
except (requests.RequestException, ValueError):
|
|
189
|
+
return None
|
|
190
|
+
if not records:
|
|
191
|
+
return ""
|
|
192
|
+
ab = records[0].get("abstract") or ""
|
|
193
|
+
if isinstance(ab, dict): # {"h1": "Abstract", "p": [...]}
|
|
194
|
+
ab = ab.get("p") or ""
|
|
195
|
+
return " ".join(ab) if isinstance(ab, list) else str(ab)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def fill_abstracts(results: list[dict[str, Any]], config: dict[str, Any] | None = None,
|
|
199
|
+
budget_s: float = 20.0, progress: Any = None) -> None:
|
|
200
|
+
"""Clean every abstract in place; fetch missing ones within BUDGET_S seconds.
|
|
201
|
+
|
|
202
|
+
``progress(done, total)``, if given, is called as papers are settled
|
|
203
|
+
(found, or every source asked said there is none), for a progress bar.
|
|
204
|
+
"""
|
|
205
|
+
for r in results:
|
|
206
|
+
r["abstract"] = clean_abstract(r.get("abstract") or "")
|
|
207
|
+
if r["abstract"] and not r.get("abstract_source"):
|
|
208
|
+
r["abstract_source"] = str(r.get("source") or "search")
|
|
209
|
+
missing = {str(r["doi"]).lower(): r for r in results if not r["abstract"] and r.get("doi")}
|
|
210
|
+
if missing and (config or {}).get("search_fill_abstracts", True):
|
|
211
|
+
deadline = time.monotonic() + budget_s
|
|
212
|
+
left = lambda: max(2.0, deadline - time.monotonic()) # noqa: E731
|
|
213
|
+
key = str((config or {}).get("elsevier_api_key") or "").strip()
|
|
214
|
+
# The contact email (Crossref polite pool); the shipped placeholder does not count.
|
|
215
|
+
email = str((config or {}).get("email") or "").strip()
|
|
216
|
+
email = "" if email.endswith(".invalid") else email
|
|
217
|
+
springer_key = str((config or {}).get("springer_meta_api_key") or "").strip()
|
|
218
|
+
springer_limit = int((config or {}).get("springer_meta_daily_limit") or 500)
|
|
219
|
+
scopus = _Scopus(key) if key else None
|
|
220
|
+
|
|
221
|
+
def take(doi: str, text: str, source: str) -> None:
|
|
222
|
+
text = clean_abstract(text)
|
|
223
|
+
if text and doi in missing and not missing[doi]["abstract"]:
|
|
224
|
+
missing[doi]["abstract"] = text
|
|
225
|
+
missing[doi]["abstract_source"] = source
|
|
226
|
+
|
|
227
|
+
def ask(source: str, doi: str) -> tuple[str, str, str | None]:
|
|
228
|
+
if missing[doi]["abstract"] or time.monotonic() >= deadline: # another source was faster
|
|
229
|
+
return source, doi, None
|
|
230
|
+
if source == "scopus":
|
|
231
|
+
return source, doi, scopus.get(doi, min(15.0, left()))
|
|
232
|
+
if source == "springer":
|
|
233
|
+
return source, doi, _springer(doi, min(15.0, left()), springer_key, springer_limit)
|
|
234
|
+
return source, doi, _crossref(doi, min(15.0, left()), email)
|
|
235
|
+
|
|
236
|
+
# All sources at once, first answer wins: asking them in turn made one
|
|
237
|
+
# slow source (Semantic Scholar without a key often takes 10 s or more)
|
|
238
|
+
# hold up abstracts another source returns in a second.
|
|
239
|
+
# Scopus jobs are queued before Crossref ones, and Elsevier DOIs skip
|
|
240
|
+
# Crossref (Elsevier does not deposit abstracts there): with 150
|
|
241
|
+
# papers to fill, Crossref calls queued alongside used to crowd the
|
|
242
|
+
# Scopus ones out of the budget.
|
|
243
|
+
answered: dict[str, set[str]] = {} # sources that gave a definite reply, per DOI
|
|
244
|
+
asked: dict[str, list[str]] = {}
|
|
245
|
+
pool = ThreadPoolExecutor(max_workers=12)
|
|
246
|
+
try:
|
|
247
|
+
jobs = [pool.submit(lambda: ("semantic_scholar", _semantic_scholar(list(missing), min(8.0, left()))))]
|
|
248
|
+
if scopus:
|
|
249
|
+
jobs += [pool.submit(ask, "scopus", doi) for doi in missing]
|
|
250
|
+
asked = {doi: (["scopus"] if scopus else []) +
|
|
251
|
+
(["springer"] if springer_key and doi.startswith(SPRINGER_PREFIXES) else []) +
|
|
252
|
+
([] if scopus and doi.startswith(ELSEVIER_PREFIX) else ["crossref"]) for doi in missing}
|
|
253
|
+
jobs += [pool.submit(ask, "springer", doi) for doi in missing if "springer" in asked[doi]]
|
|
254
|
+
jobs += [pool.submit(ask, "crossref", doi) for doi in missing if "crossref" in asked[doi]]
|
|
255
|
+
settled = lambda: sum(1 for d, r in missing.items() # noqa: E731
|
|
256
|
+
if r["abstract"] or (asked.get(d) and set(asked[d]) <= answered.get(d, set())))
|
|
257
|
+
if progress:
|
|
258
|
+
progress(settled(), len(missing))
|
|
259
|
+
for job in as_completed(jobs, timeout=max(0.1, deadline - time.monotonic())):
|
|
260
|
+
res = job.result()
|
|
261
|
+
if res[0] == "semantic_scholar":
|
|
262
|
+
for doi, text in res[1].items():
|
|
263
|
+
take(doi, text, "semantic_scholar")
|
|
264
|
+
else:
|
|
265
|
+
if res[2] is not None:
|
|
266
|
+
answered.setdefault(res[1], set()).add(res[0])
|
|
267
|
+
take(res[1], res[2], res[0])
|
|
268
|
+
if progress:
|
|
269
|
+
progress(settled(), len(missing))
|
|
270
|
+
if all(r["abstract"] for r in missing.values()):
|
|
271
|
+
break
|
|
272
|
+
except FuturesTimeout:
|
|
273
|
+
log.debug("abstracts: budget used up")
|
|
274
|
+
finally:
|
|
275
|
+
pool.shutdown(wait=False, cancel_futures=True) # do not wait for the slow ones
|
|
276
|
+
# Decided: found, or every source asked gave a definite "none". Papers left
|
|
277
|
+
# undecided (timeout, throttling, budget) are worth asking again later.
|
|
278
|
+
for doi, r in missing.items():
|
|
279
|
+
r["abstract_decided"] = bool(r["abstract"]) or set(asked.get(doi, [])) <= answered.get(doi, set()) != set()
|
|
280
|
+
for r in results:
|
|
281
|
+
if not r["abstract"]:
|
|
282
|
+
r["abstract_missing"] = True # judge from the title only
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def trim_authors(results: list[dict[str, Any]], keep: int = 3) -> None:
|
|
286
|
+
"""First authors only; consortium papers list hundreds."""
|
|
287
|
+
for r in results:
|
|
288
|
+
authors = r.get("authors") or []
|
|
289
|
+
if len(authors) > keep:
|
|
290
|
+
r["author_count"] = len(authors)
|
|
291
|
+
r["authors"] = list(authors[:keep])
|