litloop 1.18.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litloop/__init__.py +89 -0
- litloop/__main__.py +6 -0
- litloop/_source_hygiene.py +158 -0
- litloop/agents/__init__.py +13 -0
- litloop/agents/client.py +164 -0
- litloop/agents/runner.py +123 -0
- litloop/agents/system.py +54 -0
- litloop/agents/tools.py +142 -0
- litloop/app_server.py +8 -0
- litloop/browser/__init__.py +1 -0
- litloop/browser/backend.py +1009 -0
- litloop/browser/camofox_api.py +368 -0
- litloop/browser/chrome_copy.py +136 -0
- litloop/browser/cloakbrowser_compat.py +72 -0
- litloop/browser/cookies.py +347 -0
- litloop/browser/engine.py +965 -0
- litloop/browser/login.py +636 -0
- litloop/browser/pdf.py +263 -0
- litloop/browser/profile_health.py +167 -0
- litloop/browser/runtime.py +210 -0
- litloop/cache.py +104 -0
- litloop/citations/__init__.py +1 -0
- litloop/citations/bib_import.py +102 -0
- litloop/citations/bibtex_crossref.py +143 -0
- litloop/citations/formats.py +225 -0
- litloop/citations/paper_list.py +200 -0
- litloop/citations/session_export.py +94 -0
- litloop/cli/__init__.py +12 -0
- litloop/cli/_utils.py +267 -0
- litloop/cli/commands.py +1098 -0
- litloop/cli/doctor.py +141 -0
- litloop/cli/main.py +1173 -0
- litloop/cli/web.py +190 -0
- litloop/config.py +413 -0
- litloop/data/VARCON-LICENSE.txt +62 -0
- litloop/data/elements.tsv +119 -0
- litloop/data/institutional_identity_policy.json +165 -0
- litloop/data/publisher_access_catalog.json +969 -0
- litloop/data/publisher_carsi.json +105 -0
- litloop/data/spelling_search.tsv.gz +0 -0
- litloop/data/spelling_us.tsv.gz +0 -0
- litloop/data/webvpn.dat +0 -0
- litloop/data/webvpn.json +603 -0
- litloop/db.py +1922 -0
- litloop/deps.py +128 -0
- litloop/desktop/__init__.py +1 -0
- litloop/desktop/agent_runner.py +537 -0
- litloop/desktop/download_jobs.py +571 -0
- litloop/desktop/error_text.py +192 -0
- litloop/desktop/server.py +999 -0
- litloop/desktop/settings.py +552 -0
- litloop/desktop/translate.py +103 -0
- litloop/diagnostics.py +126 -0
- litloop/download/__init__.py +1 -0
- litloop/download/doi_check.py +44 -0
- litloop/download/fetcher.py +975 -0
- litloop/download/markdown.py +130 -0
- litloop/download/pdf.py +454 -0
- litloop/download/pipeline.py +1166 -0
- litloop/download/progress.py +312 -0
- litloop/download/rename.py +211 -0
- litloop/extractors/__init__.py +0 -0
- litloop/extractors/html_extractor.py +31 -0
- litloop/extractors/pdf_extractor.py +151 -0
- litloop/extractors/publisher_adapters/__init__.py +0 -0
- litloop/extractors/publisher_adapters/acs.py +167 -0
- litloop/extractors/publisher_adapters/elsevier.py +178 -0
- litloop/extractors/publisher_adapters/generic.py +191 -0
- litloop/extractors/publisher_adapters/nature.py +204 -0
- litloop/extractors/publisher_adapters/rsc.py +154 -0
- litloop/extractors/publisher_adapters/tandfonline.py +154 -0
- litloop/extractors/publisher_adapters/wiley.py +160 -0
- litloop/http_utils.py +57 -0
- litloop/identifiers.py +134 -0
- litloop/institutional/__init__.py +1 -0
- litloop/institutional/auth.py +576 -0
- litloop/institutional/carsi_login.py +569 -0
- litloop/institutional/config_adapter.py +63 -0
- litloop/institutional/proxy_check.py +98 -0
- litloop/institutional/remote_assist.py +274 -0
- litloop/institutional/schools.py +102 -0
- litloop/institutional/session_heal.py +73 -0
- litloop/institutional/session_store.py +105 -0
- litloop/log.py +47 -0
- litloop/models.py +269 -0
- litloop/network.py +417 -0
- litloop/optional.py +310 -0
- litloop/publishers/__init__.py +1 -0
- litloop/publishers/access.py +232 -0
- litloop/publishers/pdf_router.py +451 -0
- litloop/publishers/profiles.py +899 -0
- litloop/publishers/strategies/__init__.py +122 -0
- litloop/publishers/strategies/_core.py +2340 -0
- litloop/publishers/strategies/_elsevier_api.py +578 -0
- litloop/publishers/strategies/acm.py +27 -0
- litloop/publishers/strategies/acs.py +42 -0
- litloop/publishers/strategies/aip.py +27 -0
- litloop/publishers/strategies/aps.py +29 -0
- litloop/publishers/strategies/asce.py +28 -0
- litloop/publishers/strategies/base.py +108 -0
- litloop/publishers/strategies/copernicus.py +24 -0
- litloop/publishers/strategies/elsevier.py +66 -0
- litloop/publishers/strategies/generic.py +39 -0
- litloop/publishers/strategies/ieee.py +28 -0
- litloop/publishers/strategies/iop.py +27 -0
- litloop/publishers/strategies/nature.py +27 -0
- litloop/publishers/strategies/oxford.py +27 -0
- litloop/publishers/strategies/registry.py +83 -0
- litloop/publishers/strategies/royalsociety.py +28 -0
- litloop/publishers/strategies/rsc.py +27 -0
- litloop/publishers/strategies/sage.py +30 -0
- litloop/publishers/strategies/science.py +27 -0
- litloop/publishers/strategies/springer.py +34 -0
- litloop/publishers/strategies/tandfonline.py +32 -0
- litloop/publishers/strategies/wiley.py +32 -0
- litloop/publishers/supplementary.py +219 -0
- litloop/quota.py +202 -0
- litloop/search/__init__.py +1 -0
- litloop/search/abstracts.py +291 -0
- litloop/search/deep.py +1216 -0
- litloop/search/quick.py +1102 -0
- litloop/search/scansci_find.py +410 -0
- litloop/search/screening.py +225 -0
- litloop/search/title_to_doi.py +163 -0
- litloop/search/watches.py +124 -0
- litloop/server.py +1731 -0
- litloop/sources/__init__.py +1339 -0
- litloop/sources/_batch.py +529 -0
- litloop/sources/_index.py +221 -0
- litloop/sources/_routes.py +290 -0
- litloop/sources/altcha.py +90 -0
- litloop/sources/arxiv.py +45 -0
- litloop/sources/attempt.py +91 -0
- litloop/sources/carsi.py +74 -0
- litloop/sources/core_api.py +46 -0
- litloop/sources/crossref.py +162 -0
- litloop/sources/elsevier_api.py +620 -0
- litloop/sources/elsevier_check.py +274 -0
- litloop/sources/europepmc.py +81 -0
- litloop/sources/ezproxy.py +317 -0
- litloop/sources/instsci.py +1295 -0
- litloop/sources/libgen.py +116 -0
- litloop/sources/nature.py +39 -0
- litloop/sources/oa_discovery.py +135 -0
- litloop/sources/openaire.py +122 -0
- litloop/sources/openalex.py +100 -0
- litloop/sources/publishers.py +684 -0
- litloop/sources/sage_cn.py +139 -0
- litloop/sources/scibban.py +10 -0
- litloop/sources/scihub.py +1379 -0
- litloop/sources/scihub_domains.py +203 -0
- litloop/sources/scihub_speed.py +187 -0
- litloop/sources/scoring.py +250 -0
- litloop/sources/semantic_scholar.py +146 -0
- litloop/sources/springer_tdm.py +172 -0
- litloop/sources/unpaywall.py +116 -0
- litloop/sources/vpnsci.py +38 -0
- litloop/store.py +264 -0
- litloop/templates/index.html +347 -0
- litloop/terms/__init__.py +1 -0
- litloop/terms/glossary.py +422 -0
- litloop/terms/spelling.py +89 -0
- litloop/terms/text.py +106 -0
- litloop/tor/__init__.py +1 -0
- litloop/tor/embedded.py +371 -0
- litloop/tor/proxy.py +110 -0
- litloop/updates.py +148 -0
- litloop/zotero/__init__.py +1 -0
- litloop/zotero/bridge.py +309 -0
- litloop/zotero/client.py +317 -0
- litloop-1.18.3.dist-info/METADATA +347 -0
- litloop-1.18.3.dist-info/RECORD +176 -0
- litloop-1.18.3.dist-info/WHEEL +5 -0
- litloop-1.18.3.dist-info/entry_points.txt +2 -0
- litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
- litloop-1.18.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""The glossary: abbreviations and synonyms a search expands by itself.
|
|
2
|
+
|
|
3
|
+
Papers write "PDC", "polymer-derived ceramic" or "preceramic polymer" for the
|
|
4
|
+
same thing. Three sources, used together (``in_use``):
|
|
5
|
+
|
|
6
|
+
* the user's entries: only someone who knows the field can say these;
|
|
7
|
+
* the chemical elements, built in (``elements``: "Al" = "aluminum", from
|
|
8
|
+
PubChem, data/elements.tsv);
|
|
9
|
+
* British and American spellings, built in and applied word by word
|
|
10
|
+
(``spelling``, from VarCon).
|
|
11
|
+
|
|
12
|
+
The user may switch off any built-in element or spelling (``set_builtin``).
|
|
13
|
+
Kept in ``<litloop home>/glossary.json`` through ``store`` (shared by the app
|
|
14
|
+
and the agent)::
|
|
15
|
+
|
|
16
|
+
{"schema": 1,
|
|
17
|
+
"entries": [{"id": "3f9a1c2e", "terms": ["PDC", "polymer-derived ceramic"],
|
|
18
|
+
"status": "active" | "suggested", "source": "user" | "agent" | "auto",
|
|
19
|
+
"papers": 4, "note": "", "added": "2026-10-25"}],
|
|
20
|
+
"dismissed": [["pdc", "polycrystalline diamond compact"]],
|
|
21
|
+
"disabled": ["element:La", "spelling:aluminum"]}
|
|
22
|
+
|
|
23
|
+
How it is used (decided with the user, 2026-10-25, after looking at how
|
|
24
|
+
others do it):
|
|
25
|
+
|
|
26
|
+
* Every entry in use applies to every search; there is no per-topic scope.
|
|
27
|
+
A search is a few concepts joined by AND, so a wrong expansion (PDC as a
|
|
28
|
+
drill-bit cutter searched as "polymer-derived ceramic ... wear") finds next
|
|
29
|
+
to nothing rather than noise. A query that is the abbreviation alone has
|
|
30
|
+
nothing to hold it, so it is not expanded (``query_variants``).
|
|
31
|
+
* Each search reports what it expanded, as PubMed's Search Details do.
|
|
32
|
+
* Most entries need not be typed: ``discover`` finds "long form (SF)" pairs in
|
|
33
|
+
the abstracts a search has read, with the Schwartz-Hearst algorithm (the
|
|
34
|
+
``abbreviations`` package), and ``suggest_from`` adds those seen in two or
|
|
35
|
+
more papers as suggestions the user confirms. Long forms are compared after
|
|
36
|
+
``normalize`` (case, dashes, plurals, British spelling), so "metal matrix
|
|
37
|
+
composite" and "metal-matrix composites" are one meaning; a short form left
|
|
38
|
+
with several meanings (CMC: carboxymethyl cellulose, ceramic matrix
|
|
39
|
+
composite) gets one suggestion per meaning, and the user keeps the right one.
|
|
40
|
+
A suggestion the user deletes is remembered (``dismissed``) and not made again.
|
|
41
|
+
* The agent may add entries too; they wait for the user's confirmation.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
import re
|
|
47
|
+
import time
|
|
48
|
+
import uuid
|
|
49
|
+
from collections import Counter
|
|
50
|
+
from functools import lru_cache
|
|
51
|
+
from pathlib import Path
|
|
52
|
+
from typing import Any
|
|
53
|
+
|
|
54
|
+
from .. import store
|
|
55
|
+
from . import text
|
|
56
|
+
|
|
57
|
+
MAX_VARIANTS = 3 # extra runs of one keyword query (own value)
|
|
58
|
+
MIN_PAPERS = 2 # a discovered pair is suggested once this many papers write it (own value)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def path() -> Path:
|
|
62
|
+
from ..config import DATA_DIR
|
|
63
|
+
return Path(DATA_DIR) / "glossary.json"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _data() -> dict[str, Any]:
|
|
67
|
+
try:
|
|
68
|
+
data = store.read_json(path())
|
|
69
|
+
except (OSError, ValueError):
|
|
70
|
+
return {"entries": [], "dismissed": [], "disabled": []}
|
|
71
|
+
data.setdefault("entries", [])
|
|
72
|
+
data.setdefault("dismissed", [])
|
|
73
|
+
data.setdefault("disabled", [])
|
|
74
|
+
return data
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def entries(status: str | None = None) -> list[dict[str, Any]]:
|
|
78
|
+
"""All entries, or those with ``status`` ("active", "suggested")."""
|
|
79
|
+
out = [e for e in _data()["entries"] if isinstance(e, dict) and e.get("terms")]
|
|
80
|
+
return [e for e in out if status is None or e.get("status") == status]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def in_use() -> list[dict[str, Any]]:
|
|
84
|
+
"""What a search expands with: the user's entries in use, then the built-in
|
|
85
|
+
elements the user has not switched off."""
|
|
86
|
+
off = disabled()
|
|
87
|
+
return entries("active") + [e for e in elements() if e["id"] not in off]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# --------------------------------------------------------------------------- built in
|
|
91
|
+
|
|
92
|
+
# Element symbols that are also everyday words (OpenAlex search ignores case): "La", "Ce"
|
|
93
|
+
# match French, "Er" German, "Eu" the EU, "In"/"As"/"Be"/"He"/"No" English.
|
|
94
|
+
AMBIGUOUS_SYMBOLS = {"as", "at", "be", "ce", "er", "es", "eu", "he", "in", "la", "no"}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@lru_cache(maxsize=1)
|
|
98
|
+
def elements() -> list[dict[str, Any]]:
|
|
99
|
+
"""The chemical elements as entries. A symbol is matched with its case. One that
|
|
100
|
+
is a single letter (C, I, W) or an everyday word (In, La) neither triggers an
|
|
101
|
+
expansion in a query nor is added to a search: the agent's "In" or "La" core
|
|
102
|
+
term still gets the name. ``match``: terms that trigger an expansion in a
|
|
103
|
+
query; ``add``: terms that may be searched for (both default to all terms)."""
|
|
104
|
+
out = []
|
|
105
|
+
for symbol, name in text.elements():
|
|
106
|
+
loose = len(symbol) == 1 or symbol.lower() in AMBIGUOUS_SYMBOLS
|
|
107
|
+
out.append({"id": f"element:{symbol}", "terms": [symbol, name], "status": "active",
|
|
108
|
+
"source": "builtin", "builtin": "element",
|
|
109
|
+
"match": [] if loose else [symbol], "add": [name] if loose else [symbol, name]})
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def disabled() -> set[str]:
|
|
114
|
+
"""Built-in entries the user switched off: "element:<symbol>", "spelling:<American word>"."""
|
|
115
|
+
return set(_data().get("disabled", []))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def set_builtin(key: str, enabled: bool) -> None:
|
|
119
|
+
"""Switch a built-in element or spelling on or off."""
|
|
120
|
+
kind, _, word = key.partition(":")
|
|
121
|
+
if kind == "element" and f"element:{word}" not in {e["id"] for e in elements()}:
|
|
122
|
+
raise KeyError("无法找到这个元素")
|
|
123
|
+
if kind == "spelling":
|
|
124
|
+
from .spelling import spelling_pairs
|
|
125
|
+
if word not in {us for us, _ in spelling_pairs(word, limit=1)}:
|
|
126
|
+
raise KeyError("内置拼写表里没有这个词")
|
|
127
|
+
elif kind != "element":
|
|
128
|
+
raise KeyError("无法找到这一条")
|
|
129
|
+
with _updating() as data:
|
|
130
|
+
off = [k for k in data["disabled"] if k != key]
|
|
131
|
+
data["disabled"] = off if enabled else off + [key]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def builtin(query: str = "", limit: int = 50) -> dict[str, Any]:
|
|
135
|
+
"""The built-in entries for the app: elements matching ``query`` (all when it is
|
|
136
|
+
empty), spellings of the words in it, and what is switched off."""
|
|
137
|
+
from .spelling import spelling_pairs, to_us
|
|
138
|
+
off = disabled()
|
|
139
|
+
q = query.strip().lower()
|
|
140
|
+
qs = {q, to_us(q)} # "aluminium" finds Al
|
|
141
|
+
els = [{"id": e["id"], "terms": e["terms"], "disabled": e["id"] in off} for e in elements()
|
|
142
|
+
if not q or any(s in t.lower() for s in qs for t in e["terms"])]
|
|
143
|
+
els.sort(key=lambda e: not qs & {t.lower() for t in e["terms"]}) # "al": Al before cobalt
|
|
144
|
+
spell = [{"id": f"spelling:{us}", "terms": [us, *others], "disabled": f"spelling:{us}" in off}
|
|
145
|
+
for us, others in spelling_pairs(q, limit)] if q else []
|
|
146
|
+
off_spell = [{"id": k, "terms": [k.split(":", 1)[1]], "disabled": True}
|
|
147
|
+
for k in sorted(off) if k.startswith("spelling:")]
|
|
148
|
+
return {"elements": els, "spelling": spell, "spelling_disabled": off_spell}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# --------------------------------------------------------------------------- normalizing
|
|
152
|
+
|
|
153
|
+
def normalize(term: str) -> str:
|
|
154
|
+
"""One spelling for comparing terms: lower case, American spelling, dashes and
|
|
155
|
+
spaces alike, the last word singular ("Metal-matrix composites" ->
|
|
156
|
+
"metal matrix composite")."""
|
|
157
|
+
words = text.plain(term).split()
|
|
158
|
+
if words:
|
|
159
|
+
words[-1] = text.singular(words[-1])
|
|
160
|
+
return " ".join(words)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _clean(terms: list[str]) -> list[str]:
|
|
164
|
+
out: list[str] = []
|
|
165
|
+
for t in terms or []:
|
|
166
|
+
t = " ".join(str(t).split())
|
|
167
|
+
if t and normalize(t) not in {normalize(o) for o in out}:
|
|
168
|
+
out.append(t)
|
|
169
|
+
return out
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# --------------------------------------------------------------------------- editing
|
|
173
|
+
|
|
174
|
+
class _updating:
|
|
175
|
+
"""``store.update`` of the glossary file, created when it is not there yet."""
|
|
176
|
+
|
|
177
|
+
def __enter__(self) -> dict[str, Any]:
|
|
178
|
+
p = path()
|
|
179
|
+
if not p.exists():
|
|
180
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
181
|
+
store.write_json(p, {"entries": [], "dismissed": []}, indent=1)
|
|
182
|
+
self._cm = store.update(p, indent=1)
|
|
183
|
+
data = self._cm.__enter__()
|
|
184
|
+
data.setdefault("entries", [])
|
|
185
|
+
data.setdefault("dismissed", [])
|
|
186
|
+
data.setdefault("disabled", [])
|
|
187
|
+
return data
|
|
188
|
+
|
|
189
|
+
def __exit__(self, *exc: Any) -> bool | None:
|
|
190
|
+
return self._cm.__exit__(*exc)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _new(terms: list[str], source: str, note: str = "", papers: int | None = None) -> dict[str, Any]:
|
|
194
|
+
e = {"id": uuid.uuid4().hex[:8], "terms": terms, "status": "active" if source == "user" else "suggested",
|
|
195
|
+
"source": source, "note": str(note or "").strip(), "added": time.strftime("%Y-%m-%d")}
|
|
196
|
+
if papers:
|
|
197
|
+
e["papers"] = papers
|
|
198
|
+
return e
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def add(terms: list[str], note: str = "", source: str = "user") -> dict[str, Any]:
|
|
202
|
+
"""A new entry; one the agent adds waits for the user's confirmation."""
|
|
203
|
+
terms = _clean(terms)
|
|
204
|
+
if len(terms) < 2:
|
|
205
|
+
raise ValueError("无法添加:一条术语至少要有两种写法")
|
|
206
|
+
with _updating() as data:
|
|
207
|
+
same = {normalize(t) for t in terms}
|
|
208
|
+
for e in data["entries"]:
|
|
209
|
+
if {normalize(t) for t in e["terms"]} == same:
|
|
210
|
+
raise ValueError("无法添加:术语表里已有这一条")
|
|
211
|
+
entry = _new(terms, source, note)
|
|
212
|
+
data["entries"].append(entry)
|
|
213
|
+
return entry
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def update(entry_id: str, *, terms: list[str] | None = None, note: str | None = None,
|
|
217
|
+
confirm: bool = False) -> dict[str, Any]:
|
|
218
|
+
"""Change an entry; ``confirm`` turns a suggestion into an entry in use."""
|
|
219
|
+
with _updating() as data:
|
|
220
|
+
e = _find(data, entry_id)
|
|
221
|
+
if terms is not None:
|
|
222
|
+
terms = _clean(terms)
|
|
223
|
+
if len(terms) < 2:
|
|
224
|
+
raise ValueError("无法保存:一条术语至少要有两种写法")
|
|
225
|
+
e["terms"] = terms
|
|
226
|
+
if note is not None:
|
|
227
|
+
e["note"] = str(note).strip()
|
|
228
|
+
if confirm:
|
|
229
|
+
e["status"] = "active"
|
|
230
|
+
return dict(e)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def delete(entry_id: str) -> None:
|
|
234
|
+
"""Remove an entry; a deleted suggestion is not suggested again."""
|
|
235
|
+
with _updating() as data:
|
|
236
|
+
e = _find(data, entry_id)
|
|
237
|
+
data["entries"].remove(e)
|
|
238
|
+
if e.get("status") == "suggested" and len(e["terms"]) == 2:
|
|
239
|
+
pair = [normalize(t) for t in e["terms"]]
|
|
240
|
+
if pair not in data["dismissed"]:
|
|
241
|
+
data["dismissed"].append(pair)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _find(data: dict[str, Any], entry_id: str) -> dict[str, Any]:
|
|
245
|
+
for e in data["entries"]:
|
|
246
|
+
if e.get("id") == entry_id:
|
|
247
|
+
return e
|
|
248
|
+
raise KeyError("无法找到这条术语:它可能已被删除")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# --------------------------------------------------------------------------- expanding
|
|
252
|
+
|
|
253
|
+
def _abbreviation(term: str) -> bool:
|
|
254
|
+
""""PDC", "SiC", "EMW" and element symbols: matched with case, so "Al" is not "al" in a sentence."""
|
|
255
|
+
return " " not in term and (sum(ch.isupper() for ch in term) >= 2 or term in text.element_symbols())
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
_D = re.escape(text.DASHES) # any dash, for the patterns below
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@lru_cache(maxsize=2048)
|
|
262
|
+
def _pattern(term: str) -> re.Pattern[str]:
|
|
263
|
+
"""``term`` as a whole word or phrase, written any way ``normalize`` treats as the
|
|
264
|
+
same: a dash or a space ("polymer derived"), British spelling, the last word
|
|
265
|
+
plural ("PDCs", "polymer-derived ceramics")."""
|
|
266
|
+
from .spelling import variants
|
|
267
|
+
if _abbreviation(term):
|
|
268
|
+
return re.compile(rf"(?<![\w{_D}]){re.escape(term)}s?(?![\w{_D}])")
|
|
269
|
+
words = [w for w in re.split(rf"[\s{_D}]+", term) if w]
|
|
270
|
+
alts = [variants(w) for w in words]
|
|
271
|
+
last = words[-1] if words else ""
|
|
272
|
+
if len(last) > 2 and last[-1:].lower() == "y" and last[-2:-1].lower() not in "aeiou":
|
|
273
|
+
alts[-1] += [last[:-1] + "ies"]
|
|
274
|
+
body = rf"[\s{_D}]+".join("(?:" + "|".join(re.escape(a) for a in alt) + ")" for alt in alts)
|
|
275
|
+
return re.compile(rf"(?<![\w{_D}]){body}(?:s|es)?(?![\w{_D}])", re.I)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _has_other_concepts(query: str, span: tuple[int, int]) -> bool:
|
|
279
|
+
"""Something besides the matched term that keeps an expansion on topic."""
|
|
280
|
+
rest = query[:span[0]] + " " + query[span[1]:]
|
|
281
|
+
return any(len(w) >= 3 and w.lower() not in text.STOPWORDS for w in re.findall(r"[A-Za-z0-9]+", rest))
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def other_spelling(query: str) -> tuple[str, list[tuple[str, str]]]:
|
|
285
|
+
"""``query`` with each word in its other spelling (British words American,
|
|
286
|
+
American words in their first British spelling), and the words changed;
|
|
287
|
+
"" when no word has another spelling. Abbreviations (PDC, SiC) stay."""
|
|
288
|
+
from .spelling import to_us, variants
|
|
289
|
+
changed: list[tuple[str, str]] = []
|
|
290
|
+
|
|
291
|
+
def swap(m: re.Match[str]) -> str:
|
|
292
|
+
w = m.group(0)
|
|
293
|
+
if not (w.islower() or w.istitle()):
|
|
294
|
+
return w
|
|
295
|
+
low = w.lower()
|
|
296
|
+
us = to_us(low)
|
|
297
|
+
other = us if us != low else next(iter(variants(low)[1:]), low)
|
|
298
|
+
if other == low:
|
|
299
|
+
return w
|
|
300
|
+
changed.append((w, other))
|
|
301
|
+
return other.capitalize() if w.istitle() else other
|
|
302
|
+
out = re.sub(r"[A-Za-z]+", swap, query)
|
|
303
|
+
return (out if changed else ""), changed
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def query_variants(query: str, use: list[dict[str, Any]]) -> tuple[list[str], list[dict[str, Any]]]:
|
|
307
|
+
"""Other wordings of a keyword query (at most MAX_VARIANTS): first the query in
|
|
308
|
+
its other spelling (OpenAlex matches "aluminum" and "aluminium" separately),
|
|
309
|
+
then one term swapped for another of its entry; and what was expanded or left
|
|
310
|
+
alone ("alone": the query is that term and nothing else)."""
|
|
311
|
+
variants: list[str] = []
|
|
312
|
+
used: list[dict[str, Any]] = []
|
|
313
|
+
spelled, changed = other_spelling(query)
|
|
314
|
+
if spelled:
|
|
315
|
+
variants.append(spelled)
|
|
316
|
+
used += [{"term": w, "with": [o], "query": query, "builtin": "spelling"} for w, o in changed]
|
|
317
|
+
for e in use:
|
|
318
|
+
for t in e.get("match", e["terms"]):
|
|
319
|
+
m = _pattern(t).search(query)
|
|
320
|
+
if not m:
|
|
321
|
+
continue
|
|
322
|
+
others = [o for o in e.get("add", e["terms"]) if o != t and not _pattern(o).search(query)]
|
|
323
|
+
if not others:
|
|
324
|
+
break
|
|
325
|
+
report = {"id": e["id"], "term": m.group(0), "with": others, "query": query}
|
|
326
|
+
if e.get("builtin"):
|
|
327
|
+
report["builtin"] = e["builtin"]
|
|
328
|
+
if not _has_other_concepts(query, m.span()):
|
|
329
|
+
used.append({**report, "alone": True})
|
|
330
|
+
break
|
|
331
|
+
for o in others:
|
|
332
|
+
v = query[:m.start()] + o + query[m.end():]
|
|
333
|
+
if len(variants) < MAX_VARIANTS and v not in variants:
|
|
334
|
+
variants.append(v)
|
|
335
|
+
used.append(report)
|
|
336
|
+
break
|
|
337
|
+
return variants, used
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def alternatives(word: str, use: list[dict[str, Any]]) -> tuple[list[str], list[str]]:
|
|
341
|
+
"""The other terms of the entries ``word`` is one of (a word of a deep search's
|
|
342
|
+
core term), and the built-in kinds among those entries ("element")."""
|
|
343
|
+
out: list[str] = []
|
|
344
|
+
kinds: list[str] = []
|
|
345
|
+
for e in use:
|
|
346
|
+
if any(_pattern(t).fullmatch(word) for t in e["terms"]):
|
|
347
|
+
more = [t for t in e.get("add", e["terms"]) if not _pattern(t).fullmatch(word)]
|
|
348
|
+
out += more
|
|
349
|
+
if more and e.get("builtin"):
|
|
350
|
+
kinds.append(e["builtin"])
|
|
351
|
+
return list(dict.fromkeys(out)), list(dict.fromkeys(kinds))
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# --------------------------------------------------------------------------- discovering
|
|
355
|
+
|
|
356
|
+
# Characterization methods nearly every materials abstract spells out: expanding
|
|
357
|
+
# them finds nothing new, and suggesting them only gives the user more to click.
|
|
358
|
+
_COMMON = set("SEM TEM HRTEM STEM XRD XPS FTIR FT-IR IR NMR TGA DTA DSC TG EDS EDX EDXS BET AFM SAED EIS "
|
|
359
|
+
"UV UV-vis CV GCD EPR ESR XAS XANES EXAFS ICP ICP-OES ICP-MS".split())
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _usable(short: str, long: str) -> bool:
|
|
363
|
+
words = long.split()
|
|
364
|
+
return (2 <= len(short) <= 12 and any(ch.isalpha() for ch in short) and _short_key(short) not in _COMMON
|
|
365
|
+
and 1 <= len(words) <= 8 and not re.search(r"[()\[\]]", long) and words[0].lower() not in text.STOPWORDS)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _short_key(short: str) -> str:
|
|
369
|
+
""""PDCs" and "PDC" are one abbreviation."""
|
|
370
|
+
if len(short) > 2 and short.endswith("s") and sum(ch.isupper() for ch in short[:-1]) >= 2:
|
|
371
|
+
return short[:-1]
|
|
372
|
+
return short
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _meaning_key(norm_long: str) -> str:
|
|
376
|
+
"""Wordings of one meaning ("scanning electron microscopy" / "microscope"): word stems."""
|
|
377
|
+
return " ".join(text.stem(w) for w in norm_long.split())
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def discover(texts: list[str]) -> dict[str, dict[str, dict[str, Any]]]:
|
|
381
|
+
""""Long form (SF)" pairs in ``texts`` (abstracts): short form -> meaning ->
|
|
382
|
+
{"text": its most common wording, "papers": how many texts write it}."""
|
|
383
|
+
from abbreviations import schwartz_hearst
|
|
384
|
+
seen: dict[str, dict[str, Counter]] = {}
|
|
385
|
+
for abstract in texts:
|
|
386
|
+
try:
|
|
387
|
+
pairs = schwartz_hearst.extract_abbreviation_definition_pairs(doc_text=abstract or "")
|
|
388
|
+
except Exception: # an odd abstract must not stop the rest
|
|
389
|
+
continue
|
|
390
|
+
for short, long in pairs.items():
|
|
391
|
+
long = text.ascii_dashes(" ".join(long.split()))
|
|
392
|
+
if _usable(short, long):
|
|
393
|
+
key = _meaning_key(normalize(long))
|
|
394
|
+
seen.setdefault(_short_key(short), {}).setdefault(key, Counter())[long.lower()] += 1
|
|
395
|
+
return {s: {m: {"text": c.most_common(1)[0][0], "papers": sum(c.values())} for m, c in longs.items()}
|
|
396
|
+
for s, longs in seen.items()}
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def suggest_from(texts: list[str], min_papers: int = MIN_PAPERS) -> list[dict[str, Any]]:
|
|
400
|
+
"""Add the pairs ``texts`` write often enough as suggestions; returns the new ones."""
|
|
401
|
+
found = discover(texts)
|
|
402
|
+
if not found:
|
|
403
|
+
return []
|
|
404
|
+
new: list[dict[str, Any]] = []
|
|
405
|
+
with _updating() as data:
|
|
406
|
+
def key(a: str, b: str) -> tuple[str, str]:
|
|
407
|
+
return normalize(_short_key(a)), _meaning_key(normalize(b))
|
|
408
|
+
|
|
409
|
+
known = {key(a, b) for e in data["entries"] + elements() for a in e["terms"] for b in e["terms"] if a != b}
|
|
410
|
+
dismissed = {key(*p) for p in data["dismissed"]}
|
|
411
|
+
for short, longs in sorted(found.items()):
|
|
412
|
+
several = sum(1 for i in longs.values() if i["papers"] >= min_papers) > 1
|
|
413
|
+
for meaning, info in sorted(longs.items(), key=lambda kv: -kv[1]["papers"]):
|
|
414
|
+
pair = key(short, info["text"])
|
|
415
|
+
if info["papers"] < min_papers or pair in known or pair in dismissed:
|
|
416
|
+
continue
|
|
417
|
+
note = (f"在 {info['papers']} 篇摘要中这样写" +
|
|
418
|
+
(";这个缩写还有其他全称,留下对的那条" if several else ""))
|
|
419
|
+
e = _new([short, info["text"]], "auto", note, papers=info["papers"])
|
|
420
|
+
data["entries"].append(e)
|
|
421
|
+
new.append(e)
|
|
422
|
+
return new
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""British and American spellings, so a search finds both.
|
|
2
|
+
|
|
3
|
+
Papers write "aluminium" or "aluminum", "fibre" or "fiber", "characterised"
|
|
4
|
+
or "characterized". The spellings come from VarCon (Kevin Atkinson and
|
|
5
|
+
Benjamin Titze, part of SCOWL; see data/VARCON-LICENSE.txt), built into two
|
|
6
|
+
tables by scripts/build_spelling_table.py:
|
|
7
|
+
|
|
8
|
+
* ``to_us`` maps text to the preferred American spelling before words are
|
|
9
|
+
compared (query matching, ranking features);
|
|
10
|
+
* ``variants`` gives the spellings of one word worth searching for, since
|
|
11
|
+
OpenAlex matches "aluminum" and "aluminium" separately.
|
|
12
|
+
|
|
13
|
+
Abbreviations, synonyms and element symbols live in ``glossary``; the user
|
|
14
|
+
may switch a word's spellings off there ("spelling:<American word>").
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import gzip
|
|
20
|
+
import re
|
|
21
|
+
from functools import lru_cache
|
|
22
|
+
from importlib.resources import files
|
|
23
|
+
|
|
24
|
+
_WORD = re.compile(r"[a-z]+")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@lru_cache(maxsize=1)
|
|
28
|
+
def _tables() -> tuple[dict[str, str], dict[str, list[str]]]:
|
|
29
|
+
data = files("litloop") / "data"
|
|
30
|
+
with gzip.open(data.joinpath("spelling_us.tsv.gz").open("rb"), "rt", encoding="utf-8") as fh:
|
|
31
|
+
to_us_table = dict(line.rstrip("\n").split("\t", 1) for line in fh if "\t" in line)
|
|
32
|
+
search: dict[str, list[str]] = {}
|
|
33
|
+
with gzip.open(data.joinpath("spelling_search.tsv.gz").open("rb"), "rt", encoding="utf-8") as fh:
|
|
34
|
+
for line in fh:
|
|
35
|
+
word, *others = line.rstrip("\n").split("\t")
|
|
36
|
+
search[word] = others
|
|
37
|
+
return to_us_table, search
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_OFF: list = [None, frozenset()] # (glossary file, its mtime), American words switched off
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _off() -> frozenset[str]:
|
|
44
|
+
"""American words whose other spellings the user switched off (the glossary's
|
|
45
|
+
``disabled``), read again when the file changes."""
|
|
46
|
+
from . import glossary
|
|
47
|
+
p = glossary.path()
|
|
48
|
+
try:
|
|
49
|
+
stamp = (str(p), p.stat().st_mtime_ns)
|
|
50
|
+
except OSError:
|
|
51
|
+
return frozenset()
|
|
52
|
+
if _OFF[0] != stamp:
|
|
53
|
+
_OFF[:] = [stamp, frozenset(k.split(":", 1)[1] for k in glossary.disabled() if k.startswith("spelling:"))]
|
|
54
|
+
return _OFF[1]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def to_us(text: str) -> str:
|
|
58
|
+
"""Lower-case ``text`` with British spellings turned American."""
|
|
59
|
+
table = _tables()[0]
|
|
60
|
+
off = _off()
|
|
61
|
+
|
|
62
|
+
def us(m: re.Match[str]) -> str:
|
|
63
|
+
w = table.get(m.group(), m.group())
|
|
64
|
+
return m.group() if w in off else w
|
|
65
|
+
return _WORD.sub(us, text.lower())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def variants(word: str) -> list[str]:
|
|
69
|
+
"""The spellings of ``word`` worth searching for, the given one first."""
|
|
70
|
+
to_us_table, search = _tables()
|
|
71
|
+
low = word.lower()
|
|
72
|
+
american = to_us_table.get(low, low)
|
|
73
|
+
if american in _off():
|
|
74
|
+
return [word]
|
|
75
|
+
out = [word]
|
|
76
|
+
for w in [american, *search.get(american, [])]:
|
|
77
|
+
if w != low and w not in out:
|
|
78
|
+
out.append(w)
|
|
79
|
+
return out
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def spelling_pairs(query: str, limit: int = 50) -> list[tuple[str, list[str]]]:
|
|
83
|
+
"""(American spelling, its British spellings) for words starting with ``query``
|
|
84
|
+
in any of their spellings, for looking them up in the app."""
|
|
85
|
+
q = query.strip().lower()
|
|
86
|
+
found = [(us, others) for us, others in _tables()[1].items()
|
|
87
|
+
if others and (us.startswith(q) or any(o.startswith(q) for o in others))]
|
|
88
|
+
found.sort(key=lambda p: (q != p[0] and q not in p[1], len(p[0]), p[0])) # the word itself, then shortest
|
|
89
|
+
return found[:limit]
|
litloop/terms/text.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Text normalisation shared by query matching, ranking, the glossary and deduplication.
|
|
2
|
+
|
|
3
|
+
One set of rules, so that the same two wordings count as the same everywhere
|
|
4
|
+
(2026-10-25: there were three stop-word lists, two plural rules, two ways of
|
|
5
|
+
stemming, four sets of dashes and four paper keys):
|
|
6
|
+
|
|
7
|
+
* ``plain`` / ``squash`` / ``ascii_dashes``: case, every kind of dash, American spelling;
|
|
8
|
+
* ``singular`` / ``stem``: the plural of a word, and the crude prefix stem used
|
|
9
|
+
for substring tests and word features;
|
|
10
|
+
* ``STOPWORDS``: words that carry no topic in a query; ``RANKING_STOPWORDS``,
|
|
11
|
+
the shorter list the ranking model's word features drop;
|
|
12
|
+
* ``element_symbols``: the chemical elements (data/elements.tsv, from PubChem);
|
|
13
|
+
* ``paper_key`` / ``title_key``: when two records are the same paper.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from functools import lru_cache
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
# Hyphen, non-breaking hyphen, figure dash, en dash, em dash, horizontal bar,
|
|
23
|
+
# minus sign, small and full-width hyphen-minus, and the ASCII hyphen-minus.
|
|
24
|
+
DASHES = "‐‑‒–—―−﹣--"
|
|
25
|
+
_DASH = re.compile("[" + re.escape(DASHES) + "]")
|
|
26
|
+
_DASH_RUN = re.compile("[" + re.escape(DASHES) + "]+")
|
|
27
|
+
_SQUASH = re.compile(r"[\s/" + re.escape(DASHES) + "]+")
|
|
28
|
+
|
|
29
|
+
STOPWORDS = frozenset(
|
|
30
|
+
"a an and the of in on for to with by from at as or via its their into using based under over between "
|
|
31
|
+
"is are was were be this that these which we our than also "
|
|
32
|
+
"study studies effect effects analysis review".split()
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# The ranking model's word features (deep_search ``_tokens``) drop only these.
|
|
36
|
+
# Not STOPWORDS: on SYNERGY, dropping review/study/effect/analysis/using/based
|
|
37
|
+
# from the features made Leenaars_2019 and Appenzeller-Herzog_2019 worse, and
|
|
38
|
+
# adding just via/under/over/between changed the ranking too (DECISIONS,
|
|
39
|
+
# 2026-10-25). Changing this list changes the ranking: run
|
|
40
|
+
# scripts/eval_synergy.py before and after.
|
|
41
|
+
RANKING_STOPWORDS = frozenset(
|
|
42
|
+
"the and of in for with by on to a an is are was were from as at this that these be or its "
|
|
43
|
+
"their which we our into than also".split()
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def ascii_dashes(text: str) -> str:
|
|
48
|
+
"""Every kind of dash as "-" (DOIs and titles copied from PDFs)."""
|
|
49
|
+
return _DASH.sub("-", text or "")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def plain(text: str) -> str:
|
|
53
|
+
"""Lower case, American spelling, every kind of dash read as a space."""
|
|
54
|
+
from .spelling import to_us
|
|
55
|
+
return _DASH_RUN.sub(" ", to_us(text or ""))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def squash(text: str) -> str:
|
|
59
|
+
"""Lower case with spaces, dashes and slashes removed: "Polymer–derived" -> "polymerderived"."""
|
|
60
|
+
return _SQUASH.sub("", (text or "").lower())
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def singular(word: str) -> str:
|
|
64
|
+
"""Crude singular of a lower-case word: "ceramics" -> "ceramic", "properties" -> "property";
|
|
65
|
+
"glass", "virus" and "analysis" stay."""
|
|
66
|
+
if len(word) > 4 and word.endswith("ies"):
|
|
67
|
+
return word[:-3] + "y"
|
|
68
|
+
if len(word) > 3 and word.endswith("s") and not word.endswith(("ss", "us", "is")):
|
|
69
|
+
return word[:-1]
|
|
70
|
+
return word
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def stem(word: str) -> str:
|
|
74
|
+
"""Crude stem, the first 7 letters: "infiltration" and "infiltrated" -> "infiltr"."""
|
|
75
|
+
return word[:7]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@lru_cache(maxsize=1)
|
|
79
|
+
def elements() -> tuple[tuple[str, str], ...]:
|
|
80
|
+
"""(symbol, American name) of the 118 elements."""
|
|
81
|
+
from importlib.resources import files
|
|
82
|
+
rows = (files("litloop") / "data" / "elements.tsv").read_text(encoding="utf-8").splitlines()
|
|
83
|
+
return tuple(tuple(line.split("\t")) for line in rows if line and not line.startswith("#")) # type: ignore[misc]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@lru_cache(maxsize=1)
|
|
87
|
+
def element_symbols() -> frozenset[str]:
|
|
88
|
+
return frozenset(s for s, _ in elements())
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def is_chemical_system(word: str) -> bool:
|
|
92
|
+
""""Al-Si-Mg", "Si-B-C-N": element symbols joined by dashes (papers list them in any order)."""
|
|
93
|
+
parts = _DASH.split(word)
|
|
94
|
+
return len(parts) > 1 and all(p in element_symbols() for p in parts)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def title_key(title: Any) -> str:
|
|
98
|
+
"""A title's letters and digits, lower case: the same title from two sources."""
|
|
99
|
+
return re.sub(r"[^a-z0-9]+", "", str(title or "").lower())
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def paper_key(rec: dict[str, Any]) -> str:
|
|
103
|
+
"""The key that says two records are one paper: the DOI, or else the title
|
|
104
|
+
("title:<letters and digits>"). The app's ``paperKey`` (app/src/lib/api.ts) mirrors it."""
|
|
105
|
+
from ..identifiers import canonical_doi
|
|
106
|
+
return canonical_doi(rec.get("doi")) or "title:" + title_key(rec.get("title"))
|
litloop/tor/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Tor proxy management and embedded Tor daemon support."""
|