litloop 1.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. litloop/__init__.py +89 -0
  2. litloop/__main__.py +6 -0
  3. litloop/_source_hygiene.py +158 -0
  4. litloop/agents/__init__.py +13 -0
  5. litloop/agents/client.py +164 -0
  6. litloop/agents/runner.py +123 -0
  7. litloop/agents/system.py +54 -0
  8. litloop/agents/tools.py +142 -0
  9. litloop/app_server.py +8 -0
  10. litloop/browser/__init__.py +1 -0
  11. litloop/browser/backend.py +1009 -0
  12. litloop/browser/camofox_api.py +368 -0
  13. litloop/browser/chrome_copy.py +136 -0
  14. litloop/browser/cloakbrowser_compat.py +72 -0
  15. litloop/browser/cookies.py +347 -0
  16. litloop/browser/engine.py +965 -0
  17. litloop/browser/login.py +636 -0
  18. litloop/browser/pdf.py +263 -0
  19. litloop/browser/profile_health.py +167 -0
  20. litloop/browser/runtime.py +210 -0
  21. litloop/cache.py +104 -0
  22. litloop/citations/__init__.py +1 -0
  23. litloop/citations/bib_import.py +102 -0
  24. litloop/citations/bibtex_crossref.py +143 -0
  25. litloop/citations/formats.py +225 -0
  26. litloop/citations/paper_list.py +200 -0
  27. litloop/citations/session_export.py +94 -0
  28. litloop/cli/__init__.py +12 -0
  29. litloop/cli/_utils.py +267 -0
  30. litloop/cli/commands.py +1098 -0
  31. litloop/cli/doctor.py +141 -0
  32. litloop/cli/main.py +1173 -0
  33. litloop/cli/web.py +190 -0
  34. litloop/config.py +413 -0
  35. litloop/data/VARCON-LICENSE.txt +62 -0
  36. litloop/data/elements.tsv +119 -0
  37. litloop/data/institutional_identity_policy.json +165 -0
  38. litloop/data/publisher_access_catalog.json +969 -0
  39. litloop/data/publisher_carsi.json +105 -0
  40. litloop/data/spelling_search.tsv.gz +0 -0
  41. litloop/data/spelling_us.tsv.gz +0 -0
  42. litloop/data/webvpn.dat +0 -0
  43. litloop/data/webvpn.json +603 -0
  44. litloop/db.py +1922 -0
  45. litloop/deps.py +128 -0
  46. litloop/desktop/__init__.py +1 -0
  47. litloop/desktop/agent_runner.py +537 -0
  48. litloop/desktop/download_jobs.py +571 -0
  49. litloop/desktop/error_text.py +192 -0
  50. litloop/desktop/server.py +999 -0
  51. litloop/desktop/settings.py +552 -0
  52. litloop/desktop/translate.py +103 -0
  53. litloop/diagnostics.py +126 -0
  54. litloop/download/__init__.py +1 -0
  55. litloop/download/doi_check.py +44 -0
  56. litloop/download/fetcher.py +975 -0
  57. litloop/download/markdown.py +130 -0
  58. litloop/download/pdf.py +454 -0
  59. litloop/download/pipeline.py +1166 -0
  60. litloop/download/progress.py +312 -0
  61. litloop/download/rename.py +211 -0
  62. litloop/extractors/__init__.py +0 -0
  63. litloop/extractors/html_extractor.py +31 -0
  64. litloop/extractors/pdf_extractor.py +151 -0
  65. litloop/extractors/publisher_adapters/__init__.py +0 -0
  66. litloop/extractors/publisher_adapters/acs.py +167 -0
  67. litloop/extractors/publisher_adapters/elsevier.py +178 -0
  68. litloop/extractors/publisher_adapters/generic.py +191 -0
  69. litloop/extractors/publisher_adapters/nature.py +204 -0
  70. litloop/extractors/publisher_adapters/rsc.py +154 -0
  71. litloop/extractors/publisher_adapters/tandfonline.py +154 -0
  72. litloop/extractors/publisher_adapters/wiley.py +160 -0
  73. litloop/http_utils.py +57 -0
  74. litloop/identifiers.py +134 -0
  75. litloop/institutional/__init__.py +1 -0
  76. litloop/institutional/auth.py +576 -0
  77. litloop/institutional/carsi_login.py +569 -0
  78. litloop/institutional/config_adapter.py +63 -0
  79. litloop/institutional/proxy_check.py +98 -0
  80. litloop/institutional/remote_assist.py +274 -0
  81. litloop/institutional/schools.py +102 -0
  82. litloop/institutional/session_heal.py +73 -0
  83. litloop/institutional/session_store.py +105 -0
  84. litloop/log.py +47 -0
  85. litloop/models.py +269 -0
  86. litloop/network.py +417 -0
  87. litloop/optional.py +310 -0
  88. litloop/publishers/__init__.py +1 -0
  89. litloop/publishers/access.py +232 -0
  90. litloop/publishers/pdf_router.py +451 -0
  91. litloop/publishers/profiles.py +899 -0
  92. litloop/publishers/strategies/__init__.py +122 -0
  93. litloop/publishers/strategies/_core.py +2340 -0
  94. litloop/publishers/strategies/_elsevier_api.py +578 -0
  95. litloop/publishers/strategies/acm.py +27 -0
  96. litloop/publishers/strategies/acs.py +42 -0
  97. litloop/publishers/strategies/aip.py +27 -0
  98. litloop/publishers/strategies/aps.py +29 -0
  99. litloop/publishers/strategies/asce.py +28 -0
  100. litloop/publishers/strategies/base.py +108 -0
  101. litloop/publishers/strategies/copernicus.py +24 -0
  102. litloop/publishers/strategies/elsevier.py +66 -0
  103. litloop/publishers/strategies/generic.py +39 -0
  104. litloop/publishers/strategies/ieee.py +28 -0
  105. litloop/publishers/strategies/iop.py +27 -0
  106. litloop/publishers/strategies/nature.py +27 -0
  107. litloop/publishers/strategies/oxford.py +27 -0
  108. litloop/publishers/strategies/registry.py +83 -0
  109. litloop/publishers/strategies/royalsociety.py +28 -0
  110. litloop/publishers/strategies/rsc.py +27 -0
  111. litloop/publishers/strategies/sage.py +30 -0
  112. litloop/publishers/strategies/science.py +27 -0
  113. litloop/publishers/strategies/springer.py +34 -0
  114. litloop/publishers/strategies/tandfonline.py +32 -0
  115. litloop/publishers/strategies/wiley.py +32 -0
  116. litloop/publishers/supplementary.py +219 -0
  117. litloop/quota.py +202 -0
  118. litloop/search/__init__.py +1 -0
  119. litloop/search/abstracts.py +291 -0
  120. litloop/search/deep.py +1216 -0
  121. litloop/search/quick.py +1102 -0
  122. litloop/search/scansci_find.py +410 -0
  123. litloop/search/screening.py +225 -0
  124. litloop/search/title_to_doi.py +163 -0
  125. litloop/search/watches.py +124 -0
  126. litloop/server.py +1731 -0
  127. litloop/sources/__init__.py +1339 -0
  128. litloop/sources/_batch.py +529 -0
  129. litloop/sources/_index.py +221 -0
  130. litloop/sources/_routes.py +290 -0
  131. litloop/sources/altcha.py +90 -0
  132. litloop/sources/arxiv.py +45 -0
  133. litloop/sources/attempt.py +91 -0
  134. litloop/sources/carsi.py +74 -0
  135. litloop/sources/core_api.py +46 -0
  136. litloop/sources/crossref.py +162 -0
  137. litloop/sources/elsevier_api.py +620 -0
  138. litloop/sources/elsevier_check.py +274 -0
  139. litloop/sources/europepmc.py +81 -0
  140. litloop/sources/ezproxy.py +317 -0
  141. litloop/sources/instsci.py +1295 -0
  142. litloop/sources/libgen.py +116 -0
  143. litloop/sources/nature.py +39 -0
  144. litloop/sources/oa_discovery.py +135 -0
  145. litloop/sources/openaire.py +122 -0
  146. litloop/sources/openalex.py +100 -0
  147. litloop/sources/publishers.py +684 -0
  148. litloop/sources/sage_cn.py +139 -0
  149. litloop/sources/scibban.py +10 -0
  150. litloop/sources/scihub.py +1379 -0
  151. litloop/sources/scihub_domains.py +203 -0
  152. litloop/sources/scihub_speed.py +187 -0
  153. litloop/sources/scoring.py +250 -0
  154. litloop/sources/semantic_scholar.py +146 -0
  155. litloop/sources/springer_tdm.py +172 -0
  156. litloop/sources/unpaywall.py +116 -0
  157. litloop/sources/vpnsci.py +38 -0
  158. litloop/store.py +264 -0
  159. litloop/templates/index.html +347 -0
  160. litloop/terms/__init__.py +1 -0
  161. litloop/terms/glossary.py +422 -0
  162. litloop/terms/spelling.py +89 -0
  163. litloop/terms/text.py +106 -0
  164. litloop/tor/__init__.py +1 -0
  165. litloop/tor/embedded.py +371 -0
  166. litloop/tor/proxy.py +110 -0
  167. litloop/updates.py +148 -0
  168. litloop/zotero/__init__.py +1 -0
  169. litloop/zotero/bridge.py +309 -0
  170. litloop/zotero/client.py +317 -0
  171. litloop-1.18.3.dist-info/METADATA +347 -0
  172. litloop-1.18.3.dist-info/RECORD +176 -0
  173. litloop-1.18.3.dist-info/WHEEL +5 -0
  174. litloop-1.18.3.dist-info/entry_points.txt +2 -0
  175. litloop-1.18.3.dist-info/licenses/LICENSE +197 -0
  176. litloop-1.18.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,422 @@
1
+ """The glossary: abbreviations and synonyms a search expands by itself.
2
+
3
+ Papers write "PDC", "polymer-derived ceramic" or "preceramic polymer" for the
4
+ same thing. Three sources, used together (``in_use``):
5
+
6
+ * the user's entries: only someone who knows the field can say these;
7
+ * the chemical elements, built in (``elements``: "Al" = "aluminum", from
8
+ PubChem, data/elements.tsv);
9
+ * British and American spellings, built in and applied word by word
10
+ (``spelling``, from VarCon).
11
+
12
+ The user may switch off any built-in element or spelling (``set_builtin``).
13
+ Kept in ``<litloop home>/glossary.json`` through ``store`` (shared by the app
14
+ and the agent)::
15
+
16
+ {"schema": 1,
17
+ "entries": [{"id": "3f9a1c2e", "terms": ["PDC", "polymer-derived ceramic"],
18
+ "status": "active" | "suggested", "source": "user" | "agent" | "auto",
19
+ "papers": 4, "note": "", "added": "2026-10-25"}],
20
+ "dismissed": [["pdc", "polycrystalline diamond compact"]],
21
+ "disabled": ["element:La", "spelling:aluminum"]}
22
+
23
+ How it is used (decided with the user, 2026-10-25, after looking at how
24
+ others do it):
25
+
26
+ * Every entry in use applies to every search; there is no per-topic scope.
27
+ A search is a few concepts joined by AND, so a wrong expansion (PDC as a
28
+ drill-bit cutter searched as "polymer-derived ceramic ... wear") finds next
29
+ to nothing rather than noise. A query that is the abbreviation alone has
30
+ nothing to hold it, so it is not expanded (``query_variants``).
31
+ * Each search reports what it expanded, as PubMed's Search Details do.
32
+ * Most entries need not be typed: ``discover`` finds "long form (SF)" pairs in
33
+ the abstracts a search has read, with the Schwartz-Hearst algorithm (the
34
+ ``abbreviations`` package), and ``suggest_from`` adds those seen in two or
35
+ more papers as suggestions the user confirms. Long forms are compared after
36
+ ``normalize`` (case, dashes, plurals, British spelling), so "metal matrix
37
+ composite" and "metal-matrix composites" are one meaning; a short form left
38
+ with several meanings (CMC: carboxymethyl cellulose, ceramic matrix
39
+ composite) gets one suggestion per meaning, and the user keeps the right one.
40
+ A suggestion the user deletes is remembered (``dismissed``) and not made again.
41
+ * The agent may add entries too; they wait for the user's confirmation.
42
+ """
43
+
44
+ from __future__ import annotations
45
+
46
+ import re
47
+ import time
48
+ import uuid
49
+ from collections import Counter
50
+ from functools import lru_cache
51
+ from pathlib import Path
52
+ from typing import Any
53
+
54
+ from .. import store
55
+ from . import text
56
+
57
+ MAX_VARIANTS = 3 # extra runs of one keyword query (own value)
58
+ MIN_PAPERS = 2 # a discovered pair is suggested once this many papers write it (own value)
59
+
60
+
61
+ def path() -> Path:
62
+ from ..config import DATA_DIR
63
+ return Path(DATA_DIR) / "glossary.json"
64
+
65
+
66
+ def _data() -> dict[str, Any]:
67
+ try:
68
+ data = store.read_json(path())
69
+ except (OSError, ValueError):
70
+ return {"entries": [], "dismissed": [], "disabled": []}
71
+ data.setdefault("entries", [])
72
+ data.setdefault("dismissed", [])
73
+ data.setdefault("disabled", [])
74
+ return data
75
+
76
+
77
+ def entries(status: str | None = None) -> list[dict[str, Any]]:
78
+ """All entries, or those with ``status`` ("active", "suggested")."""
79
+ out = [e for e in _data()["entries"] if isinstance(e, dict) and e.get("terms")]
80
+ return [e for e in out if status is None or e.get("status") == status]
81
+
82
+
83
+ def in_use() -> list[dict[str, Any]]:
84
+ """What a search expands with: the user's entries in use, then the built-in
85
+ elements the user has not switched off."""
86
+ off = disabled()
87
+ return entries("active") + [e for e in elements() if e["id"] not in off]
88
+
89
+
90
+ # --------------------------------------------------------------------------- built in
91
+
92
+ # Element symbols that are also everyday words (OpenAlex search ignores case): "La", "Ce"
93
+ # match French, "Er" German, "Eu" the EU, "In"/"As"/"Be"/"He"/"No" English.
94
+ AMBIGUOUS_SYMBOLS = {"as", "at", "be", "ce", "er", "es", "eu", "he", "in", "la", "no"}
95
+
96
+
97
+ @lru_cache(maxsize=1)
98
+ def elements() -> list[dict[str, Any]]:
99
+ """The chemical elements as entries. A symbol is matched with its case. One that
100
+ is a single letter (C, I, W) or an everyday word (In, La) neither triggers an
101
+ expansion in a query nor is added to a search: the agent's "In" or "La" core
102
+ term still gets the name. ``match``: terms that trigger an expansion in a
103
+ query; ``add``: terms that may be searched for (both default to all terms)."""
104
+ out = []
105
+ for symbol, name in text.elements():
106
+ loose = len(symbol) == 1 or symbol.lower() in AMBIGUOUS_SYMBOLS
107
+ out.append({"id": f"element:{symbol}", "terms": [symbol, name], "status": "active",
108
+ "source": "builtin", "builtin": "element",
109
+ "match": [] if loose else [symbol], "add": [name] if loose else [symbol, name]})
110
+ return out
111
+
112
+
113
+ def disabled() -> set[str]:
114
+ """Built-in entries the user switched off: "element:<symbol>", "spelling:<American word>"."""
115
+ return set(_data().get("disabled", []))
116
+
117
+
118
+ def set_builtin(key: str, enabled: bool) -> None:
119
+ """Switch a built-in element or spelling on or off."""
120
+ kind, _, word = key.partition(":")
121
+ if kind == "element" and f"element:{word}" not in {e["id"] for e in elements()}:
122
+ raise KeyError("无法找到这个元素")
123
+ if kind == "spelling":
124
+ from .spelling import spelling_pairs
125
+ if word not in {us for us, _ in spelling_pairs(word, limit=1)}:
126
+ raise KeyError("内置拼写表里没有这个词")
127
+ elif kind != "element":
128
+ raise KeyError("无法找到这一条")
129
+ with _updating() as data:
130
+ off = [k for k in data["disabled"] if k != key]
131
+ data["disabled"] = off if enabled else off + [key]
132
+
133
+
134
+ def builtin(query: str = "", limit: int = 50) -> dict[str, Any]:
135
+ """The built-in entries for the app: elements matching ``query`` (all when it is
136
+ empty), spellings of the words in it, and what is switched off."""
137
+ from .spelling import spelling_pairs, to_us
138
+ off = disabled()
139
+ q = query.strip().lower()
140
+ qs = {q, to_us(q)} # "aluminium" finds Al
141
+ els = [{"id": e["id"], "terms": e["terms"], "disabled": e["id"] in off} for e in elements()
142
+ if not q or any(s in t.lower() for s in qs for t in e["terms"])]
143
+ els.sort(key=lambda e: not qs & {t.lower() for t in e["terms"]}) # "al": Al before cobalt
144
+ spell = [{"id": f"spelling:{us}", "terms": [us, *others], "disabled": f"spelling:{us}" in off}
145
+ for us, others in spelling_pairs(q, limit)] if q else []
146
+ off_spell = [{"id": k, "terms": [k.split(":", 1)[1]], "disabled": True}
147
+ for k in sorted(off) if k.startswith("spelling:")]
148
+ return {"elements": els, "spelling": spell, "spelling_disabled": off_spell}
149
+
150
+
151
+ # --------------------------------------------------------------------------- normalizing
152
+
153
+ def normalize(term: str) -> str:
154
+ """One spelling for comparing terms: lower case, American spelling, dashes and
155
+ spaces alike, the last word singular ("Metal-matrix composites" ->
156
+ "metal matrix composite")."""
157
+ words = text.plain(term).split()
158
+ if words:
159
+ words[-1] = text.singular(words[-1])
160
+ return " ".join(words)
161
+
162
+
163
+ def _clean(terms: list[str]) -> list[str]:
164
+ out: list[str] = []
165
+ for t in terms or []:
166
+ t = " ".join(str(t).split())
167
+ if t and normalize(t) not in {normalize(o) for o in out}:
168
+ out.append(t)
169
+ return out
170
+
171
+
172
+ # --------------------------------------------------------------------------- editing
173
+
174
+ class _updating:
175
+ """``store.update`` of the glossary file, created when it is not there yet."""
176
+
177
+ def __enter__(self) -> dict[str, Any]:
178
+ p = path()
179
+ if not p.exists():
180
+ p.parent.mkdir(parents=True, exist_ok=True)
181
+ store.write_json(p, {"entries": [], "dismissed": []}, indent=1)
182
+ self._cm = store.update(p, indent=1)
183
+ data = self._cm.__enter__()
184
+ data.setdefault("entries", [])
185
+ data.setdefault("dismissed", [])
186
+ data.setdefault("disabled", [])
187
+ return data
188
+
189
+ def __exit__(self, *exc: Any) -> bool | None:
190
+ return self._cm.__exit__(*exc)
191
+
192
+
193
+ def _new(terms: list[str], source: str, note: str = "", papers: int | None = None) -> dict[str, Any]:
194
+ e = {"id": uuid.uuid4().hex[:8], "terms": terms, "status": "active" if source == "user" else "suggested",
195
+ "source": source, "note": str(note or "").strip(), "added": time.strftime("%Y-%m-%d")}
196
+ if papers:
197
+ e["papers"] = papers
198
+ return e
199
+
200
+
201
+ def add(terms: list[str], note: str = "", source: str = "user") -> dict[str, Any]:
202
+ """A new entry; one the agent adds waits for the user's confirmation."""
203
+ terms = _clean(terms)
204
+ if len(terms) < 2:
205
+ raise ValueError("无法添加:一条术语至少要有两种写法")
206
+ with _updating() as data:
207
+ same = {normalize(t) for t in terms}
208
+ for e in data["entries"]:
209
+ if {normalize(t) for t in e["terms"]} == same:
210
+ raise ValueError("无法添加:术语表里已有这一条")
211
+ entry = _new(terms, source, note)
212
+ data["entries"].append(entry)
213
+ return entry
214
+
215
+
216
+ def update(entry_id: str, *, terms: list[str] | None = None, note: str | None = None,
217
+ confirm: bool = False) -> dict[str, Any]:
218
+ """Change an entry; ``confirm`` turns a suggestion into an entry in use."""
219
+ with _updating() as data:
220
+ e = _find(data, entry_id)
221
+ if terms is not None:
222
+ terms = _clean(terms)
223
+ if len(terms) < 2:
224
+ raise ValueError("无法保存:一条术语至少要有两种写法")
225
+ e["terms"] = terms
226
+ if note is not None:
227
+ e["note"] = str(note).strip()
228
+ if confirm:
229
+ e["status"] = "active"
230
+ return dict(e)
231
+
232
+
233
+ def delete(entry_id: str) -> None:
234
+ """Remove an entry; a deleted suggestion is not suggested again."""
235
+ with _updating() as data:
236
+ e = _find(data, entry_id)
237
+ data["entries"].remove(e)
238
+ if e.get("status") == "suggested" and len(e["terms"]) == 2:
239
+ pair = [normalize(t) for t in e["terms"]]
240
+ if pair not in data["dismissed"]:
241
+ data["dismissed"].append(pair)
242
+
243
+
244
+ def _find(data: dict[str, Any], entry_id: str) -> dict[str, Any]:
245
+ for e in data["entries"]:
246
+ if e.get("id") == entry_id:
247
+ return e
248
+ raise KeyError("无法找到这条术语:它可能已被删除")
249
+
250
+
251
+ # --------------------------------------------------------------------------- expanding
252
+
253
+ def _abbreviation(term: str) -> bool:
254
+ """"PDC", "SiC", "EMW" and element symbols: matched with case, so "Al" is not "al" in a sentence."""
255
+ return " " not in term and (sum(ch.isupper() for ch in term) >= 2 or term in text.element_symbols())
256
+
257
+
258
+ _D = re.escape(text.DASHES) # any dash, for the patterns below
259
+
260
+
261
+ @lru_cache(maxsize=2048)
262
+ def _pattern(term: str) -> re.Pattern[str]:
263
+ """``term`` as a whole word or phrase, written any way ``normalize`` treats as the
264
+ same: a dash or a space ("polymer derived"), British spelling, the last word
265
+ plural ("PDCs", "polymer-derived ceramics")."""
266
+ from .spelling import variants
267
+ if _abbreviation(term):
268
+ return re.compile(rf"(?<![\w{_D}]){re.escape(term)}s?(?![\w{_D}])")
269
+ words = [w for w in re.split(rf"[\s{_D}]+", term) if w]
270
+ alts = [variants(w) for w in words]
271
+ last = words[-1] if words else ""
272
+ if len(last) > 2 and last[-1:].lower() == "y" and last[-2:-1].lower() not in "aeiou":
273
+ alts[-1] += [last[:-1] + "ies"]
274
+ body = rf"[\s{_D}]+".join("(?:" + "|".join(re.escape(a) for a in alt) + ")" for alt in alts)
275
+ return re.compile(rf"(?<![\w{_D}]){body}(?:s|es)?(?![\w{_D}])", re.I)
276
+
277
+
278
+ def _has_other_concepts(query: str, span: tuple[int, int]) -> bool:
279
+ """Something besides the matched term that keeps an expansion on topic."""
280
+ rest = query[:span[0]] + " " + query[span[1]:]
281
+ return any(len(w) >= 3 and w.lower() not in text.STOPWORDS for w in re.findall(r"[A-Za-z0-9]+", rest))
282
+
283
+
284
+ def other_spelling(query: str) -> tuple[str, list[tuple[str, str]]]:
285
+ """``query`` with each word in its other spelling (British words American,
286
+ American words in their first British spelling), and the words changed;
287
+ "" when no word has another spelling. Abbreviations (PDC, SiC) stay."""
288
+ from .spelling import to_us, variants
289
+ changed: list[tuple[str, str]] = []
290
+
291
+ def swap(m: re.Match[str]) -> str:
292
+ w = m.group(0)
293
+ if not (w.islower() or w.istitle()):
294
+ return w
295
+ low = w.lower()
296
+ us = to_us(low)
297
+ other = us if us != low else next(iter(variants(low)[1:]), low)
298
+ if other == low:
299
+ return w
300
+ changed.append((w, other))
301
+ return other.capitalize() if w.istitle() else other
302
+ out = re.sub(r"[A-Za-z]+", swap, query)
303
+ return (out if changed else ""), changed
304
+
305
+
306
+ def query_variants(query: str, use: list[dict[str, Any]]) -> tuple[list[str], list[dict[str, Any]]]:
307
+ """Other wordings of a keyword query (at most MAX_VARIANTS): first the query in
308
+ its other spelling (OpenAlex matches "aluminum" and "aluminium" separately),
309
+ then one term swapped for another of its entry; and what was expanded or left
310
+ alone ("alone": the query is that term and nothing else)."""
311
+ variants: list[str] = []
312
+ used: list[dict[str, Any]] = []
313
+ spelled, changed = other_spelling(query)
314
+ if spelled:
315
+ variants.append(spelled)
316
+ used += [{"term": w, "with": [o], "query": query, "builtin": "spelling"} for w, o in changed]
317
+ for e in use:
318
+ for t in e.get("match", e["terms"]):
319
+ m = _pattern(t).search(query)
320
+ if not m:
321
+ continue
322
+ others = [o for o in e.get("add", e["terms"]) if o != t and not _pattern(o).search(query)]
323
+ if not others:
324
+ break
325
+ report = {"id": e["id"], "term": m.group(0), "with": others, "query": query}
326
+ if e.get("builtin"):
327
+ report["builtin"] = e["builtin"]
328
+ if not _has_other_concepts(query, m.span()):
329
+ used.append({**report, "alone": True})
330
+ break
331
+ for o in others:
332
+ v = query[:m.start()] + o + query[m.end():]
333
+ if len(variants) < MAX_VARIANTS and v not in variants:
334
+ variants.append(v)
335
+ used.append(report)
336
+ break
337
+ return variants, used
338
+
339
+
340
+ def alternatives(word: str, use: list[dict[str, Any]]) -> tuple[list[str], list[str]]:
341
+ """The other terms of the entries ``word`` is one of (a word of a deep search's
342
+ core term), and the built-in kinds among those entries ("element")."""
343
+ out: list[str] = []
344
+ kinds: list[str] = []
345
+ for e in use:
346
+ if any(_pattern(t).fullmatch(word) for t in e["terms"]):
347
+ more = [t for t in e.get("add", e["terms"]) if not _pattern(t).fullmatch(word)]
348
+ out += more
349
+ if more and e.get("builtin"):
350
+ kinds.append(e["builtin"])
351
+ return list(dict.fromkeys(out)), list(dict.fromkeys(kinds))
352
+
353
+
354
+ # --------------------------------------------------------------------------- discovering
355
+
356
+ # Characterization methods nearly every materials abstract spells out: expanding
357
+ # them finds nothing new, and suggesting them only gives the user more to click.
358
+ _COMMON = set("SEM TEM HRTEM STEM XRD XPS FTIR FT-IR IR NMR TGA DTA DSC TG EDS EDX EDXS BET AFM SAED EIS "
359
+ "UV UV-vis CV GCD EPR ESR XAS XANES EXAFS ICP ICP-OES ICP-MS".split())
360
+
361
+
362
+ def _usable(short: str, long: str) -> bool:
363
+ words = long.split()
364
+ return (2 <= len(short) <= 12 and any(ch.isalpha() for ch in short) and _short_key(short) not in _COMMON
365
+ and 1 <= len(words) <= 8 and not re.search(r"[()\[\]]", long) and words[0].lower() not in text.STOPWORDS)
366
+
367
+
368
+ def _short_key(short: str) -> str:
369
+ """"PDCs" and "PDC" are one abbreviation."""
370
+ if len(short) > 2 and short.endswith("s") and sum(ch.isupper() for ch in short[:-1]) >= 2:
371
+ return short[:-1]
372
+ return short
373
+
374
+
375
+ def _meaning_key(norm_long: str) -> str:
376
+ """Wordings of one meaning ("scanning electron microscopy" / "microscope"): word stems."""
377
+ return " ".join(text.stem(w) for w in norm_long.split())
378
+
379
+
380
+ def discover(texts: list[str]) -> dict[str, dict[str, dict[str, Any]]]:
381
+ """"Long form (SF)" pairs in ``texts`` (abstracts): short form -> meaning ->
382
+ {"text": its most common wording, "papers": how many texts write it}."""
383
+ from abbreviations import schwartz_hearst
384
+ seen: dict[str, dict[str, Counter]] = {}
385
+ for abstract in texts:
386
+ try:
387
+ pairs = schwartz_hearst.extract_abbreviation_definition_pairs(doc_text=abstract or "")
388
+ except Exception: # an odd abstract must not stop the rest
389
+ continue
390
+ for short, long in pairs.items():
391
+ long = text.ascii_dashes(" ".join(long.split()))
392
+ if _usable(short, long):
393
+ key = _meaning_key(normalize(long))
394
+ seen.setdefault(_short_key(short), {}).setdefault(key, Counter())[long.lower()] += 1
395
+ return {s: {m: {"text": c.most_common(1)[0][0], "papers": sum(c.values())} for m, c in longs.items()}
396
+ for s, longs in seen.items()}
397
+
398
+
399
+ def suggest_from(texts: list[str], min_papers: int = MIN_PAPERS) -> list[dict[str, Any]]:
400
+ """Add the pairs ``texts`` write often enough as suggestions; returns the new ones."""
401
+ found = discover(texts)
402
+ if not found:
403
+ return []
404
+ new: list[dict[str, Any]] = []
405
+ with _updating() as data:
406
+ def key(a: str, b: str) -> tuple[str, str]:
407
+ return normalize(_short_key(a)), _meaning_key(normalize(b))
408
+
409
+ known = {key(a, b) for e in data["entries"] + elements() for a in e["terms"] for b in e["terms"] if a != b}
410
+ dismissed = {key(*p) for p in data["dismissed"]}
411
+ for short, longs in sorted(found.items()):
412
+ several = sum(1 for i in longs.values() if i["papers"] >= min_papers) > 1
413
+ for meaning, info in sorted(longs.items(), key=lambda kv: -kv[1]["papers"]):
414
+ pair = key(short, info["text"])
415
+ if info["papers"] < min_papers or pair in known or pair in dismissed:
416
+ continue
417
+ note = (f"在 {info['papers']} 篇摘要中这样写" +
418
+ (";这个缩写还有其他全称,留下对的那条" if several else ""))
419
+ e = _new([short, info["text"]], "auto", note, papers=info["papers"])
420
+ data["entries"].append(e)
421
+ new.append(e)
422
+ return new
@@ -0,0 +1,89 @@
1
+ """British and American spellings, so a search finds both.
2
+
3
+ Papers write "aluminium" or "aluminum", "fibre" or "fiber", "characterised"
4
+ or "characterized". The spellings come from VarCon (Kevin Atkinson and
5
+ Benjamin Titze, part of SCOWL; see data/VARCON-LICENSE.txt), built into two
6
+ tables by scripts/build_spelling_table.py:
7
+
8
+ * ``to_us`` maps text to the preferred American spelling before words are
9
+ compared (query matching, ranking features);
10
+ * ``variants`` gives the spellings of one word worth searching for, since
11
+ OpenAlex matches "aluminum" and "aluminium" separately.
12
+
13
+ Abbreviations, synonyms and element symbols live in ``glossary``; the user
14
+ may switch a word's spellings off there ("spelling:<American word>").
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import gzip
20
+ import re
21
+ from functools import lru_cache
22
+ from importlib.resources import files
23
+
24
+ _WORD = re.compile(r"[a-z]+")
25
+
26
+
27
+ @lru_cache(maxsize=1)
28
+ def _tables() -> tuple[dict[str, str], dict[str, list[str]]]:
29
+ data = files("litloop") / "data"
30
+ with gzip.open(data.joinpath("spelling_us.tsv.gz").open("rb"), "rt", encoding="utf-8") as fh:
31
+ to_us_table = dict(line.rstrip("\n").split("\t", 1) for line in fh if "\t" in line)
32
+ search: dict[str, list[str]] = {}
33
+ with gzip.open(data.joinpath("spelling_search.tsv.gz").open("rb"), "rt", encoding="utf-8") as fh:
34
+ for line in fh:
35
+ word, *others = line.rstrip("\n").split("\t")
36
+ search[word] = others
37
+ return to_us_table, search
38
+
39
+
40
+ _OFF: list = [None, frozenset()] # (glossary file, its mtime), American words switched off
41
+
42
+
43
+ def _off() -> frozenset[str]:
44
+ """American words whose other spellings the user switched off (the glossary's
45
+ ``disabled``), read again when the file changes."""
46
+ from . import glossary
47
+ p = glossary.path()
48
+ try:
49
+ stamp = (str(p), p.stat().st_mtime_ns)
50
+ except OSError:
51
+ return frozenset()
52
+ if _OFF[0] != stamp:
53
+ _OFF[:] = [stamp, frozenset(k.split(":", 1)[1] for k in glossary.disabled() if k.startswith("spelling:"))]
54
+ return _OFF[1]
55
+
56
+
57
+ def to_us(text: str) -> str:
58
+ """Lower-case ``text`` with British spellings turned American."""
59
+ table = _tables()[0]
60
+ off = _off()
61
+
62
+ def us(m: re.Match[str]) -> str:
63
+ w = table.get(m.group(), m.group())
64
+ return m.group() if w in off else w
65
+ return _WORD.sub(us, text.lower())
66
+
67
+
68
+ def variants(word: str) -> list[str]:
69
+ """The spellings of ``word`` worth searching for, the given one first."""
70
+ to_us_table, search = _tables()
71
+ low = word.lower()
72
+ american = to_us_table.get(low, low)
73
+ if american in _off():
74
+ return [word]
75
+ out = [word]
76
+ for w in [american, *search.get(american, [])]:
77
+ if w != low and w not in out:
78
+ out.append(w)
79
+ return out
80
+
81
+
82
+ def spelling_pairs(query: str, limit: int = 50) -> list[tuple[str, list[str]]]:
83
+ """(American spelling, its British spellings) for words starting with ``query``
84
+ in any of their spellings, for looking them up in the app."""
85
+ q = query.strip().lower()
86
+ found = [(us, others) for us, others in _tables()[1].items()
87
+ if others and (us.startswith(q) or any(o.startswith(q) for o in others))]
88
+ found.sort(key=lambda p: (q != p[0] and q not in p[1], len(p[0]), p[0])) # the word itself, then shortest
89
+ return found[:limit]
litloop/terms/text.py ADDED
@@ -0,0 +1,106 @@
1
+ """Text normalisation shared by query matching, ranking, the glossary and deduplication.
2
+
3
+ One set of rules, so that the same two wordings count as the same everywhere
4
+ (2026-10-25: there were three stop-word lists, two plural rules, two ways of
5
+ stemming, four sets of dashes and four paper keys):
6
+
7
+ * ``plain`` / ``squash`` / ``ascii_dashes``: case, every kind of dash, American spelling;
8
+ * ``singular`` / ``stem``: the plural of a word, and the crude prefix stem used
9
+ for substring tests and word features;
10
+ * ``STOPWORDS``: words that carry no topic in a query; ``RANKING_STOPWORDS``,
11
+ the shorter list the ranking model's word features drop;
12
+ * ``element_symbols``: the chemical elements (data/elements.tsv, from PubChem);
13
+ * ``paper_key`` / ``title_key``: when two records are the same paper.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import re
19
+ from functools import lru_cache
20
+ from typing import Any
21
+
22
+ # Hyphen, non-breaking hyphen, figure dash, en dash, em dash, horizontal bar,
23
+ # minus sign, small and full-width hyphen-minus, and the ASCII hyphen-minus.
24
+ DASHES = "‐‑‒–—―−﹣--"
25
+ _DASH = re.compile("[" + re.escape(DASHES) + "]")
26
+ _DASH_RUN = re.compile("[" + re.escape(DASHES) + "]+")
27
+ _SQUASH = re.compile(r"[\s/" + re.escape(DASHES) + "]+")
28
+
29
+ STOPWORDS = frozenset(
30
+ "a an and the of in on for to with by from at as or via its their into using based under over between "
31
+ "is are was were be this that these which we our than also "
32
+ "study studies effect effects analysis review".split()
33
+ )
34
+
35
+ # The ranking model's word features (deep_search ``_tokens``) drop only these.
36
+ # Not STOPWORDS: on SYNERGY, dropping review/study/effect/analysis/using/based
37
+ # from the features made Leenaars_2019 and Appenzeller-Herzog_2019 worse, and
38
+ # adding just via/under/over/between changed the ranking too (DECISIONS,
39
+ # 2026-10-25). Changing this list changes the ranking: run
40
+ # scripts/eval_synergy.py before and after.
41
+ RANKING_STOPWORDS = frozenset(
42
+ "the and of in for with by on to a an is are was were from as at this that these be or its "
43
+ "their which we our into than also".split()
44
+ )
45
+
46
+
47
+ def ascii_dashes(text: str) -> str:
48
+ """Every kind of dash as "-" (DOIs and titles copied from PDFs)."""
49
+ return _DASH.sub("-", text or "")
50
+
51
+
52
+ def plain(text: str) -> str:
53
+ """Lower case, American spelling, every kind of dash read as a space."""
54
+ from .spelling import to_us
55
+ return _DASH_RUN.sub(" ", to_us(text or ""))
56
+
57
+
58
+ def squash(text: str) -> str:
59
+ """Lower case with spaces, dashes and slashes removed: "Polymer–derived" -> "polymerderived"."""
60
+ return _SQUASH.sub("", (text or "").lower())
61
+
62
+
63
+ def singular(word: str) -> str:
64
+ """Crude singular of a lower-case word: "ceramics" -> "ceramic", "properties" -> "property";
65
+ "glass", "virus" and "analysis" stay."""
66
+ if len(word) > 4 and word.endswith("ies"):
67
+ return word[:-3] + "y"
68
+ if len(word) > 3 and word.endswith("s") and not word.endswith(("ss", "us", "is")):
69
+ return word[:-1]
70
+ return word
71
+
72
+
73
+ def stem(word: str) -> str:
74
+ """Crude stem, the first 7 letters: "infiltration" and "infiltrated" -> "infiltr"."""
75
+ return word[:7]
76
+
77
+
78
+ @lru_cache(maxsize=1)
79
+ def elements() -> tuple[tuple[str, str], ...]:
80
+ """(symbol, American name) of the 118 elements."""
81
+ from importlib.resources import files
82
+ rows = (files("litloop") / "data" / "elements.tsv").read_text(encoding="utf-8").splitlines()
83
+ return tuple(tuple(line.split("\t")) for line in rows if line and not line.startswith("#")) # type: ignore[misc]
84
+
85
+
86
+ @lru_cache(maxsize=1)
87
+ def element_symbols() -> frozenset[str]:
88
+ return frozenset(s for s, _ in elements())
89
+
90
+
91
+ def is_chemical_system(word: str) -> bool:
92
+ """"Al-Si-Mg", "Si-B-C-N": element symbols joined by dashes (papers list them in any order)."""
93
+ parts = _DASH.split(word)
94
+ return len(parts) > 1 and all(p in element_symbols() for p in parts)
95
+
96
+
97
+ def title_key(title: Any) -> str:
98
+ """A title's letters and digits, lower case: the same title from two sources."""
99
+ return re.sub(r"[^a-z0-9]+", "", str(title or "").lower())
100
+
101
+
102
+ def paper_key(rec: dict[str, Any]) -> str:
103
+ """The key that says two records are one paper: the DOI, or else the title
104
+ ("title:<letters and digits>"). The app's ``paperKey`` (app/src/lib/api.ts) mirrors it."""
105
+ from ..identifiers import canonical_doi
106
+ return canonical_doi(rec.get("doi")) or "title:" + title_key(rec.get("title"))
@@ -0,0 +1 @@
1
+ """Tor proxy management and embedded Tor daemon support."""