glitchlings 0.4.2__cp312-cp312-win_amd64.whl → 0.4.4__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of glitchlings might be problematic. Click here for more details.
- glitchlings/__init__.py +4 -0
- glitchlings/_zoo_rust.cp312-win_amd64.pyd +0 -0
- glitchlings/compat.py +80 -11
- glitchlings/config.py +32 -19
- glitchlings/config.toml +1 -1
- glitchlings/dlc/__init__.py +3 -1
- glitchlings/dlc/_shared.py +86 -1
- glitchlings/dlc/pytorch.py +166 -0
- glitchlings/dlc/pytorch_lightning.py +215 -0
- glitchlings/lexicon/__init__.py +10 -16
- glitchlings/lexicon/_cache.py +21 -15
- glitchlings/lexicon/data/default_vector_cache.json +80 -14
- glitchlings/lexicon/vector.py +94 -15
- glitchlings/lexicon/wordnet.py +66 -25
- glitchlings/main.py +21 -11
- glitchlings/zoo/__init__.py +5 -1
- glitchlings/zoo/_rate.py +114 -1
- glitchlings/zoo/_rust_extensions.py +143 -0
- glitchlings/zoo/adjax.py +5 -6
- glitchlings/zoo/apostrofae.py +127 -0
- glitchlings/zoo/assets/__init__.py +0 -0
- glitchlings/zoo/assets/apostrofae_pairs.json +32 -0
- glitchlings/zoo/core.py +61 -23
- glitchlings/zoo/jargoyle.py +50 -36
- glitchlings/zoo/redactyl.py +15 -13
- glitchlings/zoo/reduple.py +5 -6
- glitchlings/zoo/rushmore.py +5 -6
- glitchlings/zoo/scannequin.py +5 -6
- glitchlings/zoo/typogre.py +8 -6
- glitchlings/zoo/zeedub.py +8 -6
- {glitchlings-0.4.2.dist-info → glitchlings-0.4.4.dist-info}/METADATA +40 -4
- glitchlings-0.4.4.dist-info/RECORD +47 -0
- glitchlings/lexicon/graph.py +0 -282
- glitchlings-0.4.2.dist-info/RECORD +0 -42
- {glitchlings-0.4.2.dist-info → glitchlings-0.4.4.dist-info}/WHEEL +0 -0
- {glitchlings-0.4.2.dist-info → glitchlings-0.4.4.dist-info}/entry_points.txt +0 -0
- {glitchlings-0.4.2.dist-info → glitchlings-0.4.4.dist-info}/licenses/LICENSE +0 -0
- {glitchlings-0.4.2.dist-info → glitchlings-0.4.4.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: glitchlings
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.4
|
|
4
4
|
Summary: Monsters for your language games.
|
|
5
5
|
Author: osoleve
|
|
6
6
|
License: Apache License
|
|
@@ -217,6 +217,7 @@ Classifier: Programming Language :: Python :: 3
|
|
|
217
217
|
Classifier: Programming Language :: Python :: 3.10
|
|
218
218
|
Classifier: Programming Language :: Python :: 3.11
|
|
219
219
|
Classifier: Programming Language :: Python :: 3.12
|
|
220
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
220
221
|
Classifier: Programming Language :: Rust
|
|
221
222
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
222
223
|
Classifier: Topic :: Software Development :: Testing
|
|
@@ -226,19 +227,43 @@ License-File: LICENSE
|
|
|
226
227
|
Requires-Dist: confusable-homoglyphs>=3.3.1
|
|
227
228
|
Requires-Dist: tomli>=2.0.1; python_version < "3.11"
|
|
228
229
|
Requires-Dist: pyyaml>=6.0.0
|
|
230
|
+
Provides-Extra: all
|
|
231
|
+
Requires-Dist: black>=24.4.0; extra == "all"
|
|
232
|
+
Requires-Dist: hypothesis>=6.140.0; extra == "all"
|
|
233
|
+
Requires-Dist: interrogate>=1.5.0; extra == "all"
|
|
234
|
+
Requires-Dist: jellyfish>=1.2.0; extra == "all"
|
|
235
|
+
Requires-Dist: isort>=5.13.0; extra == "all"
|
|
236
|
+
Requires-Dist: mkdocs>=1.6.0; extra == "all"
|
|
237
|
+
Requires-Dist: mkdocs-material>=9.5.0; extra == "all"
|
|
238
|
+
Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "all"
|
|
239
|
+
Requires-Dist: mkdocstrings-python>=1.10.0; extra == "all"
|
|
240
|
+
Requires-Dist: mypy>=1.8.0; extra == "all"
|
|
241
|
+
Requires-Dist: numpy<3.0,>=1.24; extra == "all"
|
|
242
|
+
Requires-Dist: pre-commit>=3.8.0; extra == "all"
|
|
243
|
+
Requires-Dist: pytest>=8.0.0; extra == "all"
|
|
244
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "all"
|
|
245
|
+
Requires-Dist: ruff>=0.6.0; extra == "all"
|
|
246
|
+
Requires-Dist: verifiers>=0.1.3.post0; extra == "all"
|
|
229
247
|
Provides-Extra: hf
|
|
230
248
|
Requires-Dist: datasets>=4.0.0; extra == "hf"
|
|
249
|
+
Provides-Extra: lightning
|
|
250
|
+
Requires-Dist: pytorch_lightning>=2.0.0; extra == "lightning"
|
|
231
251
|
Provides-Extra: vectors
|
|
232
|
-
Requires-Dist: numpy
|
|
252
|
+
Requires-Dist: numpy<3.0,>=1.24; extra == "vectors"
|
|
233
253
|
Requires-Dist: spacy>=3.7.2; extra == "vectors"
|
|
234
254
|
Requires-Dist: gensim>=4.3.2; extra == "vectors"
|
|
255
|
+
Provides-Extra: st
|
|
256
|
+
Requires-Dist: sentence-transformers>=3.0.0; extra == "st"
|
|
235
257
|
Provides-Extra: prime
|
|
236
258
|
Requires-Dist: verifiers>=0.1.3.post0; extra == "prime"
|
|
237
259
|
Requires-Dist: jellyfish>=1.2.0; extra == "prime"
|
|
260
|
+
Provides-Extra: torch
|
|
261
|
+
Requires-Dist: torch>=2.0.0; extra == "torch"
|
|
238
262
|
Provides-Extra: dev
|
|
239
263
|
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
264
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
240
265
|
Requires-Dist: hypothesis>=6.140.0; extra == "dev"
|
|
241
|
-
Requires-Dist: numpy
|
|
266
|
+
Requires-Dist: numpy<3.0,>=1.24; extra == "dev"
|
|
242
267
|
Requires-Dist: mkdocs>=1.6.0; extra == "dev"
|
|
243
268
|
Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "dev"
|
|
244
269
|
Requires-Dist: mkdocs-material>=9.5.0; extra == "dev"
|
|
@@ -307,7 +332,7 @@ print(gaggle(SAMPLE_TEXT))
|
|
|
307
332
|
> Onҽ mھrning, wһen Gregor Samƽa woke from trouble𝐝 𝑑reams, he found himself transformed in his bed into a horrible vermin٠ He l lay on his armour-like back, and if he lifted his head a little he could see his brown belly, slightlh domed and divided by arches ino stiff sections. The bedding was adly able to cover it and and seemed ready to slide off any moment. His many legxs, pitifully thin compared with the size of the the rest of him, waved about helplessly ashe looked looked.
|
|
308
333
|
|
|
309
334
|
Consult the [Glitchlings Usage Guide](docs/index.md)
|
|
310
|
-
for end-to-end instructions spanning the Python API, CLI, HuggingFace and Prime Intellect
|
|
335
|
+
for end-to-end instructions spanning the Python API, CLI, HuggingFace, PyTorch, and Prime Intellect
|
|
311
336
|
integrations, and the autodetected Rust pipeline (enabled whenever the extension is present).
|
|
312
337
|
|
|
313
338
|
## Motivation
|
|
@@ -356,6 +381,7 @@ glitchlings --list
|
|
|
356
381
|
|
|
357
382
|
```text
|
|
358
383
|
Typogre — scope: Character, order: early
|
|
384
|
+
Apostrofae — scope: Character, order: normal
|
|
359
385
|
Mim1c — scope: Character, order: last
|
|
360
386
|
Jargoyle — scope: Word, order: normal
|
|
361
387
|
Adjax — scope: Word, order: normal
|
|
@@ -458,6 +484,16 @@ _What a nice word, would be a shame if something happened to it._
|
|
|
458
484
|
> - `keyboard (str)`: Keyboard layout key-neighbor map to use (default: "CURATOR_QWERTY"; also accepts "QWERTY", "DVORAK", "COLEMAK", and "AZERTY").
|
|
459
485
|
> - `seed (int)`: The random seed for reproducibility (default: 151).
|
|
460
486
|
|
|
487
|
+
### Apostrofae
|
|
488
|
+
|
|
489
|
+
_It looks like you're trying to paste some text. Can I help?_
|
|
490
|
+
|
|
491
|
+
> _**Paperclip Manager.**_ Apostrofae scans for balanced runs of straight quotes, apostrophes, and backticks before replacing them with randomly sampled smart-quote pairs from a curated lookup table. The swap happens in-place so contractions and unpaired glyphs remain untouched.
|
|
492
|
+
>
|
|
493
|
+
> Args
|
|
494
|
+
>
|
|
495
|
+
> - `seed (int)`: Optional seed controlling the deterministic smart-quote sampling (default: 151).
|
|
496
|
+
|
|
461
497
|
### Mim1c
|
|
462
498
|
|
|
463
499
|
_Wait, was that...?_
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
glitchlings/__init__.py,sha256=jyfvslkaFlYVefDFq0CQKkc52F8p8I1UrbFbcyihFCU,1249
|
|
2
|
+
glitchlings/__main__.py,sha256=nB7btO_T4wBFOcyawfWpjEindVrUfTqqV5hdeeS1HT8,128
|
|
3
|
+
glitchlings/_zoo_rust.cp312-win_amd64.pyd,sha256=5Phrwe90toWkUNAq3nvpTKaSFCbR6L-2fagCTSvYXu0,2218496
|
|
4
|
+
glitchlings/compat.py,sha256=xM5fT5RELgIdQTmgKQFZaPZJJpOg0noC-gLDcO390Ro,9192
|
|
5
|
+
glitchlings/config.py,sha256=pKpM5onr9UG8sHKVIZKj2Ti2gD5bZ6N1wyxOhzgrviQ,13341
|
|
6
|
+
glitchlings/config.toml,sha256=OywXmEpuOPtyJRbcRt4cwQkHiZ__5axEHoCaX9ye-uA,102
|
|
7
|
+
glitchlings/main.py,sha256=eCUEFsu8-NLDz1xyKNDIucVm975HNQZJYm6YCv8RIyg,10987
|
|
8
|
+
glitchlings/dlc/__init__.py,sha256=iFDTwkaWl2C0_QUYykIXfmOUzy__oURX_BiJhexf-8o,312
|
|
9
|
+
glitchlings/dlc/_shared.py,sha256=L6dc4Xi0q2K4ZwkyytXIU8Zu3MPugIDXhseZNuJNqyY,4560
|
|
10
|
+
glitchlings/dlc/huggingface.py,sha256=Ym8dTArb-43AnCyukOO1m66iAbs8al9YkIWB3rGdhTk,2657
|
|
11
|
+
glitchlings/dlc/prime.py,sha256=KY3so8WOwksbsKhZXfWorsZSYvIdPiUtc8e9apHabkM,8861
|
|
12
|
+
glitchlings/dlc/pytorch.py,sha256=FZZbWHVX94FA9Ab77twZPUwQNno4IKTAjuHPXNhdY6M,6399
|
|
13
|
+
glitchlings/dlc/pytorch_lightning.py,sha256=cKLFn3cFcOhFSN3QFlcvRPP5sLQts0M6DQ1J1cQuqRM,8289
|
|
14
|
+
glitchlings/lexicon/__init__.py,sha256=oSLI1bOMfTpqiI7bMp9beeQ7Vp81G5coXxwdmCIQfV0,6199
|
|
15
|
+
glitchlings/lexicon/_cache.py,sha256=KWqJ__WrM2ccIlplaaqoVT0ns65uU5WHewlJd4BvnJE,4196
|
|
16
|
+
glitchlings/lexicon/metrics.py,sha256=TZAafSKgHpUS4h6vCuhTKGsvu_fru9kMqsXqLID6BTM,4734
|
|
17
|
+
glitchlings/lexicon/vector.py,sha256=x9iT1O8Osolwt08g41V_70WHZt_b4OGzHBU72YHkmwg,23181
|
|
18
|
+
glitchlings/lexicon/wordnet.py,sha256=fJi5SNa-sLpQiTIoXorkYzc2ZArejIms6zhoe8TPIOg,7840
|
|
19
|
+
glitchlings/lexicon/data/default_vector_cache.json,sha256=bnMV4tHIVOQtK7FDH81yqSLRkeViEzclGKXrrS8fEJ8,1079
|
|
20
|
+
glitchlings/util/__init__.py,sha256=Q5lkncOaM6f2eJK3HAtZyxpCjGnekCpwPloqasS3JDo,4869
|
|
21
|
+
glitchlings/util/adapters.py,sha256=mFhPlE8JaFuO_C-3_aqhgwkqa6isV8Y2ifqVh3Iv9JM,720
|
|
22
|
+
glitchlings/zoo/__init__.py,sha256=j21naQtFunJYgdgYsyTNYUSa7sl88yNQWaGP8sWyw5U,5411
|
|
23
|
+
glitchlings/zoo/_ocr_confusions.py,sha256=pPlvJOoan3ouwwGt8hATcO-9luIrGJl0vwUqssUMXD8,1236
|
|
24
|
+
glitchlings/zoo/_rate.py,sha256=KxFDFJEGWsv76v1JcoHXIETj9kGdbbAiUqCPwAcWcDw,3710
|
|
25
|
+
glitchlings/zoo/_rust_extensions.py,sha256=SdU06m-qjf-rRHnqM5OMophx1IacoI4KbyRu2HXrTUc,4354
|
|
26
|
+
glitchlings/zoo/_sampling.py,sha256=AAPLObjqKrmX882TX8hdvPHReBOcv0Z4pUuW6AxuGgU,1640
|
|
27
|
+
glitchlings/zoo/_text_utils.py,sha256=LqCa33E-Qxbk6N5AVfxEmAz6C2u7_mCF0xPT9-404A8,2854
|
|
28
|
+
glitchlings/zoo/adjax.py,sha256=8AqTfcJe50vPVxER5wREuyr4qqsmyKqi-klDW77HVYM,3646
|
|
29
|
+
glitchlings/zoo/apostrofae.py,sha256=WB1zvCc1_YvTD9XdTNtFrK8H9FleaF-UNMCai_E7lqE,3949
|
|
30
|
+
glitchlings/zoo/core.py,sha256=tkJFqGwpVa7qQxkUr9lsHOB8zS3lDCo987R6Nvz375U,21257
|
|
31
|
+
glitchlings/zoo/jargoyle.py,sha256=kgKw_hsaJaKQjrxzt_CMlgtZApedM6I2-U3d3aeipXs,12014
|
|
32
|
+
glitchlings/zoo/mim1c.py,sha256=GqUMErVAVcqMAZjx4hhJ0Af25CxA0Aorv3U_fTqLZek,3546
|
|
33
|
+
glitchlings/zoo/ocr_confusions.tsv,sha256=S-IJEYCIXYKT1Uu7Id8Lnvg5pw528yNigTtWUdnMv9k,213
|
|
34
|
+
glitchlings/zoo/redactyl.py,sha256=xSdt-Az3HI8XLC-uV8hK6JIoNa1nL-cTCFCoShFvKt0,5752
|
|
35
|
+
glitchlings/zoo/reduple.py,sha256=YfdKZPr2TO2ZK7h8K5xlsnoubEZ3kQLRpDX65GzrKls,4372
|
|
36
|
+
glitchlings/zoo/rushmore.py,sha256=JjLKTKQoXlvp0uvCzMRlJSXE_N2VFZPHMF_nkolKm3g,4439
|
|
37
|
+
glitchlings/zoo/scannequin.py,sha256=-YG6tRTE72MGw4CpWAPo42gFaCTpyDvOJnk0gynitCg,5041
|
|
38
|
+
glitchlings/zoo/typogre.py,sha256=X6jcnCYifNtoSwe2ig7YS3QrODyrACirMZ9pjN5LLBA,6917
|
|
39
|
+
glitchlings/zoo/zeedub.py,sha256=KApSCSiTr3b412vsA8UHWFhYQDxe_jg0308wWK-GNtM,5025
|
|
40
|
+
glitchlings/zoo/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
+
glitchlings/zoo/assets/apostrofae_pairs.json,sha256=lPLFLndzn_f7_5wZizxsLMnwBY4O63zsCvDjyJ56MLA,553
|
|
42
|
+
glitchlings-0.4.4.dist-info/licenses/LICENSE,sha256=EFEP1evBfHaxsMTBjxm0sZVRp2wct8QLvHE1saII5FI,11538
|
|
43
|
+
glitchlings-0.4.4.dist-info/METADATA,sha256=G5yU2jAph6LYY0Hvr5WTa3tnSdmDI890nTiecmP7P68,33015
|
|
44
|
+
glitchlings-0.4.4.dist-info/WHEEL,sha256=8UP9x9puWI0P1V_d7K2oMTBqfeLNm21CTzZ_Ptr0NXU,101
|
|
45
|
+
glitchlings-0.4.4.dist-info/entry_points.txt,sha256=kGOwuAsjFDLtztLisaXtOouq9wFVMOJg5FzaAkg-Hto,54
|
|
46
|
+
glitchlings-0.4.4.dist-info/top_level.txt,sha256=VHFNBrLjtDwPCYXbGKi6o17Eueedi81eNbR3hBOoST0,12
|
|
47
|
+
glitchlings-0.4.4.dist-info/RECORD,,
|
glitchlings/lexicon/graph.py
DELETED
|
@@ -1,282 +0,0 @@
|
|
|
1
|
-
"""Graph-based lexicon backed by ConceptNet/Numberbatch embeddings."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import re
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
from typing import Iterable, Mapping, MutableMapping, Sequence
|
|
8
|
-
|
|
9
|
-
from . import LexiconBackend
|
|
10
|
-
from ._cache import CacheSnapshot
|
|
11
|
-
from ._cache import load_cache as _load_cache_file
|
|
12
|
-
from ._cache import write_cache as _write_cache_file
|
|
13
|
-
from .vector import VectorLexicon
|
|
14
|
-
|
|
15
|
-
_CONCEPT_RE = re.compile(r"^/c/(?P<lang>[a-z]{2})/(?P<term>[^/]+)")
|
|
16
|
-
_PUNCTUATION_RE = re.compile(r"[^\w\s-]+", re.UNICODE)
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def _lemmatize_token(token: str) -> str:
|
|
20
|
-
"""Return a lightweight lemma for ``token`` using heuristic rules."""
|
|
21
|
-
irregular = {
|
|
22
|
-
"children": "child",
|
|
23
|
-
"mice": "mouse",
|
|
24
|
-
"geese": "goose",
|
|
25
|
-
"feet": "foot",
|
|
26
|
-
"teeth": "tooth",
|
|
27
|
-
"men": "man",
|
|
28
|
-
"women": "woman",
|
|
29
|
-
"better": "good",
|
|
30
|
-
"worse": "bad",
|
|
31
|
-
}
|
|
32
|
-
lowered = token.lower()
|
|
33
|
-
if lowered in irregular:
|
|
34
|
-
return irregular[lowered]
|
|
35
|
-
|
|
36
|
-
if lowered.endswith("ies") and len(lowered) > 3:
|
|
37
|
-
return lowered[:-3] + "y"
|
|
38
|
-
if lowered.endswith("ves") and len(lowered) > 3:
|
|
39
|
-
return lowered[:-3] + "f"
|
|
40
|
-
if lowered.endswith("men") and len(lowered) > 3:
|
|
41
|
-
return lowered[:-3] + "man"
|
|
42
|
-
if lowered.endswith("ses") and len(lowered) > 3:
|
|
43
|
-
return lowered[:-2]
|
|
44
|
-
if lowered.endswith("es") and len(lowered) > 3:
|
|
45
|
-
return lowered[:-2]
|
|
46
|
-
if lowered.endswith("s") and len(lowered) > 2 and not lowered.endswith("ss"):
|
|
47
|
-
return lowered[:-1]
|
|
48
|
-
if lowered.endswith("ing") and len(lowered) > 4:
|
|
49
|
-
stem = lowered[:-3]
|
|
50
|
-
if len(stem) > 2 and stem[-1] == stem[-2]:
|
|
51
|
-
stem = stem[:-1]
|
|
52
|
-
return stem
|
|
53
|
-
if lowered.endswith("ed") and len(lowered) > 3:
|
|
54
|
-
stem = lowered[:-2]
|
|
55
|
-
if len(stem) > 2 and stem[-1] == stem[-2]:
|
|
56
|
-
stem = stem[:-1]
|
|
57
|
-
return stem
|
|
58
|
-
return lowered
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def _normalize_phrase(phrase: str) -> str:
|
|
62
|
-
"""Normalise ``phrase`` for ConceptNet lookups."""
|
|
63
|
-
stripped = _PUNCTUATION_RE.sub(" ", phrase.lower())
|
|
64
|
-
tokens = [token for token in stripped.split() if token]
|
|
65
|
-
if not tokens:
|
|
66
|
-
return ""
|
|
67
|
-
lemmatised = [_lemmatize_token(token) for token in tokens]
|
|
68
|
-
return " ".join(lemmatised)
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def _concept_terms(normalized: str) -> list[str]:
|
|
72
|
-
"""Return ConceptNet term variants for ``normalized``."""
|
|
73
|
-
collapsed = normalized.replace(" ", "_")
|
|
74
|
-
if not collapsed:
|
|
75
|
-
return []
|
|
76
|
-
variants = {collapsed}
|
|
77
|
-
variants.add(collapsed.replace("_", "-"))
|
|
78
|
-
variants.add(collapsed.replace("-", "_"))
|
|
79
|
-
return list(variants)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def _surface_from_concept(concept: str) -> str | None:
|
|
83
|
-
"""Return a human-readable surface form for ``concept``."""
|
|
84
|
-
match = _CONCEPT_RE.match(concept)
|
|
85
|
-
if match is None:
|
|
86
|
-
return None
|
|
87
|
-
term = match.group("term")
|
|
88
|
-
surface = term.replace("_", " ")
|
|
89
|
-
surface = surface.replace("-", " ")
|
|
90
|
-
return " ".join(surface.split())
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def _language_from_concept(concept: str) -> str | None:
|
|
94
|
-
match = _CONCEPT_RE.match(concept)
|
|
95
|
-
if match is None:
|
|
96
|
-
return None
|
|
97
|
-
return match.group("lang")
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def _load_numberbatch(path: Path, *, languages: set[str]) -> Mapping[str, list[float]]:
|
|
101
|
-
"""Load ConceptNet Numberbatch embeddings from ``path``."""
|
|
102
|
-
if not path.exists():
|
|
103
|
-
return {}
|
|
104
|
-
|
|
105
|
-
if path.suffix == ".gz":
|
|
106
|
-
import gzip
|
|
107
|
-
|
|
108
|
-
handle = gzip.open(path, "rt", encoding="utf8")
|
|
109
|
-
else:
|
|
110
|
-
handle = path.open("r", encoding="utf8")
|
|
111
|
-
|
|
112
|
-
with handle as stream:
|
|
113
|
-
header = stream.readline()
|
|
114
|
-
try:
|
|
115
|
-
parts = header.strip().split()
|
|
116
|
-
if len(parts) >= 2:
|
|
117
|
-
int(parts[0])
|
|
118
|
-
int(parts[1])
|
|
119
|
-
except ValueError:
|
|
120
|
-
stream.seek(0)
|
|
121
|
-
|
|
122
|
-
embeddings: dict[str, list[float]] = {}
|
|
123
|
-
for line in stream:
|
|
124
|
-
tokens = line.strip().split()
|
|
125
|
-
if len(tokens) <= 2:
|
|
126
|
-
continue
|
|
127
|
-
concept = tokens[0]
|
|
128
|
-
lang = _language_from_concept(concept)
|
|
129
|
-
if lang is None or lang not in languages:
|
|
130
|
-
continue
|
|
131
|
-
try:
|
|
132
|
-
vector = [float(value) for value in tokens[1:]]
|
|
133
|
-
except ValueError:
|
|
134
|
-
continue
|
|
135
|
-
embeddings[concept] = vector
|
|
136
|
-
return embeddings
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
class GraphLexicon(LexiconBackend):
|
|
140
|
-
"""Lexicon backed by ConceptNet/Numberbatch embeddings."""
|
|
141
|
-
|
|
142
|
-
def __init__(
|
|
143
|
-
self,
|
|
144
|
-
*,
|
|
145
|
-
source: Mapping[str, Sequence[float]] | str | Path | None = None,
|
|
146
|
-
cache: Mapping[str, Sequence[str]] | None = None,
|
|
147
|
-
cache_path: str | Path | None = None,
|
|
148
|
-
languages: Iterable[str] = ("en",),
|
|
149
|
-
max_neighbors: int = 50,
|
|
150
|
-
min_similarity: float = 0.0,
|
|
151
|
-
seed: int | None = None,
|
|
152
|
-
) -> None:
|
|
153
|
-
super().__init__(seed=seed)
|
|
154
|
-
self._languages = {language.lower() for language in languages}
|
|
155
|
-
if not self._languages:
|
|
156
|
-
self._languages = {"en"}
|
|
157
|
-
self._max_neighbors = max(1, max_neighbors)
|
|
158
|
-
self._min_similarity = min_similarity
|
|
159
|
-
self._cache: MutableMapping[str, list[str]] = {}
|
|
160
|
-
self._cache_path: Path | None = Path(cache_path) if cache_path is not None else None
|
|
161
|
-
self._cache_checksum: str | None = None
|
|
162
|
-
if self._cache_path is not None:
|
|
163
|
-
snapshot = _load_cache_file(self._cache_path)
|
|
164
|
-
self._cache.update(snapshot.entries)
|
|
165
|
-
self._cache_checksum = snapshot.checksum
|
|
166
|
-
if cache is not None:
|
|
167
|
-
for key, values in cache.items():
|
|
168
|
-
self._cache[str(key)] = [str(value) for value in values]
|
|
169
|
-
self._cache_dirty = False
|
|
170
|
-
|
|
171
|
-
prepared_source = self._prepare_source(source)
|
|
172
|
-
self._backend = VectorLexicon(
|
|
173
|
-
source=prepared_source if prepared_source else None,
|
|
174
|
-
max_neighbors=self._max_neighbors,
|
|
175
|
-
min_similarity=self._min_similarity,
|
|
176
|
-
case_sensitive=True,
|
|
177
|
-
seed=seed,
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
def _prepare_source(
|
|
181
|
-
self, source: Mapping[str, Sequence[float]] | str | Path | None
|
|
182
|
-
) -> Mapping[str, Sequence[float]]:
|
|
183
|
-
if source is None:
|
|
184
|
-
return {}
|
|
185
|
-
if isinstance(source, Mapping):
|
|
186
|
-
prepared: dict[str, list[float]] = {}
|
|
187
|
-
for key, vector in source.items():
|
|
188
|
-
lang = _language_from_concept(key)
|
|
189
|
-
if lang is None or lang not in self._languages:
|
|
190
|
-
continue
|
|
191
|
-
prepared[key] = [float(value) for value in vector]
|
|
192
|
-
return prepared
|
|
193
|
-
path = Path(source)
|
|
194
|
-
embeddings = _load_numberbatch(path, languages=self._languages)
|
|
195
|
-
return embeddings
|
|
196
|
-
|
|
197
|
-
def reseed(self, seed: int | None) -> None:
|
|
198
|
-
super().reseed(seed)
|
|
199
|
-
self._backend.reseed(seed)
|
|
200
|
-
|
|
201
|
-
def _concept_candidates(self, normalized: str) -> list[str]:
|
|
202
|
-
terms = _concept_terms(normalized)
|
|
203
|
-
concepts = []
|
|
204
|
-
for language in sorted(self._languages):
|
|
205
|
-
for term in terms:
|
|
206
|
-
concepts.append(f"/c/{language}/{term}")
|
|
207
|
-
return concepts
|
|
208
|
-
|
|
209
|
-
def _collect_synonyms(self, normalized: str) -> list[str]:
|
|
210
|
-
candidates: list[str] = []
|
|
211
|
-
seen: set[str] = set()
|
|
212
|
-
for concept in self._concept_candidates(normalized):
|
|
213
|
-
neighbors = self._backend.precompute(concept, limit=self._max_neighbors)
|
|
214
|
-
for neighbor in neighbors:
|
|
215
|
-
lang = _language_from_concept(neighbor)
|
|
216
|
-
if lang is None or lang not in self._languages:
|
|
217
|
-
continue
|
|
218
|
-
surface = _surface_from_concept(neighbor)
|
|
219
|
-
if surface is None:
|
|
220
|
-
continue
|
|
221
|
-
surface_norm = _normalize_phrase(surface)
|
|
222
|
-
if not surface_norm or surface_norm == normalized:
|
|
223
|
-
continue
|
|
224
|
-
if surface_norm in seen:
|
|
225
|
-
continue
|
|
226
|
-
seen.add(surface_norm)
|
|
227
|
-
candidates.append(surface)
|
|
228
|
-
return candidates
|
|
229
|
-
|
|
230
|
-
def _ensure_cached(self, normalized: str) -> list[str]:
|
|
231
|
-
if normalized in self._cache:
|
|
232
|
-
return self._cache[normalized]
|
|
233
|
-
synonyms = self._collect_synonyms(normalized)
|
|
234
|
-
self._cache[normalized] = synonyms
|
|
235
|
-
if self._cache_path is not None:
|
|
236
|
-
self._cache_dirty = True
|
|
237
|
-
return synonyms
|
|
238
|
-
|
|
239
|
-
def get_synonyms(self, word: str, pos: str | None = None, n: int = 5) -> list[str]:
|
|
240
|
-
normalized = _normalize_phrase(word)
|
|
241
|
-
if not normalized:
|
|
242
|
-
return []
|
|
243
|
-
synonyms = self._ensure_cached(normalized)
|
|
244
|
-
return self._deterministic_sample(synonyms, limit=n, word=word, pos=pos)
|
|
245
|
-
|
|
246
|
-
def precompute(self, word: str) -> list[str]:
|
|
247
|
-
normalized = _normalize_phrase(word)
|
|
248
|
-
if not normalized:
|
|
249
|
-
return []
|
|
250
|
-
return list(self._ensure_cached(normalized))
|
|
251
|
-
|
|
252
|
-
def export_cache(self) -> dict[str, list[str]]:
|
|
253
|
-
return {key: list(values) for key, values in self._cache.items()}
|
|
254
|
-
|
|
255
|
-
@classmethod
|
|
256
|
-
def load_cache(cls, path: str | Path) -> CacheSnapshot:
|
|
257
|
-
"""Load and validate a persisted ConceptNet cache file."""
|
|
258
|
-
return _load_cache_file(Path(path))
|
|
259
|
-
|
|
260
|
-
def save_cache(self, path: str | Path | None = None) -> Path:
|
|
261
|
-
if path is None:
|
|
262
|
-
if self._cache_path is None:
|
|
263
|
-
raise RuntimeError("No cache path supplied to GraphLexicon.")
|
|
264
|
-
target = self._cache_path
|
|
265
|
-
else:
|
|
266
|
-
target = Path(path)
|
|
267
|
-
self._cache_path = target
|
|
268
|
-
snapshot = _write_cache_file(target, self._cache)
|
|
269
|
-
self._cache_checksum = snapshot.checksum
|
|
270
|
-
self._cache_dirty = False
|
|
271
|
-
return target
|
|
272
|
-
|
|
273
|
-
def supports_pos(self, pos: str | None) -> bool:
|
|
274
|
-
return True
|
|
275
|
-
|
|
276
|
-
def __repr__(self) -> str: # pragma: no cover - debug helper
|
|
277
|
-
adapter = getattr(self._backend, "_adapter", None)
|
|
278
|
-
state = "loaded" if adapter else "empty"
|
|
279
|
-
return (
|
|
280
|
-
f"GraphLexicon(languages={sorted(self._languages)!r}, "
|
|
281
|
-
f"max_neighbors={self._max_neighbors}, seed={self.seed!r}, state={state})"
|
|
282
|
-
)
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
glitchlings/__init__.py,sha256=PggQt66bImHyWBggz5CTTFvB19P7iOCMjp2uncvcgXA,1177
|
|
2
|
-
glitchlings/__main__.py,sha256=nB7btO_T4wBFOcyawfWpjEindVrUfTqqV5hdeeS1HT8,128
|
|
3
|
-
glitchlings/_zoo_rust.cp312-win_amd64.pyd,sha256=_J4R5tA4q8CesIZ-K-3OJybUoy-NM_-kmHyo5-GE9vU,2103808
|
|
4
|
-
glitchlings/compat.py,sha256=qba3_W4W5edy9_hoPkekBRNhYy34eG2yCdyWaNrbxeg,7042
|
|
5
|
-
glitchlings/config.py,sha256=USnSw5-HO0ghsuwbmEvimZ4acU2EeNChB6RPlBFj4hc,13013
|
|
6
|
-
glitchlings/config.toml,sha256=EOEqKUwPygOv7MuEuExil5ZQfwKV1H2hU-Z5aBKQ440,111
|
|
7
|
-
glitchlings/main.py,sha256=JDgfHPDhRvZVb7bYQAFGHTRxH8yGNlgStlBqv-y5Thk,10444
|
|
8
|
-
glitchlings/dlc/__init__.py,sha256=IHD-GGhVFb7SVzErvf2YCJkOR4wGo0nFHXkn_daMvS8,146
|
|
9
|
-
glitchlings/dlc/_shared.py,sha256=i66HyJJiHpR0mug7zmfmxB17Vs3CSqBTsk5CuVaLDk0,2040
|
|
10
|
-
glitchlings/dlc/huggingface.py,sha256=Ym8dTArb-43AnCyukOO1m66iAbs8al9YkIWB3rGdhTk,2657
|
|
11
|
-
glitchlings/dlc/prime.py,sha256=KY3so8WOwksbsKhZXfWorsZSYvIdPiUtc8e9apHabkM,8861
|
|
12
|
-
glitchlings/lexicon/__init__.py,sha256=N1G-0xL-dkjRECEhLE1j7BrpBW6evqtjmcG5kiAczTM,6442
|
|
13
|
-
glitchlings/lexicon/_cache.py,sha256=cQCO-dizyClUz8FU8xyaeaJDBxqwrDo6v3YyJ-MQfp0,4052
|
|
14
|
-
glitchlings/lexicon/graph.py,sha256=00HQEbjNVXuBkgCyQy-ru6BB9GHDKiSJXv1TIillXRw,10339
|
|
15
|
-
glitchlings/lexicon/metrics.py,sha256=TZAafSKgHpUS4h6vCuhTKGsvu_fru9kMqsXqLID6BTM,4734
|
|
16
|
-
glitchlings/lexicon/vector.py,sha256=6QDQB2LAjRu1dZwlGYhkkdHyGv9UO79jUbf6KEtjvac,20243
|
|
17
|
-
glitchlings/lexicon/wordnet.py,sha256=YoEOMSIPrkrIORR3wMnPftKJLYh5PBQ7PTNrzZGchxg,6504
|
|
18
|
-
glitchlings/lexicon/data/default_vector_cache.json,sha256=fLT-v1sgF0lv88aPwOP23br9azrjeAkJ1ft6OgPlMeM,741
|
|
19
|
-
glitchlings/util/__init__.py,sha256=Q5lkncOaM6f2eJK3HAtZyxpCjGnekCpwPloqasS3JDo,4869
|
|
20
|
-
glitchlings/util/adapters.py,sha256=mFhPlE8JaFuO_C-3_aqhgwkqa6isV8Y2ifqVh3Iv9JM,720
|
|
21
|
-
glitchlings/zoo/__init__.py,sha256=EljxDKxbl6E4hTuC7Ul98dsp-FvJ2nXQTvD5LoWToWc,5271
|
|
22
|
-
glitchlings/zoo/_ocr_confusions.py,sha256=pPlvJOoan3ouwwGt8hATcO-9luIrGJl0vwUqssUMXD8,1236
|
|
23
|
-
glitchlings/zoo/_rate.py,sha256=o7B9_EfadjshSGLH0B5BHgj1J0OJWVCSXkEE8hljmxc,522
|
|
24
|
-
glitchlings/zoo/_sampling.py,sha256=AAPLObjqKrmX882TX8hdvPHReBOcv0Z4pUuW6AxuGgU,1640
|
|
25
|
-
glitchlings/zoo/_text_utils.py,sha256=LqCa33E-Qxbk6N5AVfxEmAz6C2u7_mCF0xPT9-404A8,2854
|
|
26
|
-
glitchlings/zoo/adjax.py,sha256=Rx8PxK9MBecaSXHJv7rXzWL2uP1QTdNuO2XohPZ61uo,3657
|
|
27
|
-
glitchlings/zoo/core.py,sha256=OE3zZYtyqlQoJkycxl5ZOG52Rxqi05Hd4rr2CAGUhrw,19929
|
|
28
|
-
glitchlings/zoo/jargoyle.py,sha256=fnSXlHlOlVFNKUP44V1TyXFmDYAt1dm1tV2dGL8hCFU,11773
|
|
29
|
-
glitchlings/zoo/mim1c.py,sha256=GqUMErVAVcqMAZjx4hhJ0Af25CxA0Aorv3U_fTqLZek,3546
|
|
30
|
-
glitchlings/zoo/ocr_confusions.tsv,sha256=S-IJEYCIXYKT1Uu7Id8Lnvg5pw528yNigTtWUdnMv9k,213
|
|
31
|
-
glitchlings/zoo/redactyl.py,sha256=0xGFuygN-vCsNplm3UddzFf2pWqvl7RShUFyXNdSzJ4,5627
|
|
32
|
-
glitchlings/zoo/reduple.py,sha256=lpKNqpSk-gzLKryxvobuikTpG3uOXZqPntyxmeeuPhg,4390
|
|
33
|
-
glitchlings/zoo/rushmore.py,sha256=_jU78_q_iJAWffjfq24WwQ2teyQ4brKOFoOU4IZnM7k,4459
|
|
34
|
-
glitchlings/zoo/scannequin.py,sha256=INTZlzskkXPy8zz6O19DBAbyRLQxvFK0AYhwjLfY7Ec,5055
|
|
35
|
-
glitchlings/zoo/typogre.py,sha256=zXvZX8V_0yDmEaLheWKBRa9vCEnSc9TiekYUqZITx0w,6889
|
|
36
|
-
glitchlings/zoo/zeedub.py,sha256=0k7IuTWXbMt5B-1IOwzGk29EvHZNkxyEQUlZloIcrOc,5006
|
|
37
|
-
glitchlings-0.4.2.dist-info/licenses/LICENSE,sha256=EFEP1evBfHaxsMTBjxm0sZVRp2wct8QLvHE1saII5FI,11538
|
|
38
|
-
glitchlings-0.4.2.dist-info/METADATA,sha256=9h2Wc3BbR0pm6d-_AfOcMvTJFL06cVuQFw43d2BVYns,31312
|
|
39
|
-
glitchlings-0.4.2.dist-info/WHEEL,sha256=8UP9x9puWI0P1V_d7K2oMTBqfeLNm21CTzZ_Ptr0NXU,101
|
|
40
|
-
glitchlings-0.4.2.dist-info/entry_points.txt,sha256=kGOwuAsjFDLtztLisaXtOouq9wFVMOJg5FzaAkg-Hto,54
|
|
41
|
-
glitchlings-0.4.2.dist-info/top_level.txt,sha256=VHFNBrLjtDwPCYXbGKi6o17Eueedi81eNbR3hBOoST0,12
|
|
42
|
-
glitchlings-0.4.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|