glitchlings 0.4.5__cp311-cp311-macosx_11_0_universal2.whl → 0.5.1__cp311-cp311-macosx_11_0_universal2.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of glitchlings might be problematic. Click here for more details.
- glitchlings/__init__.py +33 -0
- glitchlings/_zoo_rust.cpython-311-darwin.so +0 -0
- glitchlings/assets/ekkokin_homophones.json +1995 -0
- glitchlings/compat.py +98 -8
- glitchlings/config.py +12 -24
- glitchlings/dev/__init__.py +5 -0
- glitchlings/dev/sync_assets.py +130 -0
- glitchlings/dlc/pytorch_lightning.py +13 -1
- glitchlings/spectroll.py +5 -0
- glitchlings/util/stretchability.py +4 -9
- glitchlings/zoo/__init__.py +10 -2
- glitchlings/zoo/_ocr_confusions.py +3 -3
- glitchlings/zoo/_text_utils.py +10 -9
- glitchlings/zoo/adjax.py +3 -18
- glitchlings/zoo/apostrofae.py +2 -5
- glitchlings/zoo/assets/__init__.py +91 -0
- glitchlings/zoo/ekkokin.py +226 -0
- glitchlings/zoo/jargoyle.py +2 -16
- glitchlings/zoo/mim1c.py +2 -17
- glitchlings/zoo/redactyl.py +3 -17
- glitchlings/zoo/reduple.py +3 -17
- glitchlings/zoo/rushmore.py +3 -20
- glitchlings/zoo/scannequin.py +3 -20
- glitchlings/zoo/spectroll.py +159 -0
- glitchlings/zoo/typogre.py +2 -19
- glitchlings/zoo/zeedub.py +2 -13
- {glitchlings-0.4.5.dist-info → glitchlings-0.5.1.dist-info}/METADATA +22 -7
- glitchlings-0.5.1.dist-info/RECORD +57 -0
- glitchlings/data/__init__.py +0 -1
- glitchlings/zoo/_rate.py +0 -131
- glitchlings-0.4.5.dist-info/RECORD +0 -53
- /glitchlings/{zoo/assets → assets}/apostrofae_pairs.json +0 -0
- /glitchlings/{data → assets}/hokey_assets.json +0 -0
- /glitchlings/{zoo → assets}/ocr_confusions.tsv +0 -0
- {glitchlings-0.4.5.dist-info → glitchlings-0.5.1.dist-info}/WHEEL +0 -0
- {glitchlings-0.4.5.dist-info → glitchlings-0.5.1.dist-info}/entry_points.txt +0 -0
- {glitchlings-0.4.5.dist-info → glitchlings-0.5.1.dist-info}/licenses/LICENSE +0 -0
- {glitchlings-0.4.5.dist-info → glitchlings-0.5.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from functools import cache
|
|
5
|
+
from hashlib import blake2b
|
|
6
|
+
from importlib import resources
|
|
7
|
+
from importlib.resources.abc import Traversable
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, BinaryIO, Iterable, TextIO, cast
|
|
10
|
+
|
|
11
|
+
_DEFAULT_DIGEST_SIZE = 32
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _iter_asset_roots() -> Iterable[Traversable]:
|
|
15
|
+
"""Yield candidate locations for the shared glitchling asset bundle."""
|
|
16
|
+
|
|
17
|
+
package_root: Traversable | None
|
|
18
|
+
try:
|
|
19
|
+
package_root = resources.files("glitchlings").joinpath("assets")
|
|
20
|
+
except ModuleNotFoundError: # pragma: no cover - defensive guard for install issues
|
|
21
|
+
package_root = None
|
|
22
|
+
if package_root is not None and package_root.is_dir():
|
|
23
|
+
yield package_root
|
|
24
|
+
|
|
25
|
+
repo_root = Path(__file__).resolve().parents[4] / "assets"
|
|
26
|
+
if repo_root.is_dir():
|
|
27
|
+
yield cast(Traversable, repo_root)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _asset(name: str) -> Traversable:
|
|
31
|
+
asset_roots = list(_iter_asset_roots())
|
|
32
|
+
for root in asset_roots:
|
|
33
|
+
candidate = root.joinpath(name)
|
|
34
|
+
if candidate.is_file():
|
|
35
|
+
return candidate
|
|
36
|
+
|
|
37
|
+
searched = ", ".join(str(root.joinpath(name)) for root in asset_roots) or "<unavailable>"
|
|
38
|
+
raise FileNotFoundError(f"Asset '{name}' not found in: {searched}")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def read_text(name: str, *, encoding: str = "utf-8") -> str:
|
|
42
|
+
"""Return the decoded contents of a bundled text asset."""
|
|
43
|
+
|
|
44
|
+
return cast(str, _asset(name).read_text(encoding=encoding))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def open_text(name: str, *, encoding: str = "utf-8") -> TextIO:
|
|
48
|
+
"""Open a bundled text asset for reading."""
|
|
49
|
+
|
|
50
|
+
return cast(TextIO, _asset(name).open("r", encoding=encoding))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def open_binary(name: str) -> BinaryIO:
|
|
54
|
+
"""Open a bundled binary asset for reading."""
|
|
55
|
+
|
|
56
|
+
return cast(BinaryIO, _asset(name).open("rb"))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def load_json(name: str, *, encoding: str = "utf-8") -> Any:
|
|
60
|
+
"""Deserialize a JSON asset using the shared loader helpers."""
|
|
61
|
+
|
|
62
|
+
with open_text(name, encoding=encoding) as handle:
|
|
63
|
+
return json.load(handle)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def hash_asset(name: str) -> str:
|
|
67
|
+
"""Return a BLAKE2b digest for the bundled asset ``name``."""
|
|
68
|
+
|
|
69
|
+
digest = blake2b(digest_size=_DEFAULT_DIGEST_SIZE)
|
|
70
|
+
with open_binary(name) as handle:
|
|
71
|
+
for chunk in iter(lambda: handle.read(8192), b""):
|
|
72
|
+
digest.update(chunk)
|
|
73
|
+
return digest.hexdigest()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@cache
|
|
77
|
+
def load_homophone_groups(name: str = "ekkokin_homophones.json") -> tuple[tuple[str, ...], ...]:
|
|
78
|
+
"""Return the curated homophone sets bundled for the Ekkokin glitchling."""
|
|
79
|
+
|
|
80
|
+
data: list[list[str]] = load_json(name)
|
|
81
|
+
return tuple(tuple(group) for group in data)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
__all__ = [
|
|
85
|
+
"read_text",
|
|
86
|
+
"open_text",
|
|
87
|
+
"open_binary",
|
|
88
|
+
"load_json",
|
|
89
|
+
"hash_asset",
|
|
90
|
+
"load_homophone_groups",
|
|
91
|
+
]
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Homophone substitution glitchling implementation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import random
|
|
7
|
+
from typing import Any, Iterable, Mapping, Sequence, cast
|
|
8
|
+
|
|
9
|
+
from ._rust_extensions import get_rust_operation
|
|
10
|
+
from ._text_utils import WordToken, collect_word_tokens, split_preserving_whitespace
|
|
11
|
+
from .assets import load_homophone_groups
|
|
12
|
+
from .core import AttackOrder, AttackWave, Glitchling
|
|
13
|
+
|
|
14
|
+
_DEFAULT_RATE = 0.02
|
|
15
|
+
_DEFAULT_WEIGHTING = "flat"
|
|
16
|
+
_VALID_WEIGHTINGS = {_DEFAULT_WEIGHTING}
|
|
17
|
+
|
|
18
|
+
_homophone_groups: tuple[tuple[str, ...], ...] = load_homophone_groups()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _normalise_group(group: Sequence[str]) -> tuple[str, ...]:
|
|
22
|
+
"""Return a tuple of lowercase homophones preserving original order."""
|
|
23
|
+
|
|
24
|
+
# Use dict.fromkeys to preserve the original ordering while de-duplicating.
|
|
25
|
+
return tuple(dict.fromkeys(word.lower() for word in group if word))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _build_lookup(groups: Iterable[Sequence[str]]) -> Mapping[str, tuple[str, ...]]:
|
|
29
|
+
"""Return a mapping from word -> homophone group."""
|
|
30
|
+
|
|
31
|
+
lookup: dict[str, tuple[str, ...]] = {}
|
|
32
|
+
for group in groups:
|
|
33
|
+
normalised = _normalise_group(group)
|
|
34
|
+
if len(normalised) < 2:
|
|
35
|
+
continue
|
|
36
|
+
for word in normalised:
|
|
37
|
+
lookup[word] = normalised
|
|
38
|
+
return lookup
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
_homophone_lookup = _build_lookup(_homophone_groups)
|
|
42
|
+
_ekkokin_rust = get_rust_operation("ekkokin_homophones")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _normalise_weighting(weighting: str | None) -> str:
|
|
46
|
+
if weighting is None:
|
|
47
|
+
return _DEFAULT_WEIGHTING
|
|
48
|
+
lowered = weighting.lower()
|
|
49
|
+
if lowered not in _VALID_WEIGHTINGS:
|
|
50
|
+
options = ", ".join(sorted(_VALID_WEIGHTINGS))
|
|
51
|
+
raise ValueError(f"Unsupported weighting '{weighting}'. Expected one of: {options}")
|
|
52
|
+
return lowered
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _apply_casing(template: str, candidate: str) -> str:
|
|
56
|
+
"""Return ``candidate`` adjusted to mirror the casing pattern of ``template``."""
|
|
57
|
+
|
|
58
|
+
if not candidate:
|
|
59
|
+
return candidate
|
|
60
|
+
if template.isupper():
|
|
61
|
+
return candidate.upper()
|
|
62
|
+
if template.islower():
|
|
63
|
+
return candidate.lower()
|
|
64
|
+
if template[:1].isupper() and template[1:].islower():
|
|
65
|
+
return candidate.capitalize()
|
|
66
|
+
return candidate
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _choose_alternative(
|
|
70
|
+
*,
|
|
71
|
+
group: Sequence[str],
|
|
72
|
+
source_word: str,
|
|
73
|
+
weighting: str,
|
|
74
|
+
rng: random.Random,
|
|
75
|
+
) -> str | None:
|
|
76
|
+
"""Return a replacement for ``source_word`` drawn from ``group``."""
|
|
77
|
+
|
|
78
|
+
del weighting # Reserved for future weighting strategies.
|
|
79
|
+
lowered = source_word.lower()
|
|
80
|
+
candidates = [candidate for candidate in group if candidate != lowered]
|
|
81
|
+
if not candidates:
|
|
82
|
+
return None
|
|
83
|
+
index = rng.randrange(len(candidates))
|
|
84
|
+
replacement = candidates[index]
|
|
85
|
+
return _apply_casing(source_word, replacement)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _python_substitute_homophones(
|
|
89
|
+
text: str,
|
|
90
|
+
*,
|
|
91
|
+
rate: float,
|
|
92
|
+
weighting: str,
|
|
93
|
+
rng: random.Random,
|
|
94
|
+
) -> str:
|
|
95
|
+
"""Replace words in ``text`` with curated homophones."""
|
|
96
|
+
|
|
97
|
+
if not text:
|
|
98
|
+
return text
|
|
99
|
+
|
|
100
|
+
if math.isnan(rate):
|
|
101
|
+
return text
|
|
102
|
+
|
|
103
|
+
clamped_rate = max(0.0, min(1.0, rate))
|
|
104
|
+
if clamped_rate <= 0.0:
|
|
105
|
+
return text
|
|
106
|
+
|
|
107
|
+
tokens = split_preserving_whitespace(text)
|
|
108
|
+
word_tokens = collect_word_tokens(tokens)
|
|
109
|
+
if not word_tokens:
|
|
110
|
+
return text
|
|
111
|
+
|
|
112
|
+
mutated = False
|
|
113
|
+
for token in word_tokens:
|
|
114
|
+
replacement = _maybe_replace_token(token, clamped_rate, weighting, rng)
|
|
115
|
+
if replacement is None:
|
|
116
|
+
continue
|
|
117
|
+
tokens[token.index] = replacement
|
|
118
|
+
mutated = True
|
|
119
|
+
|
|
120
|
+
if not mutated:
|
|
121
|
+
return text
|
|
122
|
+
return "".join(tokens)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _maybe_replace_token(
|
|
126
|
+
token: WordToken,
|
|
127
|
+
rate: float,
|
|
128
|
+
weighting: str,
|
|
129
|
+
rng: random.Random,
|
|
130
|
+
) -> str | None:
|
|
131
|
+
lookup = _homophone_lookup.get(token.core.lower())
|
|
132
|
+
if lookup is None:
|
|
133
|
+
return None
|
|
134
|
+
if rng.random() >= rate:
|
|
135
|
+
return None
|
|
136
|
+
replacement_core = _choose_alternative(
|
|
137
|
+
group=lookup,
|
|
138
|
+
source_word=token.core,
|
|
139
|
+
weighting=weighting,
|
|
140
|
+
rng=rng,
|
|
141
|
+
)
|
|
142
|
+
if replacement_core is None:
|
|
143
|
+
return None
|
|
144
|
+
return f"{token.prefix}{replacement_core}{token.suffix}"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def substitute_homophones(
|
|
148
|
+
text: str,
|
|
149
|
+
rate: float | None = None,
|
|
150
|
+
seed: int | None = None,
|
|
151
|
+
rng: random.Random | None = None,
|
|
152
|
+
*,
|
|
153
|
+
weighting: str | None = None,
|
|
154
|
+
) -> str:
|
|
155
|
+
"""Replace words in ``text`` with curated homophones."""
|
|
156
|
+
|
|
157
|
+
effective_rate = _DEFAULT_RATE if rate is None else rate
|
|
158
|
+
normalized_weighting = _normalise_weighting(weighting)
|
|
159
|
+
|
|
160
|
+
active_rng = rng if rng is not None else random.Random(seed)
|
|
161
|
+
|
|
162
|
+
clamped_rate = 0.0 if math.isnan(effective_rate) else max(0.0, min(1.0, effective_rate))
|
|
163
|
+
if _ekkokin_rust is not None:
|
|
164
|
+
return cast(
|
|
165
|
+
str,
|
|
166
|
+
_ekkokin_rust(text, clamped_rate, normalized_weighting, active_rng),
|
|
167
|
+
)
|
|
168
|
+
return _python_substitute_homophones(
|
|
169
|
+
text,
|
|
170
|
+
rate=clamped_rate,
|
|
171
|
+
weighting=normalized_weighting,
|
|
172
|
+
rng=active_rng,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class Ekkokin(Glitchling):
|
|
177
|
+
"""Glitchling that swaps words for curated homophones."""
|
|
178
|
+
|
|
179
|
+
def __init__(
|
|
180
|
+
self,
|
|
181
|
+
*,
|
|
182
|
+
rate: float | None = None,
|
|
183
|
+
seed: int | None = None,
|
|
184
|
+
weighting: str | None = None,
|
|
185
|
+
) -> None:
|
|
186
|
+
effective_rate = _DEFAULT_RATE if rate is None else rate
|
|
187
|
+
normalized_weighting = _normalise_weighting(weighting)
|
|
188
|
+
super().__init__(
|
|
189
|
+
name="Ekkokin",
|
|
190
|
+
corruption_function=substitute_homophones,
|
|
191
|
+
scope=AttackWave.WORD,
|
|
192
|
+
order=AttackOrder.EARLY,
|
|
193
|
+
seed=seed,
|
|
194
|
+
pipeline_operation=_build_pipeline_descriptor,
|
|
195
|
+
rate=effective_rate,
|
|
196
|
+
weighting=normalized_weighting,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def set_param(self, key: str, value: Any) -> None:
|
|
200
|
+
"""Normalise weighting updates before storing them on the glitchling."""
|
|
201
|
+
if key == "weighting":
|
|
202
|
+
value = _normalise_weighting(cast(str | None, value))
|
|
203
|
+
super().set_param(key, value)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _build_pipeline_descriptor(glitch: Glitchling) -> dict[str, object] | None:
|
|
207
|
+
rate = glitch.kwargs.get("rate")
|
|
208
|
+
if rate is None:
|
|
209
|
+
return None
|
|
210
|
+
weighting = _normalise_weighting(cast(str | None, glitch.kwargs.get("weighting")))
|
|
211
|
+
return {
|
|
212
|
+
"type": "ekkokin",
|
|
213
|
+
"rate": float(rate),
|
|
214
|
+
"weighting": str(weighting),
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
ekkokin = Ekkokin()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
__all__ = [
|
|
222
|
+
"Ekkokin",
|
|
223
|
+
"ekkokin",
|
|
224
|
+
"substitute_homophones",
|
|
225
|
+
"_python_substitute_homophones",
|
|
226
|
+
]
|
glitchlings/zoo/jargoyle.py
CHANGED
|
@@ -7,7 +7,6 @@ from typing import Any, Literal, cast
|
|
|
7
7
|
|
|
8
8
|
from glitchlings.lexicon import Lexicon, get_default_lexicon
|
|
9
9
|
|
|
10
|
-
from ._rate import resolve_rate
|
|
11
10
|
from .core import AttackWave, Glitchling
|
|
12
11
|
|
|
13
12
|
_wordnet_module: ModuleType | None
|
|
@@ -119,7 +118,6 @@ def substitute_random_synonyms(
|
|
|
119
118
|
seed: int | None = None,
|
|
120
119
|
rng: random.Random | None = None,
|
|
121
120
|
*,
|
|
122
|
-
replacement_rate: float | None = None,
|
|
123
121
|
lexicon: Lexicon | None = None,
|
|
124
122
|
) -> str:
|
|
125
123
|
"""Replace words with random lexicon-driven synonyms.
|
|
@@ -144,12 +142,7 @@ def substitute_random_synonyms(
|
|
|
144
142
|
deterministic subsets per word and part-of-speech using the active seed.
|
|
145
143
|
|
|
146
144
|
"""
|
|
147
|
-
effective_rate =
|
|
148
|
-
rate=rate,
|
|
149
|
-
legacy_value=replacement_rate,
|
|
150
|
-
default=0.1,
|
|
151
|
-
legacy_name="replacement_rate",
|
|
152
|
-
)
|
|
145
|
+
effective_rate = 0.1 if rate is None else rate
|
|
153
146
|
|
|
154
147
|
active_rng: random.Random
|
|
155
148
|
if rng is not None:
|
|
@@ -258,23 +251,16 @@ class Jargoyle(Glitchling):
|
|
|
258
251
|
self,
|
|
259
252
|
*,
|
|
260
253
|
rate: float | None = None,
|
|
261
|
-
replacement_rate: float | None = None,
|
|
262
254
|
part_of_speech: PartOfSpeechInput = "n",
|
|
263
255
|
seed: int | None = None,
|
|
264
256
|
lexicon: Lexicon | None = None,
|
|
265
257
|
) -> None:
|
|
266
|
-
self._param_aliases = {"replacement_rate": "rate"}
|
|
267
258
|
self._owns_lexicon = lexicon is None
|
|
268
259
|
self._external_lexicon_original_seed = (
|
|
269
260
|
lexicon.seed if isinstance(lexicon, Lexicon) else None
|
|
270
261
|
)
|
|
271
262
|
self._initializing = True
|
|
272
|
-
effective_rate =
|
|
273
|
-
rate=rate,
|
|
274
|
-
legacy_value=replacement_rate,
|
|
275
|
-
default=0.01,
|
|
276
|
-
legacy_name="replacement_rate",
|
|
277
|
-
)
|
|
263
|
+
effective_rate = 0.01 if rate is None else rate
|
|
278
264
|
prepared_lexicon = lexicon or get_default_lexicon(seed=seed)
|
|
279
265
|
if lexicon is not None and seed is not None:
|
|
280
266
|
prepared_lexicon.reseed(seed)
|
glitchlings/zoo/mim1c.py
CHANGED
|
@@ -4,7 +4,6 @@ from typing import Literal
|
|
|
4
4
|
|
|
5
5
|
from confusable_homoglyphs import confusables
|
|
6
6
|
|
|
7
|
-
from ._rate import resolve_rate
|
|
8
7
|
from .core import AttackOrder, AttackWave, Glitchling
|
|
9
8
|
|
|
10
9
|
|
|
@@ -15,8 +14,6 @@ def swap_homoglyphs(
|
|
|
15
14
|
banned_characters: Collection[str] | None = None,
|
|
16
15
|
seed: int | None = None,
|
|
17
16
|
rng: random.Random | None = None,
|
|
18
|
-
*,
|
|
19
|
-
replacement_rate: float | None = None,
|
|
20
17
|
) -> str:
|
|
21
18
|
"""Replace characters with visually confusable homoglyphs.
|
|
22
19
|
|
|
@@ -37,12 +34,7 @@ def swap_homoglyphs(
|
|
|
37
34
|
- Maintains determinism by shuffling candidates and sampling via the provided RNG.
|
|
38
35
|
|
|
39
36
|
"""
|
|
40
|
-
effective_rate =
|
|
41
|
-
rate=rate,
|
|
42
|
-
legacy_value=replacement_rate,
|
|
43
|
-
default=0.02,
|
|
44
|
-
legacy_name="replacement_rate",
|
|
45
|
-
)
|
|
37
|
+
effective_rate = 0.02 if rate is None else rate
|
|
46
38
|
|
|
47
39
|
if rng is None:
|
|
48
40
|
rng = random.Random(seed)
|
|
@@ -79,18 +71,11 @@ class Mim1c(Glitchling):
|
|
|
79
71
|
self,
|
|
80
72
|
*,
|
|
81
73
|
rate: float | None = None,
|
|
82
|
-
replacement_rate: float | None = None,
|
|
83
74
|
classes: list[str] | Literal["all"] | None = None,
|
|
84
75
|
banned_characters: Collection[str] | None = None,
|
|
85
76
|
seed: int | None = None,
|
|
86
77
|
) -> None:
|
|
87
|
-
|
|
88
|
-
effective_rate = resolve_rate(
|
|
89
|
-
rate=rate,
|
|
90
|
-
legacy_value=replacement_rate,
|
|
91
|
-
default=0.02,
|
|
92
|
-
legacy_name="replacement_rate",
|
|
93
|
-
)
|
|
78
|
+
effective_rate = 0.02 if rate is None else rate
|
|
94
79
|
super().__init__(
|
|
95
80
|
name="Mim1c",
|
|
96
81
|
corruption_function=swap_homoglyphs,
|
glitchlings/zoo/redactyl.py
CHANGED
|
@@ -2,7 +2,6 @@ import random
|
|
|
2
2
|
import re
|
|
3
3
|
from typing import Any, cast
|
|
4
4
|
|
|
5
|
-
from ._rate import resolve_rate
|
|
6
5
|
from ._rust_extensions import get_rust_operation
|
|
7
6
|
from ._sampling import weighted_sample_without_replacement
|
|
8
7
|
from ._text_utils import (
|
|
@@ -97,16 +96,10 @@ def redact_words(
|
|
|
97
96
|
seed: int = 151,
|
|
98
97
|
rng: random.Random | None = None,
|
|
99
98
|
*,
|
|
100
|
-
redaction_rate: float | None = None,
|
|
101
99
|
unweighted: bool = False,
|
|
102
100
|
) -> str:
|
|
103
101
|
"""Redact random words by replacing their characters."""
|
|
104
|
-
effective_rate =
|
|
105
|
-
rate=rate,
|
|
106
|
-
legacy_value=redaction_rate,
|
|
107
|
-
default=0.025,
|
|
108
|
-
legacy_name="redaction_rate",
|
|
109
|
-
)
|
|
102
|
+
effective_rate = 0.025 if rate is None else rate
|
|
110
103
|
|
|
111
104
|
if rng is None:
|
|
112
105
|
rng = random.Random(seed)
|
|
@@ -148,18 +141,11 @@ class Redactyl(Glitchling):
|
|
|
148
141
|
*,
|
|
149
142
|
replacement_char: str = FULL_BLOCK,
|
|
150
143
|
rate: float | None = None,
|
|
151
|
-
redaction_rate: float | None = None,
|
|
152
144
|
merge_adjacent: bool = False,
|
|
153
145
|
seed: int = 151,
|
|
154
146
|
unweighted: bool = False,
|
|
155
147
|
) -> None:
|
|
156
|
-
|
|
157
|
-
effective_rate = resolve_rate(
|
|
158
|
-
rate=rate,
|
|
159
|
-
legacy_value=redaction_rate,
|
|
160
|
-
default=0.025,
|
|
161
|
-
legacy_name="redaction_rate",
|
|
162
|
-
)
|
|
148
|
+
effective_rate = 0.025 if rate is None else rate
|
|
163
149
|
super().__init__(
|
|
164
150
|
name="Redactyl",
|
|
165
151
|
corruption_function=redact_words,
|
|
@@ -181,7 +167,7 @@ class Redactyl(Glitchling):
|
|
|
181
167
|
return {
|
|
182
168
|
"type": "redact",
|
|
183
169
|
"replacement_char": str(replacement_char),
|
|
184
|
-
"
|
|
170
|
+
"rate": float(rate),
|
|
185
171
|
"merge_adjacent": bool(merge_adjacent),
|
|
186
172
|
"unweighted": unweighted,
|
|
187
173
|
}
|
glitchlings/zoo/reduple.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import random
|
|
2
2
|
from typing import Any, cast
|
|
3
3
|
|
|
4
|
-
from ._rate import resolve_rate
|
|
5
4
|
from ._rust_extensions import get_rust_operation
|
|
6
5
|
from ._text_utils import WordToken, collect_word_tokens, split_preserving_whitespace
|
|
7
6
|
from .core import AttackWave, Glitchling
|
|
@@ -71,7 +70,6 @@ def reduplicate_words(
|
|
|
71
70
|
seed: int | None = None,
|
|
72
71
|
rng: random.Random | None = None,
|
|
73
72
|
*,
|
|
74
|
-
reduplication_rate: float | None = None,
|
|
75
73
|
unweighted: bool = False,
|
|
76
74
|
) -> str:
|
|
77
75
|
"""Randomly reduplicate words in the text.
|
|
@@ -79,12 +77,7 @@ def reduplicate_words(
|
|
|
79
77
|
Falls back to the Python implementation when the optional Rust
|
|
80
78
|
extension is unavailable.
|
|
81
79
|
"""
|
|
82
|
-
effective_rate =
|
|
83
|
-
rate=rate,
|
|
84
|
-
legacy_value=reduplication_rate,
|
|
85
|
-
default=0.01,
|
|
86
|
-
legacy_name="reduplication_rate",
|
|
87
|
-
)
|
|
80
|
+
effective_rate = 0.01 if rate is None else rate
|
|
88
81
|
|
|
89
82
|
if rng is None:
|
|
90
83
|
rng = random.Random(seed)
|
|
@@ -110,17 +103,10 @@ class Reduple(Glitchling):
|
|
|
110
103
|
self,
|
|
111
104
|
*,
|
|
112
105
|
rate: float | None = None,
|
|
113
|
-
reduplication_rate: float | None = None,
|
|
114
106
|
seed: int | None = None,
|
|
115
107
|
unweighted: bool = False,
|
|
116
108
|
) -> None:
|
|
117
|
-
|
|
118
|
-
effective_rate = resolve_rate(
|
|
119
|
-
rate=rate,
|
|
120
|
-
legacy_value=reduplication_rate,
|
|
121
|
-
default=0.01,
|
|
122
|
-
legacy_name="reduplication_rate",
|
|
123
|
-
)
|
|
109
|
+
effective_rate = 0.01 if rate is None else rate
|
|
124
110
|
super().__init__(
|
|
125
111
|
name="Reduple",
|
|
126
112
|
corruption_function=reduplicate_words,
|
|
@@ -137,7 +123,7 @@ class Reduple(Glitchling):
|
|
|
137
123
|
unweighted = bool(self.kwargs.get("unweighted", False))
|
|
138
124
|
return {
|
|
139
125
|
"type": "reduplicate",
|
|
140
|
-
"
|
|
126
|
+
"rate": float(rate),
|
|
141
127
|
"unweighted": unweighted,
|
|
142
128
|
}
|
|
143
129
|
|
glitchlings/zoo/rushmore.py
CHANGED
|
@@ -3,7 +3,6 @@ import random
|
|
|
3
3
|
import re
|
|
4
4
|
from typing import Any, cast
|
|
5
5
|
|
|
6
|
-
from ._rate import resolve_rate
|
|
7
6
|
from ._rust_extensions import get_rust_operation
|
|
8
7
|
from ._text_utils import WordToken, collect_word_tokens, split_preserving_whitespace
|
|
9
8
|
from .core import AttackWave, Glitchling
|
|
@@ -74,20 +73,13 @@ def delete_random_words(
|
|
|
74
73
|
rate: float | None = None,
|
|
75
74
|
seed: int | None = None,
|
|
76
75
|
rng: random.Random | None = None,
|
|
77
|
-
*,
|
|
78
|
-
max_deletion_rate: float | None = None,
|
|
79
76
|
unweighted: bool = False,
|
|
80
77
|
) -> str:
|
|
81
78
|
"""Delete random words from the input text.
|
|
82
79
|
|
|
83
80
|
Uses the optional Rust implementation when available.
|
|
84
81
|
"""
|
|
85
|
-
effective_rate =
|
|
86
|
-
rate=rate,
|
|
87
|
-
legacy_value=max_deletion_rate,
|
|
88
|
-
default=0.01,
|
|
89
|
-
legacy_name="max_deletion_rate",
|
|
90
|
-
)
|
|
82
|
+
effective_rate = 0.01 if rate is None else rate
|
|
91
83
|
|
|
92
84
|
if rng is None:
|
|
93
85
|
rng = random.Random(seed)
|
|
@@ -113,17 +105,10 @@ class Rushmore(Glitchling):
|
|
|
113
105
|
self,
|
|
114
106
|
*,
|
|
115
107
|
rate: float | None = None,
|
|
116
|
-
max_deletion_rate: float | None = None,
|
|
117
108
|
seed: int | None = None,
|
|
118
109
|
unweighted: bool = False,
|
|
119
110
|
) -> None:
|
|
120
|
-
|
|
121
|
-
effective_rate = resolve_rate(
|
|
122
|
-
rate=rate,
|
|
123
|
-
legacy_value=max_deletion_rate,
|
|
124
|
-
default=0.01,
|
|
125
|
-
legacy_name="max_deletion_rate",
|
|
126
|
-
)
|
|
111
|
+
effective_rate = 0.01 if rate is None else rate
|
|
127
112
|
super().__init__(
|
|
128
113
|
name="Rushmore",
|
|
129
114
|
corruption_function=delete_random_words,
|
|
@@ -135,14 +120,12 @@ class Rushmore(Glitchling):
|
|
|
135
120
|
|
|
136
121
|
def pipeline_operation(self) -> dict[str, Any] | None:
|
|
137
122
|
rate = self.kwargs.get("rate")
|
|
138
|
-
if rate is None:
|
|
139
|
-
rate = self.kwargs.get("max_deletion_rate")
|
|
140
123
|
if rate is None:
|
|
141
124
|
return None
|
|
142
125
|
unweighted = bool(self.kwargs.get("unweighted", False))
|
|
143
126
|
return {
|
|
144
127
|
"type": "delete",
|
|
145
|
-
"
|
|
128
|
+
"rate": float(rate),
|
|
146
129
|
"unweighted": unweighted,
|
|
147
130
|
}
|
|
148
131
|
|
glitchlings/zoo/scannequin.py
CHANGED
|
@@ -3,7 +3,6 @@ import re
|
|
|
3
3
|
from typing import Any, cast
|
|
4
4
|
|
|
5
5
|
from ._ocr_confusions import load_confusion_table
|
|
6
|
-
from ._rate import resolve_rate
|
|
7
6
|
from ._rust_extensions import get_rust_operation
|
|
8
7
|
from .core import AttackOrder, AttackWave, Glitchling
|
|
9
8
|
|
|
@@ -102,8 +101,6 @@ def ocr_artifacts(
|
|
|
102
101
|
rate: float | None = None,
|
|
103
102
|
seed: int | None = None,
|
|
104
103
|
rng: random.Random | None = None,
|
|
105
|
-
*,
|
|
106
|
-
error_rate: float | None = None,
|
|
107
104
|
) -> str:
|
|
108
105
|
"""Introduce OCR-like artifacts into text.
|
|
109
106
|
|
|
@@ -112,12 +109,7 @@ def ocr_artifacts(
|
|
|
112
109
|
if not text:
|
|
113
110
|
return text
|
|
114
111
|
|
|
115
|
-
effective_rate =
|
|
116
|
-
rate=rate,
|
|
117
|
-
legacy_value=error_rate,
|
|
118
|
-
default=0.02,
|
|
119
|
-
legacy_name="error_rate",
|
|
120
|
-
)
|
|
112
|
+
effective_rate = 0.02 if rate is None else rate
|
|
121
113
|
|
|
122
114
|
if rng is None:
|
|
123
115
|
rng = random.Random(seed)
|
|
@@ -137,16 +129,9 @@ class Scannequin(Glitchling):
|
|
|
137
129
|
self,
|
|
138
130
|
*,
|
|
139
131
|
rate: float | None = None,
|
|
140
|
-
error_rate: float | None = None,
|
|
141
132
|
seed: int | None = None,
|
|
142
133
|
) -> None:
|
|
143
|
-
|
|
144
|
-
effective_rate = resolve_rate(
|
|
145
|
-
rate=rate,
|
|
146
|
-
legacy_value=error_rate,
|
|
147
|
-
default=0.02,
|
|
148
|
-
legacy_name="error_rate",
|
|
149
|
-
)
|
|
134
|
+
effective_rate = 0.02 if rate is None else rate
|
|
150
135
|
super().__init__(
|
|
151
136
|
name="Scannequin",
|
|
152
137
|
corruption_function=ocr_artifacts,
|
|
@@ -158,11 +143,9 @@ class Scannequin(Glitchling):
|
|
|
158
143
|
|
|
159
144
|
def pipeline_operation(self) -> dict[str, Any] | None:
|
|
160
145
|
rate = self.kwargs.get("rate")
|
|
161
|
-
if rate is None:
|
|
162
|
-
rate = self.kwargs.get("error_rate")
|
|
163
146
|
if rate is None:
|
|
164
147
|
return None
|
|
165
|
-
return {"type": "ocr", "
|
|
148
|
+
return {"type": "ocr", "rate": float(rate)}
|
|
166
149
|
|
|
167
150
|
|
|
168
151
|
scannequin = Scannequin()
|