entropy-protocol 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- entropy_protocol/__init__.py +11 -0
- entropy_protocol/_unicode14.py +120 -0
- entropy_protocol/attest.py +62 -0
- entropy_protocol/attested/__init__.py +6 -0
- entropy_protocol/attested/amd.py +221 -0
- entropy_protocol/attested/appraise.py +325 -0
- entropy_protocol/attested/attester.py +58 -0
- entropy_protocol/attested/authority.py +239 -0
- entropy_protocol/attested/azure_cgpu.py +186 -0
- entropy_protocol/attested/binding.py +71 -0
- entropy_protocol/attested/capture.py +109 -0
- entropy_protocol/attested/ear.py +94 -0
- entropy_protocol/attested/enroll.py +119 -0
- entropy_protocol/attested/evidence.py +106 -0
- entropy_protocol/attested/hcl.py +109 -0
- entropy_protocol/attested/nvidia.py +168 -0
- entropy_protocol/attested/ocsp.py +90 -0
- entropy_protocol/attested/pki.py +121 -0
- entropy_protocol/attested/rim.py +130 -0
- entropy_protocol/attested/service.py +242 -0
- entropy_protocol/attested/tpm.py +259 -0
- entropy_protocol/backend_hf.py +439 -0
- entropy_protocol/batch.py +537 -0
- entropy_protocol/calibrate.py +99 -0
- entropy_protocol/canon.py +321 -0
- entropy_protocol/canongen.py +325 -0
- entropy_protocol/cli.py +493 -0
- entropy_protocol/cose.py +260 -0
- entropy_protocol/crypto.py +140 -0
- entropy_protocol/delivery.py +31 -0
- entropy_protocol/detect.py +246 -0
- entropy_protocol/gateway.py +370 -0
- entropy_protocol/hpke.py +105 -0
- entropy_protocol/mcp.py +159 -0
- entropy_protocol/pack.py +217 -0
- entropy_protocol/plain.py +122 -0
- entropy_protocol/predicates.py +37 -0
- entropy_protocol/profiles.py +45 -0
- entropy_protocol/record.py +116 -0
- entropy_protocol/scrapi.py +203 -0
- entropy_protocol/seal.py +232 -0
- entropy_protocol/statements.py +148 -0
- entropy_protocol/stats.py +20 -0
- entropy_protocol/tlog.py +147 -0
- entropy_protocol/ts3161.py +205 -0
- entropy_protocol/typography.py +73 -0
- entropy_protocol/verify.py +232 -0
- entropy_protocol-0.1.0.dist-info/METADATA +255 -0
- entropy_protocol-0.1.0.dist-info/RECORD +54 -0
- entropy_protocol-0.1.0.dist-info/WHEEL +5 -0
- entropy_protocol-0.1.0.dist-info/entry_points.txt +2 -0
- entropy_protocol-0.1.0.dist-info/licenses/LICENSE +201 -0
- entropy_protocol-0.1.0.dist-info/licenses/NOTICE +4 -0
- entropy_protocol-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"""Canonical text form shared by sealer and verifier.
|
|
2
|
+
|
|
3
|
+
A record is measured on the canonical form of the final text, so anything the canonical form
|
|
4
|
+
erases — carriage returns, byte-order marks, invisible format characters, an embedded C2PA text
|
|
5
|
+
manifest, no-break spaces, trailing spaces at line ends, blank lines around a paste, and (v4) the
|
|
6
|
+
typographic substitutions word processors make on paste — cannot break verification. The transform is
|
|
7
|
+
deterministic and idempotent:
|
|
8
|
+
1. Unicode NFC
|
|
9
|
+
2. delete every U+000D (CR) and U+FEFF (BOM): LF is the only line break, so CRLF, a naive
|
|
10
|
+
LF -> CRLF conversion of text that already holds CRLF (CR CR LF) and stray CRs all agree
|
|
11
|
+
3. delete invisible format characters that carry no content: U+200B (zero-width space),
|
|
12
|
+
U+2060 (word joiner), U+00AD (soft hyphen). U+200C/U+200D (ZWNJ/ZWJ) are kept: scripts and
|
|
13
|
+
emoji sequences depend on them
|
|
14
|
+
4. delete embedded payloads written as variation selectors (U+FE00-FE0F, U+E0100-E01EF): a U+FEFF
|
|
15
|
+
followed by 8 or more selectors (C2PA 2.4 A.8 text-manifest wrapper) is removed as a unit, then
|
|
16
|
+
any remaining run of 8 or more selectors; a single selector after a base character (e.g. an emoji
|
|
17
|
+
presentation selector) is content and is kept
|
|
18
|
+
5. U+00A0, U+2007, U+202F (no-break spaces) -> U+0020
|
|
19
|
+
6. (v4) typographic folding, the inverse of what word processors do on paste (and of Ukrainian
|
|
20
|
+
orthography): double quotes U+201C U+201D U+201E U+201F U+00AB U+00BB -> U+0022; apostrophes and single
|
|
21
|
+
quotes U+2018 U+2019 U+201A U+201B U+02BC -> U+0027; hyphens and en dash U+2010-U+2013, minus U+2212 ->
|
|
22
|
+
U+002D; em dash U+2014 and horizontal bar U+2015 -> "--" (Word turns " - " into an en dash and "--" into
|
|
23
|
+
an em dash); ellipsis U+2026 -> "..." (Word turns "..." into an ellipsis)
|
|
24
|
+
7. delete spaces and tabs that end a line or the text; strip the text
|
|
25
|
+
Blocks (content layer): sentence-like spans, split at line breaks and after . ! ? followed by
|
|
26
|
+
whitespace; v4 also after the full stops of scripts written without spaces (Chinese/Japanese, Arabic,
|
|
27
|
+
Devanagari, Ethiopic, Armenian) and keeps closing quotes and brackets with their sentence.
|
|
28
|
+
Records name their version; a verifier reads every supported version.
|
|
29
|
+
"""
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import unicodedata
|
|
33
|
+
|
|
34
|
+
CANON_V3 = "ep1-canon/v3:nfc+del-cr-bom-zw-shy+del-vs-runs+nbsp+rstrip-lines+strip"
|
|
35
|
+
CANON_V4 = "ep1-canon/v4:nfc+del-cr-bom-zw-shy+del-vs-runs+nbsp+typo+rstrip-lines+strip;seg2"
|
|
36
|
+
# v5: the deletions before NFC. In v4 a deleted character could leave combining marks adjacent in a non-canonical
|
|
37
|
+
# order, or a base and a combining mark that NFC would compose: canonicalize(canonicalize(x)) != canonicalize(x), so a
|
|
38
|
+
# zero-width character inserted between such marks broke verification after transport (found by fuzzing, 27.10.2026:
|
|
39
|
+
# U+0E48 U+0301 ZWJ U+0E48 U+0301). With NFC after every deletion, and only starters replaced or removed afterwards,
|
|
40
|
+
# the form is idempotent by construction (tests/test_fuzz.py checks it).
|
|
41
|
+
# NFC is that of Unicode 14.0 (the oldest Python the package supports): the text is cut at every code point not
|
|
42
|
+
# assigned in 14.0 - to 14.0 a starter nothing composes or reorders across - and the pieces normalized; by Unicode's
|
|
43
|
+
# normalization stability policy every later version normalizes them the same way. Without the pin a combining mark
|
|
44
|
+
# assigned later reorders under a newer library only (fuzzing, 27.10.2026: U+1AE8 U+0E48 - Node 26 with Unicode 17
|
|
45
|
+
# swapped them, Python 3.12 with Unicode 15.0 did not).
|
|
46
|
+
CANON_V5 = "ep1-canon/v5:del-cr-bom-zw-shy+del-vs-runs+nfc-u14+nbsp+typo+rstrip-lines+strip;seg2"
|
|
47
|
+
CANON_ID = CANON_V5 # new records
|
|
48
|
+
SUPPORTED = (CANON_V3, CANON_V4, CANON_V5)
|
|
49
|
+
_TRAILING = " \t"
|
|
50
|
+
_DELETE = {"\r", "", "", "", ""}
|
|
51
|
+
_SPACE = {" ", " ", " "}
|
|
52
|
+
_VS_RUN = 8
|
|
53
|
+
# v4 typographic folding (code points written as numbers: the source stays ASCII)
|
|
54
|
+
_FOLD = {**{chr(c): '"' for c in (0x201C, 0x201D, 0x201E, 0x201F, 0x00AB, 0x00BB)},
|
|
55
|
+
**{chr(c): "'" for c in (0x2018, 0x2019, 0x201A, 0x201B, 0x02BC)},
|
|
56
|
+
**{chr(c): "-" for c in (0x2010, 0x2011, 0x2012, 0x2013, 0x2212)},
|
|
57
|
+
**{chr(c): "--" for c in (0x2014, 0x2015)},
|
|
58
|
+
chr(0x2026): "..."}
|
|
59
|
+
# v4 segmentation: full stops of scripts written without spaces, closing quotes and brackets
|
|
60
|
+
_SCRIPT_STOPS = frozenset(map(chr, (0x3002, 0xFF01, 0xFF1F, 0xFF0E, 0x061F, 0x06D4, 0x0964, 0x0965, 0x1362, 0x0589)))
|
|
61
|
+
_CLOSERS = frozenset(")]}\"'" + "".join(map(chr, (0x300D, 0x300F, 0x3011, 0x300B, 0x3009, 0xFF09, 0xFF3D))))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _is_vs(ch: str) -> bool:
|
|
65
|
+
o = ord(ch)
|
|
66
|
+
return 0xFE00 <= o <= 0xFE0F or 0xE0100 <= o <= 0xE01EF
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _check(version: str) -> None:
|
|
70
|
+
if version not in SUPPORTED:
|
|
71
|
+
raise ValueError(f"unsupported canonical form {version!r}")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def canonicalize(text: str, version: str = CANON_ID) -> str:
|
|
75
|
+
return canonicalize_with_map(text, version)[0]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def segment(ctext: str, version: str = CANON_ID) -> list[tuple[int, int]]:
|
|
79
|
+
"""Content-defined blocks of canonical text: sentence-like spans, split at line breaks and after . ! ?
|
|
80
|
+
followed by whitespace (or at the end); v4 also after the full stops of scripts written without spaces,
|
|
81
|
+
with closing quotes and brackets kept in the sentence. Surrounding whitespace excluded. Deterministic."""
|
|
82
|
+
_check(version)
|
|
83
|
+
spans: list[tuple[int, int]] = []
|
|
84
|
+
start, n = 0, len(ctext)
|
|
85
|
+
v4 = version in (CANON_V4, CANON_V5) # seg2
|
|
86
|
+
|
|
87
|
+
def emit(a: int, b: int) -> None:
|
|
88
|
+
piece = ctext[a:b]
|
|
89
|
+
if piece.strip():
|
|
90
|
+
s = a + (len(piece) - len(piece.lstrip()))
|
|
91
|
+
e = b - (len(piece) - len(piece.rstrip()))
|
|
92
|
+
spans.append((s, e))
|
|
93
|
+
|
|
94
|
+
i = 0
|
|
95
|
+
while i < n:
|
|
96
|
+
ch = ctext[i]
|
|
97
|
+
if ch == "\n":
|
|
98
|
+
emit(start, i)
|
|
99
|
+
start = i + 1
|
|
100
|
+
elif not v4:
|
|
101
|
+
if ch in ".!?" and (i + 1 == n or ctext[i + 1].isspace()):
|
|
102
|
+
emit(start, i + 1)
|
|
103
|
+
start = i + 1
|
|
104
|
+
elif ch in ".!?" or ch in _SCRIPT_STOPS:
|
|
105
|
+
j = i + 1
|
|
106
|
+
while j < n and ctext[j] in _CLOSERS:
|
|
107
|
+
j += 1
|
|
108
|
+
if ch in _SCRIPT_STOPS or j == n or ctext[j].isspace():
|
|
109
|
+
emit(start, j)
|
|
110
|
+
start = j
|
|
111
|
+
i = j - 1
|
|
112
|
+
i += 1
|
|
113
|
+
emit(start, n)
|
|
114
|
+
return spans
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def block_digest(piece: str) -> str:
|
|
118
|
+
import hashlib
|
|
119
|
+
return hashlib.sha256(piece.encode()).hexdigest()[:16]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def canonicalize_with_map(text: str, version: str = CANON_ID) -> tuple[str, dict[int, int]]:
|
|
123
|
+
"""Return (canonical_text, index_map).
|
|
124
|
+
|
|
125
|
+
``index_map`` maps indices of the source to indices of the canonical text - for v3 and v4 indices of
|
|
126
|
+
``unicodedata.normalize("NFC", text)``, for v5 indices of ``text`` itself; characters removed by
|
|
127
|
+
canonicalization are absent from the map (v4, v5: a character folded into several, an ellipsis or an em dash,
|
|
128
|
+
maps to the first of them; v5: characters NFC composes map to the composed character).
|
|
129
|
+
"""
|
|
130
|
+
_check(version)
|
|
131
|
+
if version == CANON_V5:
|
|
132
|
+
return _canonicalize_v5(text)
|
|
133
|
+
v4 = version == CANON_V4
|
|
134
|
+
src = unicodedata.normalize("NFC", text)
|
|
135
|
+
kept: list[tuple[str, int]] = []
|
|
136
|
+
k, n = 0, len(src)
|
|
137
|
+
while k < n: # steps 2-3, 5 and 6: deletions, no-break spaces -> space, typographic folding
|
|
138
|
+
ch = src[k]
|
|
139
|
+
if ch == "": # a U+FEFF that opens a selector run is a text-manifest wrapper: drop both
|
|
140
|
+
e = k + 1
|
|
141
|
+
while e < n and _is_vs(src[e]):
|
|
142
|
+
e += 1
|
|
143
|
+
k = e if e - (k + 1) >= _VS_RUN else k + 1
|
|
144
|
+
continue
|
|
145
|
+
if ch not in _DELETE:
|
|
146
|
+
if v4 and ch in _FOLD: # a folded character maps to the first character it becomes
|
|
147
|
+
rep = _FOLD[ch]
|
|
148
|
+
kept.extend([(rep[0], k)] + [(c, -1) for c in rep[1:]])
|
|
149
|
+
else:
|
|
150
|
+
kept.append((" " if ch in _SPACE else ch, k))
|
|
151
|
+
k += 1
|
|
152
|
+
tmp: list[str] = []
|
|
153
|
+
tmap: list[int] = []
|
|
154
|
+
k, n = 0, len(kept)
|
|
155
|
+
while k < n: # step 4, after the deletions so that runs split by a deleted character are seen whole
|
|
156
|
+
if _is_vs(kept[k][0]):
|
|
157
|
+
e = k
|
|
158
|
+
while e < n and _is_vs(kept[e][0]):
|
|
159
|
+
e += 1
|
|
160
|
+
if e - k >= _VS_RUN:
|
|
161
|
+
k = e
|
|
162
|
+
continue
|
|
163
|
+
tmp.append(kept[k][0])
|
|
164
|
+
tmap.append(kept[k][1])
|
|
165
|
+
k += 1
|
|
166
|
+
res: list[str] = []
|
|
167
|
+
rmap: list[int] = []
|
|
168
|
+
k, length = 0, len(tmp)
|
|
169
|
+
while k < length: # drop spaces/tabs that end a line or the text
|
|
170
|
+
if tmp[k] in _TRAILING:
|
|
171
|
+
e = k
|
|
172
|
+
while e < length and tmp[e] in _TRAILING:
|
|
173
|
+
e += 1
|
|
174
|
+
if e == length or tmp[e] == "\n":
|
|
175
|
+
k = e
|
|
176
|
+
continue
|
|
177
|
+
res.append(tmp[k])
|
|
178
|
+
rmap.append(tmap[k])
|
|
179
|
+
k += 1
|
|
180
|
+
joined = "".join(res)
|
|
181
|
+
lead = len(joined) - len(joined.lstrip())
|
|
182
|
+
canon = joined.strip()
|
|
183
|
+
index_map = {src_i: ci - lead for ci, src_i in enumerate(rmap)
|
|
184
|
+
if src_i >= 0 and lead <= ci < lead + len(canon)}
|
|
185
|
+
return canon, index_map
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _deleted(text: str) -> list[tuple[str, int]]:
|
|
189
|
+
"""The deletions (CR, BOM, zero-widths, soft hyphen; a U+FEFF opening a run of selectors, with the run; runs of
|
|
190
|
+
eight or more variation selectors), as (character, source index) of what is kept."""
|
|
191
|
+
kept: list[tuple[str, int]] = []
|
|
192
|
+
k, n = 0, len(text)
|
|
193
|
+
while k < n:
|
|
194
|
+
ch = text[k]
|
|
195
|
+
if ch == "\ufeff":
|
|
196
|
+
e = k + 1
|
|
197
|
+
while e < n and _is_vs(text[e]):
|
|
198
|
+
e += 1
|
|
199
|
+
k = e if e - (k + 1) >= _VS_RUN else k + 1
|
|
200
|
+
continue
|
|
201
|
+
if ch not in _DELETE:
|
|
202
|
+
kept.append((ch, k))
|
|
203
|
+
k += 1
|
|
204
|
+
out: list[tuple[str, int]] = []
|
|
205
|
+
k, n = 0, len(kept)
|
|
206
|
+
while k < n: # runs split by a deleted character are seen whole
|
|
207
|
+
if _is_vs(kept[k][0]):
|
|
208
|
+
e = k
|
|
209
|
+
while e < n and _is_vs(kept[e][0]):
|
|
210
|
+
e += 1
|
|
211
|
+
if e - k >= _VS_RUN:
|
|
212
|
+
k = e
|
|
213
|
+
continue
|
|
214
|
+
out.append(kept[k])
|
|
215
|
+
k += 1
|
|
216
|
+
return out
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
_SECONDS: frozenset | None = None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _composing_seconds() -> frozenset:
|
|
223
|
+
"""Every character that canonical composition can join to what precedes it, starters included (Hangul vowels
|
|
224
|
+
and trailing consonants, Indic and other vowel signs with combining class 0): built once, when first needed."""
|
|
225
|
+
global _SECONDS
|
|
226
|
+
if _SECONDS is None:
|
|
227
|
+
seconds = set(range(0x1161, 0x1176)) | set(range(0x11A8, 0x11C3))
|
|
228
|
+
for cp in range(0x110000):
|
|
229
|
+
d = unicodedata.decomposition(chr(cp))
|
|
230
|
+
if d and not d.startswith("<"):
|
|
231
|
+
parts = d.split()
|
|
232
|
+
if len(parts) == 2:
|
|
233
|
+
seconds.add(int(parts[1], 16))
|
|
234
|
+
_SECONDS = frozenset(seconds)
|
|
235
|
+
return _SECONDS
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def nfc14(s: str) -> str:
|
|
239
|
+
"""NFC as Unicode 14.0 defines it, under any later Unicode version of the library."""
|
|
240
|
+
from ._unicode14 import assigned
|
|
241
|
+
out, start = [], 0
|
|
242
|
+
for i, ch in enumerate(s):
|
|
243
|
+
if not assigned(ord(ch)):
|
|
244
|
+
out.append(unicodedata.normalize("NFC", s[start:i]))
|
|
245
|
+
out.append(ch)
|
|
246
|
+
start = i + 1
|
|
247
|
+
out.append(unicodedata.normalize("NFC", s[start:]))
|
|
248
|
+
return "".join(out)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _nfc_with_map(chars: list[tuple[str, int]]) -> list[tuple[str, int]]:
|
|
252
|
+
"""NFC (Unicode 14.0) of the characters, each result character carrying the source index of the first character
|
|
253
|
+
of its normalization segment (a segment starts at a starter that nothing before it can compose or reorder with;
|
|
254
|
+
a code point not assigned in 14.0 is always one)."""
|
|
255
|
+
from ._unicode14 import assigned
|
|
256
|
+
s = "".join(c for c, _ in chars)
|
|
257
|
+
target = nfc14(s)
|
|
258
|
+
if target == s:
|
|
259
|
+
return chars
|
|
260
|
+
seconds = _composing_seconds()
|
|
261
|
+
|
|
262
|
+
def boundary(ch: str) -> bool:
|
|
263
|
+
if not assigned(ord(ch)):
|
|
264
|
+
return True
|
|
265
|
+
if unicodedata.combining(ch) or ord(ch) in seconds:
|
|
266
|
+
return False
|
|
267
|
+
first = unicodedata.normalize("NFD", ch)[0]
|
|
268
|
+
return unicodedata.combining(first) == 0
|
|
269
|
+
|
|
270
|
+
out: list[tuple[str, int]] = []
|
|
271
|
+
a, n = 0, len(chars)
|
|
272
|
+
while a < n:
|
|
273
|
+
b = a + 1
|
|
274
|
+
while b < n and not boundary(chars[b][0]):
|
|
275
|
+
b += 1
|
|
276
|
+
seg = "".join(c for c, _ in chars[a:b])
|
|
277
|
+
norm = nfc14(seg)
|
|
278
|
+
if norm == seg:
|
|
279
|
+
out.extend(chars[a:b])
|
|
280
|
+
else:
|
|
281
|
+
out.extend((c, chars[a][1]) for c in norm)
|
|
282
|
+
a = b
|
|
283
|
+
if "".join(c for c, _ in out) != target:
|
|
284
|
+
raise AssertionError("normalization segments do not compose to the NFC of the whole")
|
|
285
|
+
return out
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _canonicalize_v5(text: str) -> tuple[str, dict[int, int]]:
|
|
289
|
+
chars = _nfc_with_map(_deleted(text))
|
|
290
|
+
tmp: list[str] = []
|
|
291
|
+
tmap: list[int] = []
|
|
292
|
+
for ch, i in chars: # no-break spaces -> space, typographic folding
|
|
293
|
+
if ch in _FOLD:
|
|
294
|
+
rep = _FOLD[ch]
|
|
295
|
+
tmp.extend(rep)
|
|
296
|
+
tmap.extend([i] + [-1] * (len(rep) - 1))
|
|
297
|
+
else:
|
|
298
|
+
tmp.append(" " if ch in _SPACE else ch)
|
|
299
|
+
tmap.append(i)
|
|
300
|
+
res: list[str] = []
|
|
301
|
+
rmap: list[int] = []
|
|
302
|
+
k, length = 0, len(tmp)
|
|
303
|
+
while k < length: # drop spaces/tabs that end a line or the text
|
|
304
|
+
if tmp[k] in _TRAILING:
|
|
305
|
+
e = k
|
|
306
|
+
while e < length and tmp[e] in _TRAILING:
|
|
307
|
+
e += 1
|
|
308
|
+
if e == length or tmp[e] == "\n":
|
|
309
|
+
k = e
|
|
310
|
+
continue
|
|
311
|
+
res.append(tmp[k])
|
|
312
|
+
rmap.append(tmap[k])
|
|
313
|
+
k += 1
|
|
314
|
+
joined = "".join(res)
|
|
315
|
+
lead = len(joined) - len(joined.lstrip())
|
|
316
|
+
canon = joined.strip()
|
|
317
|
+
index_map: dict[int, int] = {}
|
|
318
|
+
for ci, src_i in enumerate(rmap):
|
|
319
|
+
if src_i >= 0 and lead <= ci < lead + len(canon):
|
|
320
|
+
index_map.setdefault(src_i, ci - lead) # a composed segment maps to its first character
|
|
321
|
+
return canon, index_map
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Canonical generation: sealed text is written, token by token, exactly as the verifier will re-read it.
|
|
2
|
+
|
|
3
|
+
The canonical form (canon.py, v4) folds typographic characters, deletes invisible ones and strips whitespace at the
|
|
4
|
+
ends of lines and of the text, so that a text pasted through a word processor still verifies. A verifier or a
|
|
5
|
+
detector tokenizes the canonical text; where that differs from the tokens the seal enforced, the checkpoints near
|
|
6
|
+
the difference are read in another context. Canonical generation keeps the two identical:
|
|
7
|
+
|
|
8
|
+
- no character the canonical form changes or deletes (curly quotes, apostrophe letters, typographic dashes, the
|
|
9
|
+
ellipsis character, guillemets, no-break and zero-width spaces, BOM, soft hyphen, word joiner, carriage return);
|
|
10
|
+
- valid UTF-8 throughout. Tokens are byte strings and a character may be split across tokens, so the guard tracks
|
|
11
|
+
the unfinished character at the end of the text: inside it only continuation bytes follow, and the text does not
|
|
12
|
+
end there. A token never stops inside a forbidden character past its lead byte; and where the verifier's first
|
|
13
|
+
choice is a banned token, no token may stop inside any character at that step, or a byte-level tokenizer would
|
|
14
|
+
complete the half-written quote into a stray neighbour (a dagger, a star);
|
|
15
|
+
- no space or tab before a line break (inside a token, or a line break right after a token that ends in one);
|
|
16
|
+
- the first token taken as the verifier reads it once leading whitespace is stripped (``lead``);
|
|
17
|
+
- no join the verifier would re-tokenize differently (``joinable``); a HIGH checkpoint never takes a byte prefix
|
|
18
|
+
of the first choice (``high_exclusion``), which the model would complete into a token read back as one.
|
|
19
|
+
|
|
20
|
+
Where the verifier's first choice is a token canonical generation may not emit, a LOW checkpoint is left free; a
|
|
21
|
+
detector recognizes these positions from the text and the model alone (``unsatisfiable``) and skips them, so its
|
|
22
|
+
exact null distribution is kept.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import codecs
|
|
27
|
+
import re
|
|
28
|
+
|
|
29
|
+
import torch
|
|
30
|
+
|
|
31
|
+
# every character canonical form v4 changes or deletes, written as code points (the source stays ASCII);
|
|
32
|
+
# long variation-selector runs are left out (a model does not write them) and single selectors are content
|
|
33
|
+
FORBIDDEN_CODEPOINTS = (0x000D, 0xFEFF, 0x200B, 0x2060, 0x00AD, 0x00A0, 0x2007, 0x202F,
|
|
34
|
+
0x201C, 0x201D, 0x201E, 0x201F, 0x00AB, 0x00BB, 0x2018, 0x2019, 0x201A, 0x201B, 0x02BC,
|
|
35
|
+
0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015, 0x2212, 0x2026)
|
|
36
|
+
# and the replacement character, never content: a model trained on scraped text writes it for a banned apostrophe
|
|
37
|
+
# ("don" + U+FFFD + "t" is how mangled web pages spell it)
|
|
38
|
+
FORBIDDEN_CODEPOINTS += (0xFFFD,)
|
|
39
|
+
FORBIDDEN = tuple(chr(c).encode("utf-8") for c in FORBIDDEN_CODEPOINTS)
|
|
40
|
+
PREFIXES = sorted({f[:j] for f in FORBIDDEN for j in range(1, len(f))}) # tracked exactly
|
|
41
|
+
PREFIX_SET = frozenset(PREFIXES)
|
|
42
|
+
DANGEROUS = frozenset(f[:j] for f in FORBIDDEN for j in range(2, len(f))) # a token never stops here
|
|
43
|
+
# stand-ins for "inside some other character, n bytes to go" (none is a prefix of a forbidden character)
|
|
44
|
+
INSIDE = {1: bytes([0xE3, 0x80]), 2: bytes([0xE3]), 3: bytes([0xF0])}
|
|
45
|
+
LINE_END_WS = (b" \n", b"\t\n", b" \r", b"\t\r")
|
|
46
|
+
# continuations tried when a join is checked: end of text, a letter, a space and a letter, a line break, a full
|
|
47
|
+
# stop, a digit. Some tokenizers split a whitespace run by what follows it (GPT-2 reads "\n\nThe" as three tokens
|
|
48
|
+
# but "\n\n The" as two), so a join is kept when some continuation leaves it intact.
|
|
49
|
+
PROBES = (b"", b"x", b" x", b"\n", b".", b"1")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _bytes_to_unicode() -> dict[int, str]:
|
|
53
|
+
"""The byte <-> printable-character table of byte-level BPE (GPT-2, Llama 3, Qwen)."""
|
|
54
|
+
bs = list(range(ord("!"), ord("~") + 1)) + list(range(0xA1, 0xAD)) + list(range(0xAE, 0x100))
|
|
55
|
+
cs, n = bs[:], 0
|
|
56
|
+
for b in range(256):
|
|
57
|
+
if b not in bs:
|
|
58
|
+
bs.append(b)
|
|
59
|
+
cs.append(256 + n)
|
|
60
|
+
n += 1
|
|
61
|
+
return dict(zip(bs, (chr(c) for c in cs)))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def token_bytes(tok) -> list[bytes | None]:
|
|
65
|
+
"""Raw bytes of every token id; None for special tokens (they never enter the text)."""
|
|
66
|
+
special = set(tok.all_special_ids) | {i for i, t in tok.added_tokens_decoder.items() if getattr(t, "special", False)}
|
|
67
|
+
vocab = tok.get_vocab()
|
|
68
|
+
n = max(vocab.values()) + 1
|
|
69
|
+
pieces = [None] * n
|
|
70
|
+
for piece, i in vocab.items():
|
|
71
|
+
pieces[i] = piece
|
|
72
|
+
space_marker, sp_marker = chr(0x120), chr(0x2581) # GPT-2 and SentencePiece space marks
|
|
73
|
+
byte_level = any(space_marker in p for p in pieces[:5000] if p)
|
|
74
|
+
dec = {c: b for b, c in _bytes_to_unicode().items()}
|
|
75
|
+
fallback = re.compile(r"<0x([0-9A-Fa-f]{2})>")
|
|
76
|
+
out: list[bytes | None] = []
|
|
77
|
+
for i, p in enumerate(pieces):
|
|
78
|
+
if p is None or i in special:
|
|
79
|
+
out.append(None)
|
|
80
|
+
elif byte_level:
|
|
81
|
+
out.append(bytes(dec[c] for c in p) if all(c in dec for c in p) else p.encode("utf-8"))
|
|
82
|
+
else:
|
|
83
|
+
m = fallback.fullmatch(p)
|
|
84
|
+
out.append(bytes([int(m.group(1), 16)]) if m else p.replace(sp_marker, " ").encode("utf-8"))
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _tail(b: bytes) -> bytes:
|
|
89
|
+
"""The incomplete UTF-8 character at the end of ``b`` (empty if it ends on a character boundary)."""
|
|
90
|
+
for k in range(1, min(4, len(b)) + 1):
|
|
91
|
+
c = b[-k]
|
|
92
|
+
if c & 0xC0 != 0x80: # a lead byte (or ASCII): the last character starts here
|
|
93
|
+
need = 1 if c < 0x80 else 2 if c >> 5 == 0x6 else 3 if c >> 4 == 0xE else 4 if c >> 3 == 0x1E else 1
|
|
94
|
+
return b[-k:] if k < need else b""
|
|
95
|
+
return b""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _need(tail: bytes) -> int:
|
|
99
|
+
"""Continuation bytes still missing from an unfinished character."""
|
|
100
|
+
c = tail[0]
|
|
101
|
+
total = 2 if c >> 5 == 0x6 else 3 if c >> 4 == 0xE else 4 if c >> 3 == 0x1E else 1
|
|
102
|
+
return max(0, total - len(tail))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _utf8_ok(b: bytes) -> bool:
|
|
106
|
+
"""No invalid sequence in ``b`` (an unfinished character at its end is allowed)."""
|
|
107
|
+
try:
|
|
108
|
+
codecs.getincrementaldecoder("utf-8")().decode(b, final=False)
|
|
109
|
+
return True
|
|
110
|
+
except UnicodeDecodeError:
|
|
111
|
+
return False
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class CanonGuard:
|
|
115
|
+
"""Per-step ban masks and checks that keep generated tokens identical to the verifier's reading."""
|
|
116
|
+
|
|
117
|
+
WINDOW = 8 # tokens of context re-tokenized when a join is checked
|
|
118
|
+
COMPARE = 4 # of which the last COMPARE must come back unchanged (the window's left edge may not)
|
|
119
|
+
|
|
120
|
+
def __init__(self, tok, vocab_size: int, device="cpu"):
|
|
121
|
+
self.tok = tok
|
|
122
|
+
self.bytes = token_bytes(tok)
|
|
123
|
+
# states: at a character boundary, inside one of the forbidden characters' prefixes (tracked exactly), or
|
|
124
|
+
# inside some other character with n bytes to go
|
|
125
|
+
self.states: list[bytes] = [b""] + PREFIXES + [INSIDE[n] for n in (1, 2, 3)]
|
|
126
|
+
self.index = {s: i for i, s in enumerate(self.states)}
|
|
127
|
+
n_tok = min(vocab_size, len(self.bytes))
|
|
128
|
+
masks = torch.zeros((len(self.states), vocab_size), dtype=torch.bool)
|
|
129
|
+
masks[:, n_tok:] = True # embedding rows without a token: never emitted
|
|
130
|
+
partial = torch.zeros(vocab_size, dtype=torch.bool)
|
|
131
|
+
nl = torch.zeros(vocab_size, dtype=torch.bool)
|
|
132
|
+
for i in range(n_tok):
|
|
133
|
+
tb = self.bytes[i]
|
|
134
|
+
if tb is None:
|
|
135
|
+
masks[1:, i] = True # a special token (the end of the text) never inside a character
|
|
136
|
+
continue
|
|
137
|
+
if not tb or any(w in tb for w in LINE_END_WS):
|
|
138
|
+
masks[:, i] = True # no text, or a space or tab before a line break
|
|
139
|
+
continue
|
|
140
|
+
nl[i] = tb[:1] in (b"\n", b"\r")
|
|
141
|
+
partial[i] = bool(_tail(tb))
|
|
142
|
+
plain = max(tb) < 0x80 and b"\r" not in tb
|
|
143
|
+
for si, s in enumerate(self.states):
|
|
144
|
+
if not s and plain:
|
|
145
|
+
continue # plain ASCII at a boundary: always fine
|
|
146
|
+
joined = s + tb
|
|
147
|
+
if not _utf8_ok(joined) or any(f in joined for f in FORBIDDEN) or _tail(joined) in DANGEROUS:
|
|
148
|
+
masks[si, i] = True
|
|
149
|
+
self.masks, self.masks_cpu = masks.to(device), masks
|
|
150
|
+
self.partial, self.partial_cpu = partial.to(device), partial # tokens that stop inside a character
|
|
151
|
+
self.nl_start, self.nl_start_cpu = nl.to(device), nl # banned after a trailing space or tab
|
|
152
|
+
self.banned_any = int(masks[0, :n_tok].sum())
|
|
153
|
+
self.lead = self._lead_map(tok, vocab_size) # CPU: verifier-view first token, -1 = not first
|
|
154
|
+
self.lead_ban = (self.lead < 0).to(device)
|
|
155
|
+
self.prefixes_of = self._prefix_table(n_tok)
|
|
156
|
+
self._cache: dict[tuple, bool] = {}
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------------------------- tables
|
|
159
|
+
def _lead_map(self, tok, vocab_size: int) -> torch.Tensor:
|
|
160
|
+
"""The token the verifier reads for a first token once leading whitespace is stripped; -1 where there is no
|
|
161
|
+
single such token (pure whitespace, a remainder of several tokens or not whole UTF-8)."""
|
|
162
|
+
lead = torch.arange(vocab_size, dtype=torch.long)
|
|
163
|
+
lead[len(self.bytes):] = -1 # embedding rows without a token
|
|
164
|
+
todo_ids: list[int] = []
|
|
165
|
+
todo_text: list[str] = []
|
|
166
|
+
for i, tb in enumerate(self.bytes[:vocab_size]):
|
|
167
|
+
if tb is None or not tb:
|
|
168
|
+
continue # special tokens do not enter the text
|
|
169
|
+
try:
|
|
170
|
+
s = tb.decode("utf-8")
|
|
171
|
+
except UnicodeDecodeError: # a piece of a character
|
|
172
|
+
if tb[:1].isspace() or tb[0] & 0xC0 == 0x80:
|
|
173
|
+
lead[i] = -1 # unreadable once stripped / no character starts here
|
|
174
|
+
continue
|
|
175
|
+
rem = s.lstrip()
|
|
176
|
+
if rem == s:
|
|
177
|
+
continue
|
|
178
|
+
if rem:
|
|
179
|
+
todo_ids.append(i)
|
|
180
|
+
todo_text.append(rem)
|
|
181
|
+
else:
|
|
182
|
+
lead[i] = -1 # pure whitespace: stripped away entirely
|
|
183
|
+
if todo_text:
|
|
184
|
+
for i, e in zip(todo_ids, tok(todo_text, add_special_tokens=False)["input_ids"]):
|
|
185
|
+
lead[i] = e[0] if len(e) == 1 else -1
|
|
186
|
+
return lead
|
|
187
|
+
|
|
188
|
+
def _prefix_table(self, n_tok: int) -> dict[int, list[int]]:
|
|
189
|
+
"""Token -> the tokens whose bytes are a proper prefix of its bytes. A HIGH checkpoint must not pick a prefix
|
|
190
|
+
of the first choice (" question" for " questionnaire", "." for a full stop with line breaks): the model
|
|
191
|
+
would complete the word and the verifier re-read the merged token."""
|
|
192
|
+
by_bytes: dict[bytes, list[int]] = {}
|
|
193
|
+
for i, tb in enumerate(self.bytes[:n_tok]):
|
|
194
|
+
if tb:
|
|
195
|
+
by_bytes.setdefault(tb, []).append(i)
|
|
196
|
+
table: dict[int, list[int]] = {}
|
|
197
|
+
for i, tb in enumerate(self.bytes[:n_tok]):
|
|
198
|
+
if tb and len(tb) > 1:
|
|
199
|
+
pre = [t for m in range(1, len(tb)) for t in by_bytes.get(tb[:m], ())]
|
|
200
|
+
if pre:
|
|
201
|
+
table[i] = pre
|
|
202
|
+
# and the tokens that only add whitespace to it ("\n\n" for "\n", ".\n\n" for "."): where a whitespace run
|
|
203
|
+
# ends is decided by what follows it, so a HIGH checkpoint must not move that boundary either
|
|
204
|
+
for i, tb in enumerate(self.bytes[:n_tok]):
|
|
205
|
+
if not tb or not tb[-1:].isspace():
|
|
206
|
+
continue
|
|
207
|
+
core = len(tb.rstrip()) # tb = tb[:core] + a whitespace run
|
|
208
|
+
for m in range(max(1, core), len(tb)):
|
|
209
|
+
for t in by_bytes.get(tb[:m], ()):
|
|
210
|
+
table.setdefault(t, []).append(i)
|
|
211
|
+
return table
|
|
212
|
+
|
|
213
|
+
# ------------------------------------------------------------------------------------------------- steps
|
|
214
|
+
def mask(self, state: int) -> torch.Tensor:
|
|
215
|
+
return self.masks[state]
|
|
216
|
+
|
|
217
|
+
def state_after(self, state: int, token_id: int) -> int:
|
|
218
|
+
tb = self.bytes[token_id] if token_id < len(self.bytes) else None
|
|
219
|
+
if not tb:
|
|
220
|
+
return 0
|
|
221
|
+
tail = _tail(self.states[state] + tb)
|
|
222
|
+
if not tail:
|
|
223
|
+
return 0
|
|
224
|
+
if tail in PREFIX_SET:
|
|
225
|
+
return self.index[tail]
|
|
226
|
+
n = _need(tail)
|
|
227
|
+
return self.index[INSIDE[n]] if n in INSIDE else 0
|
|
228
|
+
|
|
229
|
+
def ends_in_space(self, token_id: int) -> bool:
|
|
230
|
+
"""Does the token end in a space or tab (so that a line break may not follow)?"""
|
|
231
|
+
tb = self.bytes[token_id] if token_id < len(self.bytes) else None
|
|
232
|
+
return bool(tb) and tb[-1:] in (b" ", b"\t")
|
|
233
|
+
|
|
234
|
+
def high_exclusion(self, first: list[int]) -> list[int]:
|
|
235
|
+
"""Tokens a HIGH checkpoint must not choose, given the first choice(s) it has to avoid."""
|
|
236
|
+
return [t for f in first for t in self.prefixes_of.get(int(f), ())]
|
|
237
|
+
|
|
238
|
+
def joinable(self, view: list[int], t: int) -> bool:
|
|
239
|
+
"""Would the verifier re-read the text of ``view`` + [t] as these very tokens, for some continuation of the
|
|
240
|
+
text? The last tokens are re-tokenized with each probe continuation; the last COMPARE tokens of the window
|
|
241
|
+
and t must come back unchanged, followed by exactly the probe. A join inside an unfinished character is
|
|
242
|
+
judged when the character is complete. A break already in the window (a join kept when nothing else
|
|
243
|
+
joined) is not held against the tokens after it."""
|
|
244
|
+
tb = self.bytes[t] if t < len(self.bytes) else None
|
|
245
|
+
if tb is None or not view:
|
|
246
|
+
return True
|
|
247
|
+
start = len(view) <= self.WINDOW # the window begins where the text begins
|
|
248
|
+
if len(view) > 1 and not self._intact(tuple(view[-self.WINDOW:]), start):
|
|
249
|
+
return True
|
|
250
|
+
return self._intact(tuple(view[-self.WINDOW:]) + (t,), start)
|
|
251
|
+
|
|
252
|
+
def _intact(self, win: tuple, start: bool) -> bool:
|
|
253
|
+
hit = self._cache.get((win, start))
|
|
254
|
+
if hit is None:
|
|
255
|
+
hit = self._check(list(win), start)
|
|
256
|
+
if len(self._cache) > 50000:
|
|
257
|
+
self._cache.clear()
|
|
258
|
+
self._cache[(win, start)] = hit
|
|
259
|
+
return hit
|
|
260
|
+
|
|
261
|
+
def _check(self, seq: list[int], start: bool = False) -> bool:
|
|
262
|
+
"""Do the last COMPARE tokens of ``seq`` come back unchanged when its text is re-tokenized with some probe?
|
|
263
|
+
At the start of a text a leading space is the tokenizer's own prefix (SentencePiece), never text: the
|
|
264
|
+
canonical text does not begin with whitespace."""
|
|
265
|
+
win, t = seq[:-1], seq[-1]
|
|
266
|
+
tb = self.bytes[t] if t < len(self.bytes) else None
|
|
267
|
+
if tb is None:
|
|
268
|
+
return True
|
|
269
|
+
parts = [self.bytes[i] if i < len(self.bytes) else None for i in win]
|
|
270
|
+
if any(p is None for p in parts):
|
|
271
|
+
return True
|
|
272
|
+
body = b"".join(parts) + tb
|
|
273
|
+
if start and body[:1] == b" ":
|
|
274
|
+
body = body[1:]
|
|
275
|
+
cut = 0
|
|
276
|
+
while cut < len(body) and body[cut] & 0xC0 == 0x80: # a character cut by the window's left edge
|
|
277
|
+
cut += 1
|
|
278
|
+
body = body[cut:]
|
|
279
|
+
if _tail(body):
|
|
280
|
+
return True
|
|
281
|
+
want = win[-self.COMPARE:] + [t]
|
|
282
|
+
try:
|
|
283
|
+
texts = [(body + p).decode("utf-8") for p in PROBES]
|
|
284
|
+
except UnicodeDecodeError:
|
|
285
|
+
return True
|
|
286
|
+
for p, enc in zip(PROBES, self.tok(texts, add_special_tokens=False)["input_ids"]):
|
|
287
|
+
for q in range(3):
|
|
288
|
+
end = len(enc) - q
|
|
289
|
+
if end < len(want) or enc[end - len(want):end] != want:
|
|
290
|
+
continue
|
|
291
|
+
if b"".join(self.bytes[i] or b"" for i in enc[end:]) == p:
|
|
292
|
+
return True
|
|
293
|
+
return False
|
|
294
|
+
|
|
295
|
+
def first_joinable(self, view: list[int], scores: torch.Tensor, avoid, stops, limit: int = 64) -> int | None:
|
|
296
|
+
"""The most probable token under ``scores`` (among the ``limit`` best, none in ``avoid`` or ``stops``) that
|
|
297
|
+
joins ``view`` as the verifier will read it; None if there is none."""
|
|
298
|
+
for t in torch.argsort(scores, descending=True)[:limit].tolist():
|
|
299
|
+
if scores[t] == float("-inf"):
|
|
300
|
+
break
|
|
301
|
+
if t not in avoid and t not in stops and self.joinable(view, t):
|
|
302
|
+
return t
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
def unsatisfiable(self, ids: list[int], positions: list[int], first: list[int], stops) -> list[bool]:
|
|
306
|
+
"""At each position (a token index of ``ids``): could canonical generation not have emitted the verifier's
|
|
307
|
+
first choice ``first`` there? (A stop token, a token the guard bans after the text so far, or a join the
|
|
308
|
+
verifier would re-read differently.) The sealer then left a LOW checkpoint free. Read from the text and the
|
|
309
|
+
model, never from the key, so a detector that skips these positions keeps the exact null."""
|
|
310
|
+
states, spaces, s, sp = [], [], 0, False
|
|
311
|
+
for t in ids[: max(positions, default=-1) + 1]:
|
|
312
|
+
states.append(s)
|
|
313
|
+
spaces.append(sp)
|
|
314
|
+
s, sp = self.state_after(s, t), self.ends_in_space(t)
|
|
315
|
+
out = []
|
|
316
|
+
for p, f in zip(positions, first):
|
|
317
|
+
if f in stops or f >= self.masks_cpu.shape[1]:
|
|
318
|
+
out.append(True)
|
|
319
|
+
elif bool(self.masks_cpu[states[p], f]) or (spaces[p] and bool(self.nl_start_cpu[f])):
|
|
320
|
+
out.append(True)
|
|
321
|
+
elif p == 0:
|
|
322
|
+
out.append(bool(self.lead[f] < 0))
|
|
323
|
+
else:
|
|
324
|
+
out.append(f != ids[p] and not self.joinable(ids[:p], f))
|
|
325
|
+
return out
|