entropy-protocol 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. entropy_protocol/__init__.py +11 -0
  2. entropy_protocol/_unicode14.py +120 -0
  3. entropy_protocol/attest.py +62 -0
  4. entropy_protocol/attested/__init__.py +6 -0
  5. entropy_protocol/attested/amd.py +221 -0
  6. entropy_protocol/attested/appraise.py +325 -0
  7. entropy_protocol/attested/attester.py +58 -0
  8. entropy_protocol/attested/authority.py +239 -0
  9. entropy_protocol/attested/azure_cgpu.py +186 -0
  10. entropy_protocol/attested/binding.py +71 -0
  11. entropy_protocol/attested/capture.py +109 -0
  12. entropy_protocol/attested/ear.py +94 -0
  13. entropy_protocol/attested/enroll.py +119 -0
  14. entropy_protocol/attested/evidence.py +106 -0
  15. entropy_protocol/attested/hcl.py +109 -0
  16. entropy_protocol/attested/nvidia.py +168 -0
  17. entropy_protocol/attested/ocsp.py +90 -0
  18. entropy_protocol/attested/pki.py +121 -0
  19. entropy_protocol/attested/rim.py +130 -0
  20. entropy_protocol/attested/service.py +242 -0
  21. entropy_protocol/attested/tpm.py +259 -0
  22. entropy_protocol/backend_hf.py +439 -0
  23. entropy_protocol/batch.py +537 -0
  24. entropy_protocol/calibrate.py +99 -0
  25. entropy_protocol/canon.py +321 -0
  26. entropy_protocol/canongen.py +325 -0
  27. entropy_protocol/cli.py +493 -0
  28. entropy_protocol/cose.py +260 -0
  29. entropy_protocol/crypto.py +140 -0
  30. entropy_protocol/delivery.py +31 -0
  31. entropy_protocol/detect.py +246 -0
  32. entropy_protocol/gateway.py +370 -0
  33. entropy_protocol/hpke.py +105 -0
  34. entropy_protocol/mcp.py +159 -0
  35. entropy_protocol/pack.py +217 -0
  36. entropy_protocol/plain.py +122 -0
  37. entropy_protocol/predicates.py +37 -0
  38. entropy_protocol/profiles.py +45 -0
  39. entropy_protocol/record.py +116 -0
  40. entropy_protocol/scrapi.py +203 -0
  41. entropy_protocol/seal.py +232 -0
  42. entropy_protocol/statements.py +148 -0
  43. entropy_protocol/stats.py +20 -0
  44. entropy_protocol/tlog.py +147 -0
  45. entropy_protocol/ts3161.py +205 -0
  46. entropy_protocol/typography.py +73 -0
  47. entropy_protocol/verify.py +232 -0
  48. entropy_protocol-0.1.0.dist-info/METADATA +255 -0
  49. entropy_protocol-0.1.0.dist-info/RECORD +54 -0
  50. entropy_protocol-0.1.0.dist-info/WHEEL +5 -0
  51. entropy_protocol-0.1.0.dist-info/entry_points.txt +2 -0
  52. entropy_protocol-0.1.0.dist-info/licenses/LICENSE +201 -0
  53. entropy_protocol-0.1.0.dist-info/licenses/NOTICE +4 -0
  54. entropy_protocol-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,321 @@
1
+ """Canonical text form shared by sealer and verifier.
2
+
3
+ A record is measured on the canonical form of the final text, so anything the canonical form
4
+ erases — carriage returns, byte-order marks, invisible format characters, an embedded C2PA text
5
+ manifest, no-break spaces, trailing spaces at line ends, blank lines around a paste, and (v4) the
6
+ typographic substitutions word processors make on paste — cannot break verification. The transform is
7
+ deterministic and idempotent:
8
+ 1. Unicode NFC
9
+ 2. delete every U+000D (CR) and U+FEFF (BOM): LF is the only line break, so CRLF, a naive
10
+ LF -> CRLF conversion of text that already holds CRLF (CR CR LF) and stray CRs all agree
11
+ 3. delete invisible format characters that carry no content: U+200B (zero-width space),
12
+ U+2060 (word joiner), U+00AD (soft hyphen). U+200C/U+200D (ZWNJ/ZWJ) are kept: scripts and
13
+ emoji sequences depend on them
14
+ 4. delete embedded payloads written as variation selectors (U+FE00-FE0F, U+E0100-E01EF): a U+FEFF
15
+ followed by 8 or more selectors (C2PA 2.4 A.8 text-manifest wrapper) is removed as a unit, then
16
+ any remaining run of 8 or more selectors; a single selector after a base character (e.g. an emoji
17
+ presentation selector) is content and is kept
18
+ 5. U+00A0, U+2007, U+202F (no-break spaces) -> U+0020
19
+ 6. (v4) typographic folding, the inverse of what word processors do on paste (and of Ukrainian
20
+ orthography): double quotes U+201C U+201D U+201E U+201F U+00AB U+00BB -> U+0022; apostrophes and single
21
+ quotes U+2018 U+2019 U+201A U+201B U+02BC -> U+0027; hyphens and en dash U+2010-U+2013, minus U+2212 ->
22
+ U+002D; em dash U+2014 and horizontal bar U+2015 -> "--" (Word turns " - " into an en dash and "--" into
23
+ an em dash); ellipsis U+2026 -> "..." (Word turns "..." into an ellipsis)
24
+ 7. delete spaces and tabs that end a line or the text; strip the text
25
+ Blocks (content layer): sentence-like spans, split at line breaks and after . ! ? followed by
26
+ whitespace; v4 also after the full stops of scripts written without spaces (Chinese/Japanese, Arabic,
27
+ Devanagari, Ethiopic, Armenian) and keeps closing quotes and brackets with their sentence.
28
+ Records name their version; a verifier reads every supported version.
29
+ """
30
+ from __future__ import annotations
31
+
32
+ import unicodedata
33
+
34
+ CANON_V3 = "ep1-canon/v3:nfc+del-cr-bom-zw-shy+del-vs-runs+nbsp+rstrip-lines+strip"
35
+ CANON_V4 = "ep1-canon/v4:nfc+del-cr-bom-zw-shy+del-vs-runs+nbsp+typo+rstrip-lines+strip;seg2"
36
+ # v5: the deletions before NFC. In v4 a deleted character could leave combining marks adjacent in a non-canonical
37
+ # order, or a base and a combining mark that NFC would compose: canonicalize(canonicalize(x)) != canonicalize(x), so a
38
+ # zero-width character inserted between such marks broke verification after transport (found by fuzzing, 27.10.2026:
39
+ # U+0E48 U+0301 ZWJ U+0E48 U+0301). With NFC after every deletion, and only starters replaced or removed afterwards,
40
+ # the form is idempotent by construction (tests/test_fuzz.py checks it).
41
+ # NFC is that of Unicode 14.0 (the oldest Python the package supports): the text is cut at every code point not
42
+ # assigned in 14.0 - to 14.0 a starter nothing composes or reorders across - and the pieces normalized; by Unicode's
43
+ # normalization stability policy every later version normalizes them the same way. Without the pin a combining mark
44
+ # assigned later reorders under a newer library only (fuzzing, 27.10.2026: U+1AE8 U+0E48 - Node 26 with Unicode 17
45
+ # swapped them, Python 3.12 with Unicode 15.0 did not).
46
+ CANON_V5 = "ep1-canon/v5:del-cr-bom-zw-shy+del-vs-runs+nfc-u14+nbsp+typo+rstrip-lines+strip;seg2"
47
+ CANON_ID = CANON_V5 # new records
48
+ SUPPORTED = (CANON_V3, CANON_V4, CANON_V5)
49
+ _TRAILING = " \t"
50
+ _DELETE = {"\r", "", "​", "⁠", "­"}
51
+ _SPACE = {" ", " ", " "}
52
+ _VS_RUN = 8
53
+ # v4 typographic folding (code points written as numbers: the source stays ASCII)
54
+ _FOLD = {**{chr(c): '"' for c in (0x201C, 0x201D, 0x201E, 0x201F, 0x00AB, 0x00BB)},
55
+ **{chr(c): "'" for c in (0x2018, 0x2019, 0x201A, 0x201B, 0x02BC)},
56
+ **{chr(c): "-" for c in (0x2010, 0x2011, 0x2012, 0x2013, 0x2212)},
57
+ **{chr(c): "--" for c in (0x2014, 0x2015)},
58
+ chr(0x2026): "..."}
59
+ # v4 segmentation: full stops of scripts written without spaces, closing quotes and brackets
60
+ _SCRIPT_STOPS = frozenset(map(chr, (0x3002, 0xFF01, 0xFF1F, 0xFF0E, 0x061F, 0x06D4, 0x0964, 0x0965, 0x1362, 0x0589)))
61
+ _CLOSERS = frozenset(")]}\"'" + "".join(map(chr, (0x300D, 0x300F, 0x3011, 0x300B, 0x3009, 0xFF09, 0xFF3D))))
62
+
63
+
64
+ def _is_vs(ch: str) -> bool:
65
+ o = ord(ch)
66
+ return 0xFE00 <= o <= 0xFE0F or 0xE0100 <= o <= 0xE01EF
67
+
68
+
69
+ def _check(version: str) -> None:
70
+ if version not in SUPPORTED:
71
+ raise ValueError(f"unsupported canonical form {version!r}")
72
+
73
+
74
+ def canonicalize(text: str, version: str = CANON_ID) -> str:
75
+ return canonicalize_with_map(text, version)[0]
76
+
77
+
78
+ def segment(ctext: str, version: str = CANON_ID) -> list[tuple[int, int]]:
79
+ """Content-defined blocks of canonical text: sentence-like spans, split at line breaks and after . ! ?
80
+ followed by whitespace (or at the end); v4 also after the full stops of scripts written without spaces,
81
+ with closing quotes and brackets kept in the sentence. Surrounding whitespace excluded. Deterministic."""
82
+ _check(version)
83
+ spans: list[tuple[int, int]] = []
84
+ start, n = 0, len(ctext)
85
+ v4 = version in (CANON_V4, CANON_V5) # seg2
86
+
87
+ def emit(a: int, b: int) -> None:
88
+ piece = ctext[a:b]
89
+ if piece.strip():
90
+ s = a + (len(piece) - len(piece.lstrip()))
91
+ e = b - (len(piece) - len(piece.rstrip()))
92
+ spans.append((s, e))
93
+
94
+ i = 0
95
+ while i < n:
96
+ ch = ctext[i]
97
+ if ch == "\n":
98
+ emit(start, i)
99
+ start = i + 1
100
+ elif not v4:
101
+ if ch in ".!?" and (i + 1 == n or ctext[i + 1].isspace()):
102
+ emit(start, i + 1)
103
+ start = i + 1
104
+ elif ch in ".!?" or ch in _SCRIPT_STOPS:
105
+ j = i + 1
106
+ while j < n and ctext[j] in _CLOSERS:
107
+ j += 1
108
+ if ch in _SCRIPT_STOPS or j == n or ctext[j].isspace():
109
+ emit(start, j)
110
+ start = j
111
+ i = j - 1
112
+ i += 1
113
+ emit(start, n)
114
+ return spans
115
+
116
+
117
+ def block_digest(piece: str) -> str:
118
+ import hashlib
119
+ return hashlib.sha256(piece.encode()).hexdigest()[:16]
120
+
121
+
122
+ def canonicalize_with_map(text: str, version: str = CANON_ID) -> tuple[str, dict[int, int]]:
123
+ """Return (canonical_text, index_map).
124
+
125
+ ``index_map`` maps indices of the source to indices of the canonical text - for v3 and v4 indices of
126
+ ``unicodedata.normalize("NFC", text)``, for v5 indices of ``text`` itself; characters removed by
127
+ canonicalization are absent from the map (v4, v5: a character folded into several, an ellipsis or an em dash,
128
+ maps to the first of them; v5: characters NFC composes map to the composed character).
129
+ """
130
+ _check(version)
131
+ if version == CANON_V5:
132
+ return _canonicalize_v5(text)
133
+ v4 = version == CANON_V4
134
+ src = unicodedata.normalize("NFC", text)
135
+ kept: list[tuple[str, int]] = []
136
+ k, n = 0, len(src)
137
+ while k < n: # steps 2-3, 5 and 6: deletions, no-break spaces -> space, typographic folding
138
+ ch = src[k]
139
+ if ch == "": # a U+FEFF that opens a selector run is a text-manifest wrapper: drop both
140
+ e = k + 1
141
+ while e < n and _is_vs(src[e]):
142
+ e += 1
143
+ k = e if e - (k + 1) >= _VS_RUN else k + 1
144
+ continue
145
+ if ch not in _DELETE:
146
+ if v4 and ch in _FOLD: # a folded character maps to the first character it becomes
147
+ rep = _FOLD[ch]
148
+ kept.extend([(rep[0], k)] + [(c, -1) for c in rep[1:]])
149
+ else:
150
+ kept.append((" " if ch in _SPACE else ch, k))
151
+ k += 1
152
+ tmp: list[str] = []
153
+ tmap: list[int] = []
154
+ k, n = 0, len(kept)
155
+ while k < n: # step 4, after the deletions so that runs split by a deleted character are seen whole
156
+ if _is_vs(kept[k][0]):
157
+ e = k
158
+ while e < n and _is_vs(kept[e][0]):
159
+ e += 1
160
+ if e - k >= _VS_RUN:
161
+ k = e
162
+ continue
163
+ tmp.append(kept[k][0])
164
+ tmap.append(kept[k][1])
165
+ k += 1
166
+ res: list[str] = []
167
+ rmap: list[int] = []
168
+ k, length = 0, len(tmp)
169
+ while k < length: # drop spaces/tabs that end a line or the text
170
+ if tmp[k] in _TRAILING:
171
+ e = k
172
+ while e < length and tmp[e] in _TRAILING:
173
+ e += 1
174
+ if e == length or tmp[e] == "\n":
175
+ k = e
176
+ continue
177
+ res.append(tmp[k])
178
+ rmap.append(tmap[k])
179
+ k += 1
180
+ joined = "".join(res)
181
+ lead = len(joined) - len(joined.lstrip())
182
+ canon = joined.strip()
183
+ index_map = {src_i: ci - lead for ci, src_i in enumerate(rmap)
184
+ if src_i >= 0 and lead <= ci < lead + len(canon)}
185
+ return canon, index_map
186
+
187
+
188
+ def _deleted(text: str) -> list[tuple[str, int]]:
189
+ """The deletions (CR, BOM, zero-widths, soft hyphen; a U+FEFF opening a run of selectors, with the run; runs of
190
+ eight or more variation selectors), as (character, source index) of what is kept."""
191
+ kept: list[tuple[str, int]] = []
192
+ k, n = 0, len(text)
193
+ while k < n:
194
+ ch = text[k]
195
+ if ch == "\ufeff":
196
+ e = k + 1
197
+ while e < n and _is_vs(text[e]):
198
+ e += 1
199
+ k = e if e - (k + 1) >= _VS_RUN else k + 1
200
+ continue
201
+ if ch not in _DELETE:
202
+ kept.append((ch, k))
203
+ k += 1
204
+ out: list[tuple[str, int]] = []
205
+ k, n = 0, len(kept)
206
+ while k < n: # runs split by a deleted character are seen whole
207
+ if _is_vs(kept[k][0]):
208
+ e = k
209
+ while e < n and _is_vs(kept[e][0]):
210
+ e += 1
211
+ if e - k >= _VS_RUN:
212
+ k = e
213
+ continue
214
+ out.append(kept[k])
215
+ k += 1
216
+ return out
217
+
218
+
219
+ _SECONDS: frozenset | None = None
220
+
221
+
222
+ def _composing_seconds() -> frozenset:
223
+ """Every character that canonical composition can join to what precedes it, starters included (Hangul vowels
224
+ and trailing consonants, Indic and other vowel signs with combining class 0): built once, when first needed."""
225
+ global _SECONDS
226
+ if _SECONDS is None:
227
+ seconds = set(range(0x1161, 0x1176)) | set(range(0x11A8, 0x11C3))
228
+ for cp in range(0x110000):
229
+ d = unicodedata.decomposition(chr(cp))
230
+ if d and not d.startswith("<"):
231
+ parts = d.split()
232
+ if len(parts) == 2:
233
+ seconds.add(int(parts[1], 16))
234
+ _SECONDS = frozenset(seconds)
235
+ return _SECONDS
236
+
237
+
238
+ def nfc14(s: str) -> str:
239
+ """NFC as Unicode 14.0 defines it, under any later Unicode version of the library."""
240
+ from ._unicode14 import assigned
241
+ out, start = [], 0
242
+ for i, ch in enumerate(s):
243
+ if not assigned(ord(ch)):
244
+ out.append(unicodedata.normalize("NFC", s[start:i]))
245
+ out.append(ch)
246
+ start = i + 1
247
+ out.append(unicodedata.normalize("NFC", s[start:]))
248
+ return "".join(out)
249
+
250
+
251
+ def _nfc_with_map(chars: list[tuple[str, int]]) -> list[tuple[str, int]]:
252
+ """NFC (Unicode 14.0) of the characters, each result character carrying the source index of the first character
253
+ of its normalization segment (a segment starts at a starter that nothing before it can compose or reorder with;
254
+ a code point not assigned in 14.0 is always one)."""
255
+ from ._unicode14 import assigned
256
+ s = "".join(c for c, _ in chars)
257
+ target = nfc14(s)
258
+ if target == s:
259
+ return chars
260
+ seconds = _composing_seconds()
261
+
262
+ def boundary(ch: str) -> bool:
263
+ if not assigned(ord(ch)):
264
+ return True
265
+ if unicodedata.combining(ch) or ord(ch) in seconds:
266
+ return False
267
+ first = unicodedata.normalize("NFD", ch)[0]
268
+ return unicodedata.combining(first) == 0
269
+
270
+ out: list[tuple[str, int]] = []
271
+ a, n = 0, len(chars)
272
+ while a < n:
273
+ b = a + 1
274
+ while b < n and not boundary(chars[b][0]):
275
+ b += 1
276
+ seg = "".join(c for c, _ in chars[a:b])
277
+ norm = nfc14(seg)
278
+ if norm == seg:
279
+ out.extend(chars[a:b])
280
+ else:
281
+ out.extend((c, chars[a][1]) for c in norm)
282
+ a = b
283
+ if "".join(c for c, _ in out) != target:
284
+ raise AssertionError("normalization segments do not compose to the NFC of the whole")
285
+ return out
286
+
287
+
288
+ def _canonicalize_v5(text: str) -> tuple[str, dict[int, int]]:
289
+ chars = _nfc_with_map(_deleted(text))
290
+ tmp: list[str] = []
291
+ tmap: list[int] = []
292
+ for ch, i in chars: # no-break spaces -> space, typographic folding
293
+ if ch in _FOLD:
294
+ rep = _FOLD[ch]
295
+ tmp.extend(rep)
296
+ tmap.extend([i] + [-1] * (len(rep) - 1))
297
+ else:
298
+ tmp.append(" " if ch in _SPACE else ch)
299
+ tmap.append(i)
300
+ res: list[str] = []
301
+ rmap: list[int] = []
302
+ k, length = 0, len(tmp)
303
+ while k < length: # drop spaces/tabs that end a line or the text
304
+ if tmp[k] in _TRAILING:
305
+ e = k
306
+ while e < length and tmp[e] in _TRAILING:
307
+ e += 1
308
+ if e == length or tmp[e] == "\n":
309
+ k = e
310
+ continue
311
+ res.append(tmp[k])
312
+ rmap.append(tmap[k])
313
+ k += 1
314
+ joined = "".join(res)
315
+ lead = len(joined) - len(joined.lstrip())
316
+ canon = joined.strip()
317
+ index_map: dict[int, int] = {}
318
+ for ci, src_i in enumerate(rmap):
319
+ if src_i >= 0 and lead <= ci < lead + len(canon):
320
+ index_map.setdefault(src_i, ci - lead) # a composed segment maps to its first character
321
+ return canon, index_map
@@ -0,0 +1,325 @@
1
+ """Canonical generation: sealed text is written, token by token, exactly as the verifier will re-read it.
2
+
3
+ The canonical form (canon.py, v4) folds typographic characters, deletes invisible ones and strips whitespace at the
4
+ ends of lines and of the text, so that a text pasted through a word processor still verifies. A verifier or a
5
+ detector tokenizes the canonical text; where that differs from the tokens the seal enforced, the checkpoints near
6
+ the difference are read in another context. Canonical generation keeps the two identical:
7
+
8
+ - no character the canonical form changes or deletes (curly quotes, apostrophe letters, typographic dashes, the
9
+ ellipsis character, guillemets, no-break and zero-width spaces, BOM, soft hyphen, word joiner, carriage return);
10
+ - valid UTF-8 throughout. Tokens are byte strings and a character may be split across tokens, so the guard tracks
11
+ the unfinished character at the end of the text: inside it only continuation bytes follow, and the text does not
12
+ end there. A token never stops inside a forbidden character past its lead byte; and where the verifier's first
13
+ choice is a banned token, no token may stop inside any character at that step, or a byte-level tokenizer would
14
+ complete the half-written quote into a stray neighbour (a dagger, a star);
15
+ - no space or tab before a line break (inside a token, or a line break right after a token that ends in one);
16
+ - the first token taken as the verifier reads it once leading whitespace is stripped (``lead``);
17
+ - no join the verifier would re-tokenize differently (``joinable``); a HIGH checkpoint never takes a byte prefix
18
+ of the first choice (``high_exclusion``), which the model would complete into a token read back as one.
19
+
20
+ Where the verifier's first choice is a token canonical generation may not emit, a LOW checkpoint is left free; a
21
+ detector recognizes these positions from the text and the model alone (``unsatisfiable``) and skips them, so its
22
+ exact null distribution is kept.
23
+ """
24
+ from __future__ import annotations
25
+
26
+ import codecs
27
+ import re
28
+
29
+ import torch
30
+
31
+ # every character canonical form v4 changes or deletes, written as code points (the source stays ASCII);
32
+ # long variation-selector runs are left out (a model does not write them) and single selectors are content
33
+ FORBIDDEN_CODEPOINTS = (0x000D, 0xFEFF, 0x200B, 0x2060, 0x00AD, 0x00A0, 0x2007, 0x202F,
34
+ 0x201C, 0x201D, 0x201E, 0x201F, 0x00AB, 0x00BB, 0x2018, 0x2019, 0x201A, 0x201B, 0x02BC,
35
+ 0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015, 0x2212, 0x2026)
36
+ # and the replacement character, never content: a model trained on scraped text writes it for a banned apostrophe
37
+ # ("don" + U+FFFD + "t" is how mangled web pages spell it)
38
+ FORBIDDEN_CODEPOINTS += (0xFFFD,)
39
+ FORBIDDEN = tuple(chr(c).encode("utf-8") for c in FORBIDDEN_CODEPOINTS)
40
+ PREFIXES = sorted({f[:j] for f in FORBIDDEN for j in range(1, len(f))}) # tracked exactly
41
+ PREFIX_SET = frozenset(PREFIXES)
42
+ DANGEROUS = frozenset(f[:j] for f in FORBIDDEN for j in range(2, len(f))) # a token never stops here
43
+ # stand-ins for "inside some other character, n bytes to go" (none is a prefix of a forbidden character)
44
+ INSIDE = {1: bytes([0xE3, 0x80]), 2: bytes([0xE3]), 3: bytes([0xF0])}
45
+ LINE_END_WS = (b" \n", b"\t\n", b" \r", b"\t\r")
46
+ # continuations tried when a join is checked: end of text, a letter, a space and a letter, a line break, a full
47
+ # stop, a digit. Some tokenizers split a whitespace run by what follows it (GPT-2 reads "\n\nThe" as three tokens
48
+ # but "\n\n The" as two), so a join is kept when some continuation leaves it intact.
49
+ PROBES = (b"", b"x", b" x", b"\n", b".", b"1")
50
+
51
+
52
+ def _bytes_to_unicode() -> dict[int, str]:
53
+ """The byte <-> printable-character table of byte-level BPE (GPT-2, Llama 3, Qwen)."""
54
+ bs = list(range(ord("!"), ord("~") + 1)) + list(range(0xA1, 0xAD)) + list(range(0xAE, 0x100))
55
+ cs, n = bs[:], 0
56
+ for b in range(256):
57
+ if b not in bs:
58
+ bs.append(b)
59
+ cs.append(256 + n)
60
+ n += 1
61
+ return dict(zip(bs, (chr(c) for c in cs)))
62
+
63
+
64
+ def token_bytes(tok) -> list[bytes | None]:
65
+ """Raw bytes of every token id; None for special tokens (they never enter the text)."""
66
+ special = set(tok.all_special_ids) | {i for i, t in tok.added_tokens_decoder.items() if getattr(t, "special", False)}
67
+ vocab = tok.get_vocab()
68
+ n = max(vocab.values()) + 1
69
+ pieces = [None] * n
70
+ for piece, i in vocab.items():
71
+ pieces[i] = piece
72
+ space_marker, sp_marker = chr(0x120), chr(0x2581) # GPT-2 and SentencePiece space marks
73
+ byte_level = any(space_marker in p for p in pieces[:5000] if p)
74
+ dec = {c: b for b, c in _bytes_to_unicode().items()}
75
+ fallback = re.compile(r"<0x([0-9A-Fa-f]{2})>")
76
+ out: list[bytes | None] = []
77
+ for i, p in enumerate(pieces):
78
+ if p is None or i in special:
79
+ out.append(None)
80
+ elif byte_level:
81
+ out.append(bytes(dec[c] for c in p) if all(c in dec for c in p) else p.encode("utf-8"))
82
+ else:
83
+ m = fallback.fullmatch(p)
84
+ out.append(bytes([int(m.group(1), 16)]) if m else p.replace(sp_marker, " ").encode("utf-8"))
85
+ return out
86
+
87
+
88
+ def _tail(b: bytes) -> bytes:
89
+ """The incomplete UTF-8 character at the end of ``b`` (empty if it ends on a character boundary)."""
90
+ for k in range(1, min(4, len(b)) + 1):
91
+ c = b[-k]
92
+ if c & 0xC0 != 0x80: # a lead byte (or ASCII): the last character starts here
93
+ need = 1 if c < 0x80 else 2 if c >> 5 == 0x6 else 3 if c >> 4 == 0xE else 4 if c >> 3 == 0x1E else 1
94
+ return b[-k:] if k < need else b""
95
+ return b""
96
+
97
+
98
+ def _need(tail: bytes) -> int:
99
+ """Continuation bytes still missing from an unfinished character."""
100
+ c = tail[0]
101
+ total = 2 if c >> 5 == 0x6 else 3 if c >> 4 == 0xE else 4 if c >> 3 == 0x1E else 1
102
+ return max(0, total - len(tail))
103
+
104
+
105
+ def _utf8_ok(b: bytes) -> bool:
106
+ """No invalid sequence in ``b`` (an unfinished character at its end is allowed)."""
107
+ try:
108
+ codecs.getincrementaldecoder("utf-8")().decode(b, final=False)
109
+ return True
110
+ except UnicodeDecodeError:
111
+ return False
112
+
113
+
114
+ class CanonGuard:
115
+ """Per-step ban masks and checks that keep generated tokens identical to the verifier's reading."""
116
+
117
+ WINDOW = 8 # tokens of context re-tokenized when a join is checked
118
+ COMPARE = 4 # of which the last COMPARE must come back unchanged (the window's left edge may not)
119
+
120
+ def __init__(self, tok, vocab_size: int, device="cpu"):
121
+ self.tok = tok
122
+ self.bytes = token_bytes(tok)
123
+ # states: at a character boundary, inside one of the forbidden characters' prefixes (tracked exactly), or
124
+ # inside some other character with n bytes to go
125
+ self.states: list[bytes] = [b""] + PREFIXES + [INSIDE[n] for n in (1, 2, 3)]
126
+ self.index = {s: i for i, s in enumerate(self.states)}
127
+ n_tok = min(vocab_size, len(self.bytes))
128
+ masks = torch.zeros((len(self.states), vocab_size), dtype=torch.bool)
129
+ masks[:, n_tok:] = True # embedding rows without a token: never emitted
130
+ partial = torch.zeros(vocab_size, dtype=torch.bool)
131
+ nl = torch.zeros(vocab_size, dtype=torch.bool)
132
+ for i in range(n_tok):
133
+ tb = self.bytes[i]
134
+ if tb is None:
135
+ masks[1:, i] = True # a special token (the end of the text) never inside a character
136
+ continue
137
+ if not tb or any(w in tb for w in LINE_END_WS):
138
+ masks[:, i] = True # no text, or a space or tab before a line break
139
+ continue
140
+ nl[i] = tb[:1] in (b"\n", b"\r")
141
+ partial[i] = bool(_tail(tb))
142
+ plain = max(tb) < 0x80 and b"\r" not in tb
143
+ for si, s in enumerate(self.states):
144
+ if not s and plain:
145
+ continue # plain ASCII at a boundary: always fine
146
+ joined = s + tb
147
+ if not _utf8_ok(joined) or any(f in joined for f in FORBIDDEN) or _tail(joined) in DANGEROUS:
148
+ masks[si, i] = True
149
+ self.masks, self.masks_cpu = masks.to(device), masks
150
+ self.partial, self.partial_cpu = partial.to(device), partial # tokens that stop inside a character
151
+ self.nl_start, self.nl_start_cpu = nl.to(device), nl # banned after a trailing space or tab
152
+ self.banned_any = int(masks[0, :n_tok].sum())
153
+ self.lead = self._lead_map(tok, vocab_size) # CPU: verifier-view first token, -1 = not first
154
+ self.lead_ban = (self.lead < 0).to(device)
155
+ self.prefixes_of = self._prefix_table(n_tok)
156
+ self._cache: dict[tuple, bool] = {}
157
+
158
+ # ---------------------------------------------------------------------------------------------- tables
159
+ def _lead_map(self, tok, vocab_size: int) -> torch.Tensor:
160
+ """The token the verifier reads for a first token once leading whitespace is stripped; -1 where there is no
161
+ single such token (pure whitespace, a remainder of several tokens or not whole UTF-8)."""
162
+ lead = torch.arange(vocab_size, dtype=torch.long)
163
+ lead[len(self.bytes):] = -1 # embedding rows without a token
164
+ todo_ids: list[int] = []
165
+ todo_text: list[str] = []
166
+ for i, tb in enumerate(self.bytes[:vocab_size]):
167
+ if tb is None or not tb:
168
+ continue # special tokens do not enter the text
169
+ try:
170
+ s = tb.decode("utf-8")
171
+ except UnicodeDecodeError: # a piece of a character
172
+ if tb[:1].isspace() or tb[0] & 0xC0 == 0x80:
173
+ lead[i] = -1 # unreadable once stripped / no character starts here
174
+ continue
175
+ rem = s.lstrip()
176
+ if rem == s:
177
+ continue
178
+ if rem:
179
+ todo_ids.append(i)
180
+ todo_text.append(rem)
181
+ else:
182
+ lead[i] = -1 # pure whitespace: stripped away entirely
183
+ if todo_text:
184
+ for i, e in zip(todo_ids, tok(todo_text, add_special_tokens=False)["input_ids"]):
185
+ lead[i] = e[0] if len(e) == 1 else -1
186
+ return lead
187
+
188
+ def _prefix_table(self, n_tok: int) -> dict[int, list[int]]:
189
+ """Token -> the tokens whose bytes are a proper prefix of its bytes. A HIGH checkpoint must not pick a prefix
190
+ of the first choice (" question" for " questionnaire", "." for a full stop with line breaks): the model
191
+ would complete the word and the verifier re-read the merged token."""
192
+ by_bytes: dict[bytes, list[int]] = {}
193
+ for i, tb in enumerate(self.bytes[:n_tok]):
194
+ if tb:
195
+ by_bytes.setdefault(tb, []).append(i)
196
+ table: dict[int, list[int]] = {}
197
+ for i, tb in enumerate(self.bytes[:n_tok]):
198
+ if tb and len(tb) > 1:
199
+ pre = [t for m in range(1, len(tb)) for t in by_bytes.get(tb[:m], ())]
200
+ if pre:
201
+ table[i] = pre
202
+ # and the tokens that only add whitespace to it ("\n\n" for "\n", ".\n\n" for "."): where a whitespace run
203
+ # ends is decided by what follows it, so a HIGH checkpoint must not move that boundary either
204
+ for i, tb in enumerate(self.bytes[:n_tok]):
205
+ if not tb or not tb[-1:].isspace():
206
+ continue
207
+ core = len(tb.rstrip()) # tb = tb[:core] + a whitespace run
208
+ for m in range(max(1, core), len(tb)):
209
+ for t in by_bytes.get(tb[:m], ()):
210
+ table.setdefault(t, []).append(i)
211
+ return table
212
+
213
+ # ------------------------------------------------------------------------------------------------- steps
214
+ def mask(self, state: int) -> torch.Tensor:
215
+ return self.masks[state]
216
+
217
+ def state_after(self, state: int, token_id: int) -> int:
218
+ tb = self.bytes[token_id] if token_id < len(self.bytes) else None
219
+ if not tb:
220
+ return 0
221
+ tail = _tail(self.states[state] + tb)
222
+ if not tail:
223
+ return 0
224
+ if tail in PREFIX_SET:
225
+ return self.index[tail]
226
+ n = _need(tail)
227
+ return self.index[INSIDE[n]] if n in INSIDE else 0
228
+
229
+ def ends_in_space(self, token_id: int) -> bool:
230
+ """Does the token end in a space or tab (so that a line break may not follow)?"""
231
+ tb = self.bytes[token_id] if token_id < len(self.bytes) else None
232
+ return bool(tb) and tb[-1:] in (b" ", b"\t")
233
+
234
+ def high_exclusion(self, first: list[int]) -> list[int]:
235
+ """Tokens a HIGH checkpoint must not choose, given the first choice(s) it has to avoid."""
236
+ return [t for f in first for t in self.prefixes_of.get(int(f), ())]
237
+
238
+ def joinable(self, view: list[int], t: int) -> bool:
239
+ """Would the verifier re-read the text of ``view`` + [t] as these very tokens, for some continuation of the
240
+ text? The last tokens are re-tokenized with each probe continuation; the last COMPARE tokens of the window
241
+ and t must come back unchanged, followed by exactly the probe. A join inside an unfinished character is
242
+ judged when the character is complete. A break already in the window (a join kept when nothing else
243
+ joined) is not held against the tokens after it."""
244
+ tb = self.bytes[t] if t < len(self.bytes) else None
245
+ if tb is None or not view:
246
+ return True
247
+ start = len(view) <= self.WINDOW # the window begins where the text begins
248
+ if len(view) > 1 and not self._intact(tuple(view[-self.WINDOW:]), start):
249
+ return True
250
+ return self._intact(tuple(view[-self.WINDOW:]) + (t,), start)
251
+
252
+ def _intact(self, win: tuple, start: bool) -> bool:
253
+ hit = self._cache.get((win, start))
254
+ if hit is None:
255
+ hit = self._check(list(win), start)
256
+ if len(self._cache) > 50000:
257
+ self._cache.clear()
258
+ self._cache[(win, start)] = hit
259
+ return hit
260
+
261
+ def _check(self, seq: list[int], start: bool = False) -> bool:
262
+ """Do the last COMPARE tokens of ``seq`` come back unchanged when its text is re-tokenized with some probe?
263
+ At the start of a text a leading space is the tokenizer's own prefix (SentencePiece), never text: the
264
+ canonical text does not begin with whitespace."""
265
+ win, t = seq[:-1], seq[-1]
266
+ tb = self.bytes[t] if t < len(self.bytes) else None
267
+ if tb is None:
268
+ return True
269
+ parts = [self.bytes[i] if i < len(self.bytes) else None for i in win]
270
+ if any(p is None for p in parts):
271
+ return True
272
+ body = b"".join(parts) + tb
273
+ if start and body[:1] == b" ":
274
+ body = body[1:]
275
+ cut = 0
276
+ while cut < len(body) and body[cut] & 0xC0 == 0x80: # a character cut by the window's left edge
277
+ cut += 1
278
+ body = body[cut:]
279
+ if _tail(body):
280
+ return True
281
+ want = win[-self.COMPARE:] + [t]
282
+ try:
283
+ texts = [(body + p).decode("utf-8") for p in PROBES]
284
+ except UnicodeDecodeError:
285
+ return True
286
+ for p, enc in zip(PROBES, self.tok(texts, add_special_tokens=False)["input_ids"]):
287
+ for q in range(3):
288
+ end = len(enc) - q
289
+ if end < len(want) or enc[end - len(want):end] != want:
290
+ continue
291
+ if b"".join(self.bytes[i] or b"" for i in enc[end:]) == p:
292
+ return True
293
+ return False
294
+
295
+ def first_joinable(self, view: list[int], scores: torch.Tensor, avoid, stops, limit: int = 64) -> int | None:
296
+ """The most probable token under ``scores`` (among the ``limit`` best, none in ``avoid`` or ``stops``) that
297
+ joins ``view`` as the verifier will read it; None if there is none."""
298
+ for t in torch.argsort(scores, descending=True)[:limit].tolist():
299
+ if scores[t] == float("-inf"):
300
+ break
301
+ if t not in avoid and t not in stops and self.joinable(view, t):
302
+ return t
303
+ return None
304
+
305
+ def unsatisfiable(self, ids: list[int], positions: list[int], first: list[int], stops) -> list[bool]:
306
+ """At each position (a token index of ``ids``): could canonical generation not have emitted the verifier's
307
+ first choice ``first`` there? (A stop token, a token the guard bans after the text so far, or a join the
308
+ verifier would re-read differently.) The sealer then left a LOW checkpoint free. Read from the text and the
309
+ model, never from the key, so a detector that skips these positions keeps the exact null."""
310
+ states, spaces, s, sp = [], [], 0, False
311
+ for t in ids[: max(positions, default=-1) + 1]:
312
+ states.append(s)
313
+ spaces.append(sp)
314
+ s, sp = self.state_after(s, t), self.ends_in_space(t)
315
+ out = []
316
+ for p, f in zip(positions, first):
317
+ if f in stops or f >= self.masks_cpu.shape[1]:
318
+ out.append(True)
319
+ elif bool(self.masks_cpu[states[p], f]) or (spaces[p] and bool(self.nl_start_cpu[f])):
320
+ out.append(True)
321
+ elif p == 0:
322
+ out.append(bool(self.lead[f] < 0))
323
+ else:
324
+ out.append(f != ids[p] and not self.joinable(ids[:p], f))
325
+ return out