exegete 0.14.1a0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2796 @@
1
+ # SPDX-License-Identifier: LGPL-3.0-or-later
2
+ """The pseudonymisation engine (v0.12 flagship, D1).
3
+
4
+ Pure functions over strings and integers. No database, no server, no new
5
+ dependency: the edit list is computed here and is exact, so every stored
6
+ offset is remapped arithmetically rather than by diffing two texts. The
7
+ parity oracles in the test suite drive this module directly, which is
8
+ why it imports nothing of ours.
9
+
10
+ What it does, in one paragraph. A mapping of names to pseudonyms is
11
+ validated, compiled into ONE whole-word, escaped, longest-first
12
+ alternation, and run over a text in a single non-chaining pass. Each
13
+ match becomes an edit. Every `code_text`, `annotation` and `case_text`
14
+ span of that text is then remapped under one of two policies:
15
+ `snap_to_pseudonym`, which treats the pseudonym as the same token as the
16
+ name it replaces and never empties or deletes a span, and
17
+ `qualcoder_edit_parity`, which reproduces the semantics QualCoder's own
18
+ coding-view editor applies when a whole token is deleted and retyped.
19
+
20
+ Upstream reference at pin 9bddf17 (QualCoder master), read-only:
21
+
22
+ - the boundary regex and the escaping: `manage_files.py:3344-3349`,
23
+ `re.sub(rf"(?<!\\w){re.escape(original)}(?!\\w)", pseudonym, text_)`,
24
+ applied entry by entry over the whole text at import;
25
+ - the minimum lengths: `pseudonyms.py:72-74` (original, 2) and
26
+ `pseudonyms.py:76-78` (pseudonym, 3);
27
+ - the span walk `qualcoder_edit_parity` reproduces: `apply_insert` at
28
+ `code_text.py:5893-5904`, `apply_delete` at `:5906-5927`, the running
29
+ offset at `:5933-5950`, the end clamp at `:6208-6209` (case links) and
30
+ `:6232-6233` (codings), and the absence of a clamp for annotations at
31
+ `:6213-6222`.
32
+
33
+ Two deliberate deviations from upstream, both documented in the tool
34
+ description and pinned by tests:
35
+
36
+ 1. ONE pass, not a chain. Upstream applies each entry as its own
37
+ `re.sub` over the whole text, so a later entry rewrites what an
38
+ earlier one produced (`[Sam -> Alex, Alex -> Pat]` turns every "Sam"
39
+ into "Pat"). This engine refuses a mapping whose pseudonym is also a
40
+ name in the mapping and matches everything in one leftmost-longest
41
+ pass, so no replacement is ever replaced again.
42
+ 2. `qualcoder_edit_parity` reproduces the SEMANTICS of master's walk for
43
+ a whole-token replacement (delete the name, insert the pseudonym at
44
+ the same offset), not the output of diff-match-patch. Upstream feeds
45
+ its walk whatever `diff_main` produced, and `diff_main` may factor a
46
+ common prefix or suffix out of a replacement, which would move the
47
+ edit boundaries. That factoring is a property of the diff library and
48
+ is not reproduced here; the claim this module makes is the narrower,
49
+ true one.
50
+ """
51
+
52
+ import re
53
+ import unicodedata
54
+ from bisect import bisect_left, bisect_right
55
+ from typing import Any, Dict, List, Optional, Sequence, Tuple
56
+
57
+ # --------------------------------------------------------------------------
58
+ # Limits and vocabularies
59
+ # --------------------------------------------------------------------------
60
+
61
+ # Upstream's own minimum for an original (pseudonyms.py:72-74). Two code
62
+ # points, so a one-letter initial cannot be turned into a rule that
63
+ # rewrites half the transcript.
64
+ MIN_ORIGINAL_CHARS = 2
65
+ # Upstream's own minimum for a pseudonym (pseudonyms.py:76-78).
66
+ MIN_PSEUDONYM_CHARS = 3
67
+ # Ours. A pseudonym is written into the researcher's text once per match,
68
+ # so its length is what bounds how far this tool can grow a file.
69
+ MAX_PSEUDONYM_CHARS = 200
70
+ # Ours. A surface form is a NAME, and 200 characters is already absurdly
71
+ # generous for one. The cap is not about the text it matches, it is about
72
+ # `max_form_len`: the overlap diagnostic scans a window of that width
73
+ # around every match, so an uncapped form makes a READ-ONLY preview, with
74
+ # no token and no approval, cost O(matches x form length). Measured
75
+ # before the cap: a 200,000-character original over a 99 KB file took 91
76
+ # seconds, and `validate_mapping` accepted a 1,000,000-character one
77
+ # (Security S4).
78
+ MAX_FORM_CHARS = 200
79
+ # Ours. The same amplification from the other side: entries were capped
80
+ # and variants were not, so one entry could carry 200,000 of them.
81
+ MAX_VARIANTS_PER_ENTRY = 50
82
+ # Ours. What the alternation, `forms_at`'s buckets and the token payload
83
+ # are actually sized by. 500 entries of 4 forms each is far past any real
84
+ # mapping.
85
+ MAX_TOTAL_FORMS = 2000
86
+ # Ours. Five hundred entries is far past any real mapping and keeps the
87
+ # alternation, the diagnostics and the token payload bounded.
88
+ MAX_ENTRIES = 500
89
+ # Ours. `include_context` returns file text; this caps one window.
90
+ MAX_CONTEXT_CHARS = 120
91
+ # Ours, and this is the cap that actually stops a preview turning into a
92
+ # transcript dump. Per-window was never enough: the number of windows was
93
+ # uncapped, so 120 characters each side of 400 shown spans returned
94
+ # 98,100 characters of a 20,800-character transcript, 4.7 times the file,
95
+ # because the windows overlap (Security S8). The budget is per preview
96
+ # call, across every file and every entry, and a block that runs out of
97
+ # it says `context_truncated`.
98
+ MAX_CONTEXT_TOTAL_CHARS = 10000
99
+ # Ours. A paging cap in the house shape (`validate_limit`): silently
100
+ # capped rather than refused, because a caller asking for more than this
101
+ # many span positions per entry is asking for "all of them".
102
+ MAX_SPANS_PER_ENTRY = 500
103
+ # Ours, and deliberately NOT driven by a tool argument: `overlap_conflicts`
104
+ # is part of the signed effect block, so its size must be the same on the
105
+ # preview call and on the execute call.
106
+ MAX_OVERLAP_CONFLICTS = 200
107
+ # Ours (fix round 6, the fourth re-verification's B4-3). The overlap
108
+ # diagnostic examines, for every match in the file a call rewrites, every
109
+ # position within the longest form's reach and there every form starting
110
+ # with that position's letter: about 0.2 microseconds a form examined on
111
+ # the development Mac, so at 2,000 names a transcript with a speaker's
112
+ # label on every line examined 17 million a megabyte (3.4 s), and a
113
+ # mapping of long forms sharing a prefix 200 ms a match. Past this many
114
+ # the diagnostic stops, lists what it found and says it was capped
115
+ # (`overlap_conflicts_capped`); about half a second here. Counted, not
116
+ # timed, so the preview and the execute call cap it at the same place:
117
+ # the diagnostic is part of the signed effect. The rewrite is not
118
+ # affected: it never reads the diagnostic.
119
+ MAX_OVERLAP_CANDIDATES = 2_000_000
120
+ # Ours, a WARNING threshold and not a limit. The residue block reads a
121
+ # name as a bare substring, and QualCoder's own two-character minimum
122
+ # is well below the length at which that reading stops being exact: a
123
+ # form of fewer than four characters makes the residue counts generous
124
+ # (re-verification 6.2 measured the turn from zero false hits to one in
125
+ # five at four). The rewrite is whole-word and is not affected.
126
+ SHORT_FORM_CHARS = 4
127
+ # Ours (v0.13, ruling 7; re-derived in fix round 1 from S-4). The
128
+ # file-text count's work budget, deterministic rather than a clock so the
129
+ # same preview always counts the same files. A file's work is
130
+ # `len(text) * (surface forms + RESIDUE_WORK_PER_CHARACTER)`: the count
131
+ # costs a little per character whatever the mapping (the reader's reading,
132
+ # the two passes' own scan), and that term is what a one-form mapping
133
+ # spends most of, the worst case a rate taken at a hundred forms cannot
134
+ # see (the Security gate measured 6 seconds at one form under the old
135
+ # model). Fix round 1 measured 1 MB of English prose at 1, 10 and 100
136
+ # forms, best of five: 4.24 ms per MB per unit of work with the
137
+ # per-character term at 2.45 on Python 3.13.5, 3.98 and 2.32 on 3.11.13,
138
+ # rounded the term up to 3 and set the budget at 350 million. Past the
139
+ # budget a file is still read, with the one cheap question "does any
140
+ # name show here", and the report says which files those are; it is
141
+ # never omitted and never reported clean when it is not.
142
+ #
143
+ # Fix round 2 (the re-verification's B-1, B-2 and B-3) re-derived all
144
+ # three budgets from the WORST of: one form on ASCII text, one form on
145
+ # text that is not ASCII, the hostile dense run, and 1,997 forms with
146
+ # Turkish capitals under an insensitive mode, on Python 3.13.5 and
147
+ # 3.11.13 (BRIEF1_FIX2_REPORT.md has the figures). The term above was
148
+ # measured on text stripped to ASCII; on any other text (one curly
149
+ # apostrophe is enough) the reader's reading costs more per character, so
150
+ # the term is priced by the text's own class (`residue_work`): measured
151
+ # 2.2 to 2.7 on ASCII, rounded up to 3, and 3.4 to 6.7 on curly quotes,
152
+ # Cyrillic, Japanese and accented Latin composed or decomposed, rounded
153
+ # up to 7 (after `_strip_unseen` stopped costing 37 to 74 ms per MB). The
154
+ # unit rate is 3.9 to 5.6 ms per MB per unit in exact mode and 4.4 to 6.0
155
+ # in the insensitive ones, the worst 5.97 (3.11.13, curly quotes).
156
+ #
157
+ # Frozen under ruling 7 from the six CI platforms (run 35920173126 at
158
+ # 15b0859, the rate line's worst case at the budgets of the time, 200
159
+ # million, 60 million and 150,000): macOS 3.10 3.15 s, macOS 3.13 3.34,
160
+ # Ubuntu 3.13 4.00, Windows 3.13 4.05, Ubuntu 3.10 4.41, Windows 3.10
161
+ # 4.43. All three budgets were scaled by 0.45 to bring the slowest to
162
+ # about two seconds (the six rate lines then read 1.51 to 1.98 s). The
163
+ # file a call names is read first from the same budgets (ruling 1 as
164
+ # amended, fix round 4), and every count is charged all it spent, one
165
+ # that stops part-way too (fix round 5, the lead's rule 1), so nothing is
166
+ # spent outside them. A unit costs what the rate line measured only if
167
+ # the estimate prices what the count reads: since fix round 6 a file is
168
+ # priced at the length of its reading (`reading_length`, B4-1) and a
169
+ # form by its length (`FORM_UNIT_CHARS`, B4-2). The lead's scope of
170
+ # ruling 7: on realistic input (natural text in any script, mappings of
171
+ # up to 2,000 forms of up to 100 characters) the worst case is the
172
+ # three budgets' cost, the figure the rate line prints (0.93 to 0.96 s
173
+ # on the development Mac; through the tool the dearest shape measured,
174
+ # filling all three, 0.80 to 0.86 s at fix round 6). Crafted input is
175
+ # bounded, and linear in the text and the mapping, but may take longer:
176
+ # the dearest measured, a text of U+3316 (NFKC reads it as six
177
+ # characters and recomposes them, at about 3.6 times a unit of prose),
178
+ # fills the budgets in 1.9 to 2.1 s there; long forms sharing a prefix,
179
+ # 1.2 to 1.3 s. A file whose estimate passes the EMPTY budget is not
180
+ # counted at all, and is too large for this mapping (rule 2). A run of
181
+ # combining marks out of order no longer costs more than its length
182
+ # (`_ordered_runs`).
183
+ RESIDUE_WORK_PER_CHARACTER = 3
184
+ RESIDUE_WORK_PER_CHARACTER_NON_ASCII = 7
185
+ # Ours (fix round 6, the fourth re-verification's B4-2). A form is priced
186
+ # one unit for each started block of this many characters of its reading,
187
+ # not one unit whatever its length: over text that runs along a prefix
188
+ # many forms share, the detector's alternation compares every one of them
189
+ # as far as the prefix goes. Measured on this Mac: 100 names of up to ten
190
+ # characters over prose cost 5.4 ns a unit for the count and 6.2 for the
191
+ # cheap question, as before; 2,000 forms of 200 characters sharing 197
192
+ # over a run of that letter, 7.6 and 12.5 ns a unit priced by blocks of
193
+ # ten (152 and 250 priced one unit a form). Ten keeps every name of up to
194
+ # ten characters at one unit; a full name of eleven to twenty is two.
195
+ FORM_UNIT_CHARS = 10
196
+ MAX_RESIDUE_SCAN_WORK = 90_000_000
197
+ # Ours (fix round 2, the lead's ruling on B-2). Past the work or the match
198
+ # budget a file is asked the cheap question "does any name show here", and
199
+ # on a file where none shows that question reads the whole text: charged
200
+ # to this budget of its own, in the same units. It costs a little more
201
+ # per unit than the count (three scans where the count on ASCII makes
202
+ # two): 2.8 to 4.2 ms per MB per unit at one form, 4.5 to 6.9 at a
203
+ # hundred, so this budget's worst case is about 27 x 6.91 ms, 0.19
204
+ # seconds here (scaled with the others under ruling 7, above). A file
205
+ # whose question passes this budget on its own is not asked and closes
206
+ # nothing (rule 2); past it a file is not checked at all: it is listed in
207
+ # `files_not_checked`, the warning says how to get it checked, and it is
208
+ # never reported clean.
209
+ MAX_RESIDUE_CHECK_WORK = 27_000_000
210
+ # Ours (fix round 1, S-4; re-derived in fix round 2). Matches cost work
211
+ # the character model does not see: a text that is nothing but the name
212
+ # repeated has a match every few characters. Every match of the three
213
+ # passes counts against this budget, a first sight of a spelling as
214
+ # `RESIDUE_FIRST_SIGHT_MATCHES`. A count that passes it stops, is charged
215
+ # what it spent, and is reported counted in part, with a lower bound: a
216
+ # name certainly shows (fix round 5, the lead's rule 3). Every file after
217
+ # it drops to the cheap question, as past the work budget.
218
+ # The dearest unit measured is a first sight of a Turkish capital at 1,997
219
+ # forms under an insensitive mode, 2.2 microseconds on 3.11.13 (the dense
220
+ # typed run, a name inside a long run of word characters, 1.3 to 1.6), so
221
+ # this budget's worst case is about 0.15 seconds here. The three
222
+ # together, and the 0.08 seconds a 2,000-form mapping costs to prepare
223
+ # once per preview, are under one second on the development Mac and
224
+ # about two seconds on the slowest CI platform (ruling 7, above).
225
+ MAX_RESIDUE_SCAN_MATCHES = 67_500
226
+ # Ours (fix round 2, the re-verification's B-3). The first sight of a
227
+ # spelling in a pass is placed (the entry it belongs to, the rewriter's
228
+ # form, whether the reader's reading sees it) and every later sight is
229
+ # looked up, so a text written to make every sight a new spelling (case
230
+ # variants, Turkish capitals under an insensitive mode) costs more per
231
+ # match than the dense run the budget above was priced on. Measured at
232
+ # 1,997 forms: a first sight costs up to 6.9 microseconds more than a
233
+ # repeated one on 3.11.13 (4.5 on 3.13.5), against 1.4 to 1.7 for an
234
+ # ordinary match, so a first sight is charged five matches, and one more
235
+ # for each further form sharing its key that has to be tried.
236
+ RESIDUE_FIRST_SIGHT_MATCHES = 5
237
+ # Ours (fix round 1, S-3). Entry rows in one file row, in wide-count
238
+ # order; the row's own totals stay complete and the row says
239
+ # `entries_truncated`. The same cap bounds each of the row's two
240
+ # spelling lists.
241
+ MAX_RESIDUE_ENTRY_ROWS = 50
242
+ # Ours (v0.13). Full rows in the residue's file-text block. The totals are
243
+ # always complete, and every further file that shows a name has a compact
244
+ # row or, past that cap too, is listed by id: the house shape of
245
+ # `max_spans_per_entry` and `MAX_OVERLAP_CONFLICTS`.
246
+ MAX_RESIDUE_FILE_ROWS = 200
247
+ # Ours (fix round 2, the lead's ruling on CORR-4). Compact rows (id, name,
248
+ # the two readings' counts) have a cap of their own, so the owner's
249
+ # "compact by default" promise of a row for each file still showing a name
250
+ # holds up to a thousand files; past it a file is listed by id only in
251
+ # `more_files_showing_a_name`, and the detail note says so.
252
+ MAX_RESIDUE_COMPACT_ROWS = 1000
253
+ # Ours (v0.13, decision B). The longer words listed per entry per file
254
+ # on the typed path, most frequent first; the list says when it stopped.
255
+ MAX_LONGER_WORDS_PER_ENTRY = 20
256
+ # Ours (v0.13, the lead's ruling on S-1). A longer word is listed only
257
+ # when it extends the matched name by at most this many characters, left
258
+ # and right together: `Thomasson` (+3), `Thomas_P01` (+4), `Thomasin`
259
+ # (+2) and `Thomas_Smith` (+6) are words; a run of word characters that
260
+ # carries a date of birth and a record number joined to the name is not,
261
+ # and is counted and not listed, the list saying it is incomplete.
262
+ MAX_LONGER_WORD_EXTENSION = 8
263
+ # Ours (v0.13, the same ruling). All the longer words one preview lists
264
+ # share this many characters; past it an occurrence is counted and not
265
+ # listed. On by default on the typed path, so it sits well under the
266
+ # 10,000 characters `include_context` may spend when the caller opts in.
267
+ MAX_LONGER_WORDS_TOTAL_CHARS = 4000
268
+ # Scripts written without separators between words, where one character
269
+ # can be a word of its own and a run of "word characters" is a clause:
270
+ # Han, Hiragana, Katakana, Thai, Lao, Khmer and Myanmar (the lead's ruling
271
+ # on S-1). A longer word whose extension carries any of these is never
272
+ # listed. Explicit ranges, because `unicodedata` does not expose the
273
+ # Script property; each range is pinned.
274
+ NO_SEPARATOR_RANGES = (
275
+ (0x0E00, 0x0E7F, "Thai"),
276
+ (0x0E80, 0x0EFF, "Lao"),
277
+ (0x1000, 0x109F, "Myanmar"),
278
+ (0x1780, 0x17FF, "Khmer"),
279
+ (0x19E0, 0x19FF, "Khmer Symbols"),
280
+ (0x2E80, 0x2EFF, "CJK Radicals Supplement"),
281
+ (0x2F00, 0x2FDF, "Kangxi Radicals"),
282
+ (0x3005, 0x3007, "CJK iteration and zero marks"),
283
+ (0x3021, 0x3029, "Hangzhou numerals"),
284
+ (0x3038, 0x303B, "CJK iteration marks"),
285
+ (0x3040, 0x309F, "Hiragana"),
286
+ (0x30A0, 0x30FF, "Katakana"),
287
+ (0x31F0, 0x31FF, "Katakana Phonetic Extensions"),
288
+ (0x3400, 0x4DBF, "CJK Unified Ideographs Extension A"),
289
+ (0x4E00, 0x9FFF, "CJK Unified Ideographs"),
290
+ (0xA9E0, 0xA9FF, "Myanmar Extended-B"),
291
+ (0xAA60, 0xAA7F, "Myanmar Extended-A"),
292
+ (0xF900, 0xFAFF, "CJK Compatibility Ideographs"),
293
+ (0xFF66, 0xFF9F, "Halfwidth Katakana"),
294
+ (0x1AFF0, 0x1B16F, "Kana Extended-B to Small Kana Extension"),
295
+ (0x20000, 0x2FA1F, "CJK Unified Ideographs Extensions B to F, "
296
+ "and the Compatibility Supplement"),
297
+ (0x30000, 0x323AF, "CJK Unified Ideographs Extensions G and H"),
298
+ )
299
+ _NO_SEPARATOR_STARTS = tuple(low for low, _, _ in NO_SEPARATOR_RANGES)
300
+
301
+ CASE_MODES = ("exact", "insensitive", "insensitive_preserve")
302
+ OVERLAP_POLICIES = ("snap_to_pseudonym", "qualcoder_edit_parity")
303
+
304
+ # How a row's span changed. One vocabulary for both policies, because the
305
+ # preview shape and the hidden-coder rule key on it. `substituted` is the
306
+ # owner's ruling of 2026-09-15 (QC40_PLATFORM 7.3(3), refining X1): a
307
+ # span that sat exactly on a replaced name and now sits exactly on its
308
+ # pseudonym is a pure substitution, whatever the two lengths, and is
309
+ # exempt from the hidden-coder override like a pure shift. `clamped` is
310
+ # the owner's ruling of 2026-09-16 (QC40_PLATFORM 7.4, decision 7): a
311
+ # span that CONTAINED a whole replaced name and changed length only
312
+ # because that name did is `resized` and is exempt too, so the one row
313
+ # whose extent changed for a reason the rewrite does not explain, the
314
+ # damaged row whose stored end lay past the end of the text, needs a
315
+ # class of its own to stay on the override side.
316
+ UNCHANGED = "unchanged"
317
+ SHIFTED = "shifted"
318
+ SUBSTITUTED = "substituted"
319
+ RESIZED = "resized"
320
+ SNAPPED = "snapped"
321
+ DELETED = "deleted"
322
+ CLAMPED = "clamped"
323
+
324
+ # Characters refused in any mapping string.
325
+ #
326
+ # D1 3.2 names line breaks, U+2029 and control characters. This pattern is
327
+ # a superset, and the additions are deliberate:
328
+ #
329
+ # - U+2028 as well as U+2029, because both are line separators Qt's
330
+ # document model treats as breaks;
331
+ # - the bidirectional formatting controls, all of them: Unicode's
332
+ # Bidi_Control set is U+061C, U+200E, U+200F, U+202A to U+202E and
333
+ # U+2066 to U+2069, because a pseudonym is written INTO the
334
+ # researcher's text and one of these would visually reorder every line
335
+ # after it while leaving the stored offsets untouched. U+061C is the
336
+ # weakest of them (an invisible strong-AL character rather than a
337
+ # range control) and was the one this list missed; no pseudonym needs
338
+ # it, so the set is now the whole property rather than an enumeration;
339
+ # - U+FEFF, because QualCoder strips a leading one on import
340
+ # (manage_files.py:3340-3341), so a pseudonym carrying one would behave
341
+ # differently depending on where it landed.
342
+ #
343
+ # The zero-width joiner and non-joiner (U+200C, U+200D) are NOT refused:
344
+ # they are needed to spell ordinary words in several scripts.
345
+ # The ranges are built with chr() rather than written as escapes so that
346
+ # this source file itself contains no control characters.
347
+ _FORBIDDEN_RANGES = (
348
+ (0x0000, 0x001F), # C0 controls, which includes every line break
349
+ (0x007F, 0x009F), # DEL and the C1 controls
350
+ (0x061C, 0x061C), # ARABIC LETTER MARK, the fifth bidi control
351
+ (0x2028, 0x2029), # LINE SEPARATOR and PARAGRAPH SEPARATOR
352
+ (0x200E, 0x200F), # LEFT-TO-RIGHT and RIGHT-TO-LEFT MARK
353
+ (0x202A, 0x202E), # the bidirectional embeddings and overrides
354
+ (0x2066, 0x2069), # the bidirectional isolates
355
+ (0xFEFF, 0xFEFF), # ZERO WIDTH NO-BREAK SPACE, the byte-order mark
356
+ )
357
+ _FORBIDDEN_CHARS_RE = re.compile(
358
+ "[" + "".join(chr(lo) + "-" + chr(hi)
359
+ for lo, hi in _FORBIDDEN_RANGES) + "]")
360
+
361
+ # QualCoder 4.0's private-memo marker. A mapping string carrying one
362
+ # could plant a private zone in any memo a later release rewrites, and
363
+ # there is no reading of a name that needs five hashes in it.
364
+ _MARKER_RUN_RE = re.compile(r"#{5,}")
365
+
366
+ # Beyond the Basic Multilingual Plane. QualCoder's editor counts UTF-16
367
+ # code units, so a single astral code point makes Qt's offsets and the
368
+ # code-point offsets every report uses diverge from that point on
369
+ # (database.py `position_safe`). A pseudonym is ours to choose, so this is
370
+ # refused rather than warned about.
371
+ _ASTRAL_MINIMUM = 0x10000
372
+
373
+
374
+ def _has_astral(value: str) -> bool:
375
+ """Whether `value` carries a code point outside the BMP."""
376
+ return any(ord(char) >= _ASTRAL_MINIMUM for char in value)
377
+
378
+
379
+ class MappingError(ValueError):
380
+ """A mapping this engine refuses, with the caller-facing text.
381
+
382
+ A ValueError subclass so the server's tool guard turns it into the
383
+ ordinary sanitised error envelope without a branch of its own.
384
+ """
385
+
386
+
387
+ # --------------------------------------------------------------------------
388
+ # Validation
389
+ # --------------------------------------------------------------------------
390
+
391
+ def _describe_bad_characters(value: str) -> Optional[str]:
392
+ """Which character rule `value` breaks, or None."""
393
+ if _FORBIDDEN_CHARS_RE.search(value):
394
+ return "control"
395
+ if _MARKER_RUN_RE.search(value):
396
+ return "marker"
397
+ return None
398
+
399
+
400
+ def _check_common(value: Any, entry_index: int, label: str) -> str:
401
+ """The rules every mapping string shares. Returns the string."""
402
+ if not isinstance(value, str):
403
+ raise MappingError(
404
+ f"mapping entry {entry_index}: {label} must be a string.")
405
+ if value != value.strip():
406
+ raise MappingError(
407
+ f"mapping entry {entry_index}: {label} must not begin or end "
408
+ f"with whitespace.")
409
+ if _describe_bad_characters(value) is not None:
410
+ raise MappingError(
411
+ f"mapping entry {entry_index}: original/pseudonym/variant must "
412
+ f"not contain line breaks, control characters, bidirectional "
413
+ f"formatting characters or a run of five or more '#' "
414
+ f"characters.")
415
+ return value
416
+
417
+
418
+ def _check_name(value: Any, entry_index: int, label: str) -> str:
419
+ """An `original` or a `variant`: a surface form to look for."""
420
+ value = _check_common(value, entry_index, label)
421
+ if len(value) < MIN_ORIGINAL_CHARS:
422
+ raise MappingError(
423
+ f"mapping entry {entry_index}: {label} must be at least "
424
+ f"{MIN_ORIGINAL_CHARS} characters.")
425
+ if len(value) > MAX_FORM_CHARS:
426
+ raise MappingError(
427
+ f"mapping entry {entry_index}: {label} must be at most "
428
+ f"{MAX_FORM_CHARS} characters.")
429
+ return value
430
+
431
+
432
+ def _check_pseudonym(value: Any, entry_index: int,
433
+ may_echo_names: bool = True) -> str:
434
+ """A `pseudonym`: a string this engine writes into the file."""
435
+ value = _check_common(value, entry_index, "pseudonym")
436
+ if len(value) < MIN_PSEUDONYM_CHARS:
437
+ raise MappingError(
438
+ f"mapping entry {entry_index}: pseudonym must be at least "
439
+ f"{MIN_PSEUDONYM_CHARS} characters (QualCoder's own minimum).")
440
+ if len(value) > MAX_PSEUDONYM_CHARS:
441
+ raise MappingError(
442
+ f"mapping entry {entry_index}: pseudonym must be at most "
443
+ f"{MAX_PSEUDONYM_CHARS} characters.")
444
+ if _has_astral(value) or "\r" in value:
445
+ shown = f"pseudonym '{value}'" if may_echo_names else "the pseudonym"
446
+ raise MappingError(
447
+ f"mapping entry {entry_index}: {shown} contains "
448
+ f"characters beyond U+FFFF (emoji or similar) or a carriage "
449
+ f"return; QualCoder's editor would misplace positions in the "
450
+ f"rewritten file. Choose a pseudonym in the Basic Multilingual "
451
+ f"Plane.")
452
+ return value
453
+
454
+
455
+ class Entry:
456
+ """One validated mapping entry."""
457
+
458
+ __slots__ = ("index", "original", "pseudonym", "variants")
459
+
460
+ def __init__(self, index: int, original: str, pseudonym: str,
461
+ variants: Tuple[str, ...]):
462
+ self.index = index
463
+ self.original = original
464
+ self.pseudonym = pseudonym
465
+ self.variants = variants
466
+
467
+ @property
468
+ def forms(self) -> Tuple[str, ...]:
469
+ """Every surface form this entry replaces, original first."""
470
+ return (self.original,) + self.variants
471
+
472
+ def __repr__(self) -> str: # pragma: no cover - debugging
473
+ return (f"Entry({self.index}, pseudonym={self.pseudonym!r}, "
474
+ f"forms={len(self.forms)})")
475
+
476
+
477
+ class Mapping:
478
+ """A validated mapping, with the case mode it was validated under."""
479
+
480
+ __slots__ = ("entries", "case_mode", "shared_pseudonyms")
481
+
482
+ def __init__(self, entries: Tuple[Entry, ...], case_mode: str,
483
+ shared_pseudonyms: Tuple[Dict[str, Any], ...]):
484
+ self.entries = entries
485
+ self.case_mode = case_mode
486
+ # Two entries pointing at one pseudonym is a legitimate choice
487
+ # (two people merged into one identity), so it is reported rather
488
+ # than refused (D1 3.2).
489
+ self.shared_pseudonyms = shared_pseudonyms
490
+
491
+ def __len__(self) -> int:
492
+ return len(self.entries)
493
+
494
+
495
+ def _fold(value: str) -> str:
496
+ return value.casefold()
497
+
498
+
499
+ def validate_mapping(raw: Any, case_mode: str = "exact",
500
+ may_echo_names: bool = True) -> Mapping:
501
+ """Validate a caller's mapping, or raise MappingError.
502
+
503
+ Refused as a whole, with nothing previewed, on any of: an empty
504
+ mapping, more than `MAX_ENTRIES` entries, a malformed entry, a string
505
+ that breaks the character rules, a duplicate surface form, or a
506
+ pseudonym that is also a name in the same mapping.
507
+
508
+ That last rule is where this engine parts company with upstream.
509
+ QualCoder applies entries sequentially (`manage_files.py:3344-3349`),
510
+ so `[Sam -> Alex, Alex -> Pat]` silently turns every "Sam" into "Pat";
511
+ the dialog checks only that the list has no duplicate originals and no
512
+ duplicate pseudonyms (`pseudonyms.py:84-89`). Refusing is the honest
513
+ answer, because there is no reading of that mapping the researcher
514
+ meant.
515
+
516
+ `case_mode` matters here: in the two insensitive modes two forms that
517
+ differ only in case ARE the same rule, so they are duplicates, and a
518
+ pseudonym that case-folds onto a name would chain.
519
+
520
+ `may_echo_names=False` says the mapping did NOT come from the caller:
521
+ it was read out of the project's own `pseudonyms.json`, which is the
522
+ researcher's reverse key. D1 5.1's premise ("the mapping is
523
+ user-supplied and therefore already in the conversation") is false on
524
+ that path and D1 3.10 promises the opposite, so no refusal text on it
525
+ quotes a value. Entry indices carry everything the reader needs, and
526
+ the chaining refusal is where it mattered most: the value it quotes
527
+ is a pseudonym that is ALSO somebody's real name, which is the whole
528
+ reason the entry is refused (Security S3).
529
+ """
530
+ if case_mode not in CASE_MODES:
531
+ raise MappingError(
532
+ f"case_mode must be one of {', '.join(CASE_MODES)}.")
533
+ if raw is None or (isinstance(raw, (list, tuple)) and not raw):
534
+ raise MappingError(
535
+ "mapping is empty: give at least one entry with original and "
536
+ "pseudonym.")
537
+ if not isinstance(raw, (list, tuple)):
538
+ raise MappingError(
539
+ "mapping must be a list of entries, each an object with "
540
+ "original and pseudonym.")
541
+ if len(raw) > MAX_ENTRIES:
542
+ raise MappingError(
543
+ f"mapping has {len(raw)} entries; at most {MAX_ENTRIES} are "
544
+ f"accepted.")
545
+
546
+ insensitive = case_mode != "exact"
547
+ entries: List[Entry] = []
548
+ for index, item in enumerate(raw):
549
+ if not isinstance(item, dict):
550
+ raise MappingError(
551
+ f"mapping entry {index}: each entry must be an object with "
552
+ f"original and pseudonym.")
553
+ unknown = set(item) - {"original", "pseudonym", "variants"}
554
+ if unknown:
555
+ raise MappingError(
556
+ f"mapping entry {index}: unknown key(s) "
557
+ f"{', '.join(sorted(unknown))}; an entry has original, "
558
+ f"pseudonym and optionally variants.")
559
+ if "original" not in item or "pseudonym" not in item:
560
+ raise MappingError(
561
+ f"mapping entry {index}: each entry must have both original "
562
+ f"and pseudonym.")
563
+ original = _check_name(item["original"], index, "original")
564
+ pseudonym = _check_pseudonym(item["pseudonym"], index,
565
+ may_echo_names)
566
+ raw_variants = item.get("variants")
567
+ variants: List[str] = []
568
+ if raw_variants is not None:
569
+ if not isinstance(raw_variants, (list, tuple)):
570
+ raise MappingError(
571
+ f"mapping entry {index}: variants must be a list of "
572
+ f"strings.")
573
+ if len(raw_variants) > MAX_VARIANTS_PER_ENTRY:
574
+ raise MappingError(
575
+ f"mapping entry {index}: has {len(raw_variants)} "
576
+ f"variants; at most {MAX_VARIANTS_PER_ENTRY} are "
577
+ f"accepted per entry.")
578
+ for variant in raw_variants:
579
+ variants.append(_check_name(variant, index, "variant"))
580
+ entries.append(Entry(index, original, pseudonym, tuple(variants)))
581
+
582
+ total_forms = sum(len(entry.forms) for entry in entries)
583
+ if total_forms > MAX_TOTAL_FORMS:
584
+ raise MappingError(
585
+ f"mapping has {total_forms} surface forms (originals plus "
586
+ f"variants); at most {MAX_TOTAL_FORMS} are accepted.")
587
+
588
+ # Duplicate surface forms, across the whole mapping and within one
589
+ # entry. Exact spelling always; case-folded as well when the run is
590
+ # case-insensitive, because there the two are one rule.
591
+ seen: Dict[str, int] = {}
592
+ for entry in entries:
593
+ for form in entry.forms:
594
+ for key in ({form, _fold(form)} if insensitive else {form}):
595
+ previous = seen.get(key)
596
+ if previous is not None and previous != entry.index:
597
+ raise MappingError(
598
+ f"mapping entries {previous} and {entry.index}: the "
599
+ f"same original (or variant) appears twice.")
600
+ if previous is not None:
601
+ raise MappingError(
602
+ f"mapping entry {entry.index}: the same original (or "
603
+ f"variant) appears twice.")
604
+ seen[key] = entry.index
605
+
606
+ # A pseudonym that is also a name in this mapping would chain upstream
607
+ # and is refused here.
608
+ for entry in entries:
609
+ keys = {entry.pseudonym, _fold(entry.pseudonym)} if insensitive \
610
+ else {entry.pseudonym}
611
+ if any(key in seen for key in keys):
612
+ shown = (f"pseudonym '{entry.pseudonym}'" if may_echo_names
613
+ else f"the pseudonym of entry {entry.index}")
614
+ raise MappingError(
615
+ f"mapping entry {entry.index}: {shown} "
616
+ f"is also an original or variant in this "
617
+ f"mapping. QualCoder's import would chain the two "
618
+ f"replacements; this tool refuses instead. Choose a "
619
+ f"pseudonym that does not occur as a name in the mapping.")
620
+
621
+ by_pseudonym: Dict[str, List[int]] = {}
622
+ for entry in entries:
623
+ by_pseudonym.setdefault(entry.pseudonym, []).append(entry.index)
624
+ shared = tuple(
625
+ {"pseudonym": pseudonym, "entries": indices}
626
+ for pseudonym, indices in sorted(by_pseudonym.items())
627
+ if len(indices) > 1)
628
+ return Mapping(tuple(entries), case_mode, shared)
629
+
630
+
631
+ def canonical_mapping(mapping: Mapping) -> List[Dict[str, Any]]:
632
+ """The mapping as the authorisation token binds it (D1 3.8).
633
+
634
+ Entries sorted by original, variants sorted, every string normalised
635
+ to NFC, so the same mapping described in a different order binds
636
+ identically and a mapping read from `pseudonyms.json` binds the same
637
+ as the identical mapping typed out by the caller (which is what keeps
638
+ the source flag itself out of the binding).
639
+
640
+ Note what NFC does NOT do: the ENGINE matches the strings exactly as
641
+ supplied, never normalised, because normalising what we look for
642
+ could silently stop matching a file stored in another normal form.
643
+ So a mapping re-sent in a different normal form binds the same and
644
+ has a different effect, which the signed state catches as
645
+ "the project changed"; a test pins that.
646
+ """
647
+ def nfc(value: str) -> str:
648
+ return unicodedata.normalize("NFC", value)
649
+
650
+ return sorted(
651
+ ({"original": nfc(entry.original),
652
+ "pseudonym": nfc(entry.pseudonym),
653
+ "variants": sorted(nfc(v) for v in entry.variants)}
654
+ for entry in mapping.entries),
655
+ key=lambda item: (item["original"], item["pseudonym"],
656
+ item["variants"]))
657
+
658
+
659
+ def canonical_entry_positions(mapping: Mapping) -> Dict[int, int]:
660
+ """Each caller entry index, as its position in `canonical_mapping`.
661
+
662
+ The signed effect block records which ENTRY made each replacement,
663
+ and it used to record the caller's index: so the identical mapping
664
+ given in a different order, which `canonical_mapping` binds to the
665
+ same token, signed a different effect and was refused as
666
+ "the project changed" on a project that had not (fix round 3, S3).
667
+ Keyed on the canonical order, the same mapping signs the same effect
668
+ however it was typed, and a token issued from `pseudonyms.json`
669
+ executes from the identical typed mapping in any order.
670
+ """
671
+ def nfc(value: str) -> str:
672
+ return unicodedata.normalize("NFC", value)
673
+
674
+ order = sorted(
675
+ range(len(mapping.entries)),
676
+ key=lambda index: (nfc(mapping.entries[index].original),
677
+ nfc(mapping.entries[index].pseudonym),
678
+ sorted(nfc(v) for v in mapping.entries[index].variants)))
679
+ return {caller: position for position, caller in enumerate(order)}
680
+
681
+
682
+ # --------------------------------------------------------------------------
683
+ # Compiling and matching
684
+ # --------------------------------------------------------------------------
685
+
686
+ def _form_sort_key(item: Tuple[str, int]) -> Tuple[int, int, str]:
687
+ """Longest first, then entry order, then alphabetical (D1 3.4).
688
+
689
+ Length first is what makes "Mary Ann" win over "Mary" and
690
+ "Jean-Paul" over "Jean": Python's alternation takes the first branch
691
+ that lets the whole pattern succeed, so the longest viable form at a
692
+ position is chosen. The two remaining keys only make ties stable, so
693
+ the same mapping always compiles to the same pattern.
694
+ """
695
+ form, index = item
696
+ return (-len(form), index, form)
697
+
698
+
699
+ # Runs of anything that is not a letter or a digit, underscore included
700
+ # (`\W` alone excludes it, and `_` is the separator this whole finding is
701
+ # about). Used to join the parts of a multi-word name, so the detector
702
+ # reads "Mary Ann", "Mary_Ann", "Mary-Ann" and "MaryAnn" as the same
703
+ # name, which is what a person reading a file list does.
704
+ _SEPARATOR_RUN = r"[\W_]*"
705
+ _SEPARATOR_SPLIT = re.compile(r"[\W_]+")
706
+
707
+ # Code points a reader does not see: Unicode's Default_Ignorable_Code_Point
708
+ # property, transcribed as ranges from DerivedCoreProperties.txt of the
709
+ # Unicode version named below (4,174 code points in seventeen ranges;
710
+ # the same set in 14.0.0 and 16.0.0, and one short in 13.0.0, which had
711
+ # not yet assigned U+180F). `unicodedata` does not expose the property,
712
+ # and an interpreter's own tables are that interpreter's (Python 3.10
713
+ # carries 13.0.0), so the table is embedded and applied as it stands on
714
+ # every interpreter. It holds the format characters a reader never sees
715
+ # (the soft hyphen, the zero-width space, joiner and non-joiner, the
716
+ # word joiner, the bidi marks and controls, the byte order mark, the
717
+ # tag characters), the marks and fillers that render as nothing (the
718
+ # combining grapheme joiner, the variation selectors, the Mongolian free
719
+ # variation selectors, the Khmer inherent vowels, the Hangul fillers)
720
+ # and the reserved code points the standard says a renderer must treat
721
+ # the same way. Fix round 3 approximated the property with category Cf
722
+ # plus a hand list that stopped at U+180D, and U+180F and the reserved
723
+ # ranges walked a name past the detector into the manifest (fix round
724
+ # 4, R2). Each range boundary is pinned against this table, and the
725
+ # count against the file's own total.
726
+ DEFAULT_IGNORABLE_UNICODE_VERSION = "15.1.0"
727
+ DEFAULT_IGNORABLE_RANGES = (
728
+ (0x00AD, 0x00AD), # SOFT HYPHEN
729
+ (0x034F, 0x034F), # COMBINING GRAPHEME JOINER
730
+ (0x061C, 0x061C), # ARABIC LETTER MARK
731
+ (0x115F, 0x1160), # HANGUL CHOSEONG FILLER, HANGUL JUNGSEONG FILLER
732
+ (0x17B4, 0x17B5), # KHMER VOWEL INHERENT AQ, AA
733
+ (0x180B, 0x180F), # MONGOLIAN FREE VARIATION SELECTORS ONE to FOUR
734
+ # and the MONGOLIAN VOWEL SEPARATOR between them
735
+ (0x200B, 0x200F), # ZERO WIDTH SPACE to RIGHT-TO-LEFT MARK
736
+ (0x202A, 0x202E), # the bidi embeddings and overrides
737
+ (0x2060, 0x206F), # WORD JOINER to NOMINAL DIGIT SHAPES, with the
738
+ # reserved U+2065 and the bidi isolates inside
739
+ (0x3164, 0x3164), # HANGUL FILLER
740
+ (0xFE00, 0xFE0F), # VARIATION SELECTOR-1 to -16
741
+ (0xFEFF, 0xFEFF), # ZERO WIDTH NO-BREAK SPACE, the byte order mark
742
+ (0xFFA0, 0xFFA0), # HALFWIDTH HANGUL FILLER
743
+ (0xFFF0, 0xFFF8), # reserved
744
+ (0x1BCA0, 0x1BCA3), # the shorthand format controls
745
+ (0x1D173, 0x1D17A), # the musical symbol beam and phrase controls
746
+ (0xE0000, 0xE0FFF), # LANGUAGE TAG, the tag characters, VARIATION
747
+ # SELECTOR-17 to -256, and the reserved code
748
+ # points between and after them
749
+ )
750
+ DEFAULT_IGNORABLE_COUNT = 4174
751
+ _DEFAULT_IGNORABLE = frozenset(
752
+ code_point for low, high in DEFAULT_IGNORABLE_RANGES
753
+ for code_point in range(low, high + 1))
754
+ # A pattern that matches nothing, for a mapping whose every surface form
755
+ # is made of characters a reader does not see.
756
+ _NEVER = re.compile(r"(?!x)x")
757
+
758
+
759
+ def is_default_ignorable(code_point: int) -> bool:
760
+ """Whether Unicode's Default_Ignorable_Code_Point property holds."""
761
+ return code_point in _DEFAULT_IGNORABLE
762
+
763
+
764
+ # The sweep below, as one `str.translate` table: every code point in
765
+ # `_DEFAULT_IGNORABLE` and every code point the RUNNING interpreter puts
766
+ # in category Cf, each mapped to None. Built on the first non-ASCII call
767
+ # and held here, never at import: enumerating Cf over the whole code
768
+ # point space costs tens of milliseconds once, which a server that only
769
+ # ever sees ASCII should not pay (v0.13). Two threads that race to build
770
+ # it build the same table and the second assignment replaces the first
771
+ # with an equal one, so no lock is needed.
772
+ _UNSEEN_TABLE: Optional[Dict[int, None]] = None
773
+ # The same code points as one compiled character class, which is what
774
+ # `_strip_unseen` applies since fix round 2 (B-1): `str.translate` with a
775
+ # table of four thousand entries looks every character of a non-ASCII
776
+ # text up in a dictionary, 36 to 42 ms per MB on Python 3.13.5, while one
777
+ # character class applied with `re.sub` costs about 6 and gives the same
778
+ # string (pinned). Built with the table, on first use.
779
+ _UNSEEN_PATTERN: Optional["re.Pattern[str]"] = None
780
+
781
+
782
+ # The combining marks (categories Mn, Mc and Me of the running
783
+ # interpreter), as ranges; found in the same one sweep of the code points
784
+ # as the Cf characters above, and made into a pattern on first use.
785
+ _MARK_RANGES: Optional[List[List[int]]] = None
786
+
787
+
788
+ # The non-starters (fix round 3, the second re-verification's B2-1): the
789
+ # code points that decompose, under NFKD, to combining marks with a
790
+ # non-zero combining class only, which canonical reordering sorts among
791
+ # themselves. Found in the same one sweep, as ranges.
792
+ _NON_STARTER_RANGES: Optional[List[List[int]]] = None
793
+
794
+ # What each code point adds to the length the count reads (fix round 6,
795
+ # the fourth re-verification's B4-1), found in the same one sweep: for a
796
+ # code point whose NFKC, casefolded, is longer than one character, that
797
+ # length less one (U+FDFA adds 17). And where a composition with a mark
798
+ # folds longer than its two parts (six Greek letters with dialytika and
799
+ # an accent: U+03CA and U+03CB with an acute, a grave or a perispomeni),
800
+ # the letter it is built on adds the difference, so the sum over a text
801
+ # bounds its reading, composed or not. Made into a pattern on first use.
802
+ _READING_EXTRA: Optional[Dict[int, int]] = None
803
+ _READING_EXTRA_PATTERN: Optional["re.Pattern[str]"] = None
804
+
805
+
806
+ def _add_to_ranges(ranges: List[List[int]], code_point: int) -> None:
807
+ """Append `code_point` to sorted `ranges`, merging a neighbour."""
808
+ if ranges and code_point == ranges[-1][1] + 1:
809
+ ranges[-1][1] = code_point
810
+ else:
811
+ ranges.append([code_point, code_point])
812
+
813
+
814
+ def _sweep_code_points() -> None:
815
+ """One pass over every code point: the Cf characters for the unseen
816
+ table, the combining marks for `_mark_run_pattern`, and the
817
+ non-starters for `_ordered_runs`. About a tenth of a second once per
818
+ process, paid on the first text that is not ASCII and never at
819
+ import."""
820
+ global _UNSEEN_TABLE, _MARK_RANGES, _NON_STARTER_RANGES, _READING_EXTRA
821
+ table = dict.fromkeys(_DEFAULT_IGNORABLE)
822
+ marks: List[List[int]] = []
823
+ non_starters: List[List[int]] = []
824
+ readings: Dict[int, str] = {}
825
+ deficit: Dict[str, int] = {}
826
+ category = unicodedata.category
827
+ combining = unicodedata.combining
828
+ decomposition = unicodedata.decomposition
829
+ for code_point in range(0x110000):
830
+ char = chr(code_point)
831
+ kind = category(char)
832
+ if kind == "Cf":
833
+ table[code_point] = None
834
+ elif kind[0] == "M":
835
+ _add_to_ranges(marks, code_point)
836
+ parts = decomposition(char)
837
+ if combining(char) or (parts and all(
838
+ combining(part) for part in
839
+ unicodedata.normalize("NFKD", char))):
840
+ _add_to_ranges(non_starters, code_point)
841
+ # Only a letter folds to more than one character, and only a
842
+ # character with a decomposition normalises to more than one.
843
+ if parts or kind in ("Ll", "Lu", "Lt"):
844
+ readings[code_point] = unicodedata.normalize(
845
+ "NFKC", char).casefold()
846
+ if parts and not parts.startswith("<") and " " in parts:
847
+ # Any number of parts: a canonical decomposition is two in
848
+ # the Unicode data, but some interpreters' `decomposition()`
849
+ # also spells a Hangul syllable as its two or three jamo
850
+ # (Python 3.13.15 on CI; 3.13.5 and 3.13.7 return nothing).
851
+ pieces = [chr(int(part, 16)) for part in parts.split()]
852
+ short = len(char.casefold()) - sum(len(piece.casefold())
853
+ for piece in pieces)
854
+ if short > 0:
855
+ deficit[pieces[0]] = max(deficit.get(pieces[0], 0), short)
856
+ extra: Dict[int, int] = {}
857
+ for code_point, reading in readings.items():
858
+ added = len(reading) - 1 + (deficit.get(reading[-1], 0)
859
+ if reading else 0)
860
+ if added > 0:
861
+ extra[code_point] = added
862
+ _MARK_RANGES = marks
863
+ _NON_STARTER_RANGES = non_starters
864
+ _READING_EXTRA = extra
865
+ _UNSEEN_TABLE = table
866
+
867
+
868
+ # A run of non-starters longer than this is put in canonical order before
869
+ # the normalisation (fix round 3, B2-1); UAX #15's stream-safe bound.
870
+ LONG_RUN = 30
871
+ _LONG_RUN_PATTERN: Optional[Tuple["re.Pattern[str]", "re.Pattern[str]"]] = None
872
+
873
+
874
+ def _ordered_runs(text: str) -> str:
875
+ """`text` with every run of more than 30 non-starters decomposed and
876
+ put in canonical order in advance, so NFKC reads it in one pass.
877
+
878
+ Fix round 3 (the second re-verification's B2-1). Canonical reordering
879
+ inside NFKC is quadratic in the length of a run of combining marks
880
+ out of order, and no budget saw it: one crafted file of 320,000 such
881
+ marks cost a preview 36 seconds for 2.8 per cent of the work budget.
882
+ Each such run is replaced by its own NFKD, character by character,
883
+ sorted by combining class with a stable sort, which is what NFKD's
884
+ reordering does to it; so NFKC of the result is NFKC of the text,
885
+ exactly (pinned), and only the cost changes. A text already in NFKD
886
+ has every run in order and is returned as it stands; ordinary text
887
+ never has a run of more than a few marks.
888
+ """
889
+ global _LONG_RUN_PATTERN
890
+ if text.isascii() or unicodedata.is_normalized("NFKD", text):
891
+ return text
892
+ patterns = _LONG_RUN_PATTERN
893
+ if patterns is None:
894
+ if _NON_STARTER_RANGES is None:
895
+ _sweep_code_points()
896
+ # The BMP non-starters exactly, and every astral code point: a
897
+ # character class with seventy astral ranges is tested range by
898
+ # range, twenty times dearer per character than this one, and the
899
+ # few astral code points are sorted out exactly below. More than
900
+ # `LONG_RUN` (30) in a row: 31 or more.
901
+ patterns = _LONG_RUN_PATTERN = (
902
+ re.compile("[" + "".join(
903
+ re.escape(chr(low)) if low == high
904
+ else re.escape(chr(low)) + "-" + re.escape(chr(high))
905
+ for low, high in _NON_STARTER_RANGES
906
+ if high <= 0xFFFF) + "\U00010000-\U0010ffff]"),
907
+ re.compile("[" + "".join(
908
+ re.escape(chr(low)) if low == high
909
+ else re.escape(chr(low)) + "-" + re.escape(chr(high))
910
+ for low, high in _NON_STARTER_RANGES
911
+ if high <= 0xFFFF) + "\U00010000-\U0010ffff]{31,}"))
912
+ first = patterns[0].search(text)
913
+ if first is None:
914
+ return text
915
+ start = first.start()
916
+ return text[:start] + patterns[1].sub(_ordered_run, text[start:])
917
+
918
+
919
+ def _is_non_starter(char: str) -> bool:
920
+ """Whether `char` is one of the non-starters of the sweep."""
921
+ ranges = _NON_STARTER_RANGES or []
922
+ code_point = ord(char)
923
+ index = bisect_right(ranges, [code_point, 0x110000]) - 1
924
+ return index >= 0 and ranges[index][0] <= code_point <= ranges[index][1]
925
+
926
+
927
+ def _ordered_run(match: "re.Match[str]") -> str:
928
+ """Each maximal run of true non-starters in `match` (the superset may
929
+ have run on across astral code points that are not), when longer
930
+ than `LONG_RUN`, decomposed character by character and sorted by
931
+ combining class, stably."""
932
+ out: List[str] = []
933
+ segment: List[str] = []
934
+
935
+ def flush() -> None:
936
+ if len(segment) > LONG_RUN:
937
+ parts = [part for char in segment
938
+ for part in unicodedata.normalize("NFKD", char)]
939
+ parts.sort(key=unicodedata.combining)
940
+ out.append("".join(parts))
941
+ else:
942
+ out.append("".join(segment))
943
+ segment.clear()
944
+
945
+ for char in match.group(0):
946
+ if _is_non_starter(char):
947
+ segment.append(char)
948
+ else:
949
+ flush()
950
+ out.append(char)
951
+ flush()
952
+ return "".join(out)
953
+
954
+
955
+ def _unseen_table() -> Dict[int, None]:
956
+ """The code points `_strip_unseen` removes, as a translation table,
957
+ built once."""
958
+ table = _UNSEEN_TABLE
959
+ if table is None:
960
+ _sweep_code_points()
961
+ table = _UNSEEN_TABLE
962
+ return table
963
+
964
+
965
+ def _unseen_pattern() -> "re.Pattern[str]":
966
+ """`_unseen_table()`'s code points as one character class, built once.
967
+
968
+ Every run of consecutive code points becomes one range, each end
969
+ through `re.escape`, so nothing in the class is ever read as
970
+ syntax.
971
+ """
972
+ global _UNSEEN_PATTERN
973
+ pattern = _UNSEEN_PATTERN
974
+ if pattern is None:
975
+ ranges: List[List[int]] = []
976
+ for code_point in sorted(_unseen_table()):
977
+ if ranges and code_point == ranges[-1][1] + 1:
978
+ ranges[-1][1] = code_point
979
+ else:
980
+ ranges.append([code_point, code_point])
981
+ pattern = _UNSEEN_PATTERN = re.compile(
982
+ "[" + "".join(
983
+ re.escape(chr(low)) if low == high
984
+ else re.escape(chr(low)) + "-" + re.escape(chr(high))
985
+ for low, high in ranges) + "]")
986
+ return pattern
987
+
988
+
989
+ def _strip_unseen(text: str) -> str:
990
+ """`text` less every character a reader does not see, unnormalised.
991
+
992
+ The property above, and category Cf beside it. The format
993
+ characters outside the property (the Arabic, Syriac and Kaithi
994
+ number signs, the interlinear annotation controls, the Egyptian
995
+ hieroglyph format controls) are visible marks by Unicode's own
996
+ account, so stripping them is a widening rather than the property;
997
+ it is kept because a wider reading here is the safe one and the
998
+ round before this one already read that wide. Unlike the table,
999
+ the sweep reads the interpreter's own Unicode tables, so which
1000
+ visible format marks it removes beyond the property varies with
1001
+ the interpreter, in the over-detect direction only (fix round 5,
1002
+ D-1 note); the table does not move.
1003
+
1004
+ How the sweep is applied, since v0.13, and nothing about what it
1005
+ removes: pure ASCII is returned as it stands, because no ASCII code
1006
+ point is default-ignorable or in category Cf; anything else goes
1007
+ through `_unseen_pattern()`, the property and the interpreter's own
1008
+ Cf sweep united in one character class (fix round 2, B-1; it was a
1009
+ `str.translate` table, which cost six times as much per character).
1010
+ The output is the per-character generator's, byte for byte (pinned
1011
+ against a copy of that generator kept in the tests), at a fraction
1012
+ of its cost.
1013
+ """
1014
+ if text.isascii():
1015
+ return text
1016
+ return _unseen_pattern().sub("", text)
1017
+
1018
+
1019
+ def _reader_sees(value: str) -> str:
1020
+ """`value` as a reader sees it.
1021
+
1022
+ Compatibility normalisation (NFKC), so a fullwidth or otherwise
1023
+ compatibility-equivalent spelling reads as the plain one, and the
1024
+ default-ignorable code points removed, so an invisible character
1025
+ inside a word does not split it. Applied to every VALUE the detector
1026
+ is asked about; the forms go through `_detector_parts`, which
1027
+ splits before it strips. What this does NOT do, stated because the
1028
+ residue prose says so too: a look-alike letter from another script
1029
+ (a Cyrillic "о" for a Latin "o") is a different code point after
1030
+ every normalisation there is, and is out of scope. Spaces of every
1031
+ width are spaces: NFKC turns a thin or hair space into a plain one,
1032
+ and a plain one is a separator, not an invisible character.
1033
+ """
1034
+ return _strip_unseen(unicodedata.normalize("NFKC", _ordered_runs(value)))
1035
+
1036
+
1037
+ def _detector_parts(form: str) -> List[str]:
1038
+ """The words of one surface form, as a reader sees them.
1039
+
1040
+ Split FIRST, on the normalised form, and strip inside each word
1041
+ afterwards. The order matters and was wrong in fix round 3, which
1042
+ stripped before splitting: `Mary<ZWSP>Ann` fused into the one word
1043
+ `MaryAnn`, which no longer found "Mary Ann" or "Mary_Ann.txt", a
1044
+ narrower reading than the round before it had (fix round 4, R1). An
1045
+ invisible character between two words is a separator to whoever
1046
+ put it there (a soft hyphen at a line break, a zero-width space
1047
+ from a web page); one inside a word that `\\w` counts as a letter (a
1048
+ Hangul filler) is nothing at all, and stripping it inside the word
1049
+ reads the word whole. Either way the reading is at least as wide as
1050
+ the split alone gave.
1051
+ """
1052
+ text = unicodedata.normalize("NFKC", form)
1053
+ return [part for part in (_strip_unseen(piece)
1054
+ for piece in _SEPARATOR_SPLIT.split(text))
1055
+ if part]
1056
+
1057
+
1058
+ def _detector_alternative(form: str, fold: bool = False) -> str:
1059
+ """One surface form as a pattern that survives a changed separator.
1060
+
1061
+ A form of one word is itself, escaped. A form of several is its
1062
+ words escaped and joined by "any run of separators, or none", so a
1063
+ transcript called `Mary_Ann.txt` and a case called `MaryAnn` are
1064
+ both found. A form with no letters or digits at all (which
1065
+ validation permits: two punctuation marks pass the length rule) has
1066
+ no words to join and is used literally; one made entirely of
1067
+ characters a reader does not see comes back empty, and the caller
1068
+ drops it. With `fold`, the casefolded reading of the same words.
1069
+ """
1070
+ parts = _detector_parts(form)
1071
+ if fold:
1072
+ parts = [_fold(part) for part in parts]
1073
+ if parts:
1074
+ return _SEPARATOR_RUN.join(re.escape(part) for part in parts)
1075
+ literal = _reader_sees(form)
1076
+ return re.escape(_fold(literal) if fold else literal)
1077
+
1078
+
1079
+ def _detector_pattern(forms: Sequence[str], fold: bool = False) -> str:
1080
+ """The alternation over `forms`, empty alternatives left out."""
1081
+ return "|".join(alternative for alternative in
1082
+ (_detector_alternative(form, fold) for form in forms)
1083
+ if alternative)
1084
+
1085
+
1086
+ class NameDetector:
1087
+ """Would a human reading this string see one of the mapping's names?
1088
+
1089
+ The DETECTOR, and it is deliberately NOT the rewriter's matcher. The
1090
+ two answer different questions under opposite correctness conditions
1091
+ and must never be swapped for one another:
1092
+
1093
+ - `Compiled.pattern` is the REWRITER's matcher: master's file-import
1094
+ form, whole-word on both sides, so "Ann" is never replaced inside
1095
+ "Anna" and an unrelated word is never corrupted. Conservative is
1096
+ right there, and nothing about it changes.
1097
+ - This class decides whether a string is REPORTED as still carrying
1098
+ a name, or WITHHELD from a durable record because it carries one.
1099
+ Both are questions about what a reader sees, not about what the
1100
+ rewrite would fire on, so it is liberal by design: a bare
1101
+ substring with no word boundary at all, because `_` is a word
1102
+ character and `Thomas_interview.txt` is the commonest transcript
1103
+ file name there is; case-insensitive under EVERY `case_mode`,
1104
+ because `THOMAS_P01` names the participant whether or not this run
1105
+ would rewrite it; over originals and variants alike.
1106
+
1107
+ The asymmetry is the whole point. Over-detecting costs a withheld
1108
+ file name, which the file id replaces, or a residue count that is
1109
+ one too high, which sends the researcher to look at a label that
1110
+ does mention the name. Under-detecting costs a participant's real
1111
+ name in a journal entry that ships inside the project.
1112
+
1113
+ The cost is real and is stated rather than hidden: an entry for
1114
+ "Tom" makes a memo that says "tomorrow" count as carrying a name,
1115
+ because a rule that catches `ThomasB.txt` cannot also spare
1116
+ `tomorrow`, and of the two mistakes only one of them ships a
1117
+ participant's name. The tool description says the residue counts
1118
+ read wider than the rewrite.
1119
+ """
1120
+
1121
+ __slots__ = ("_direct", "_folded")
1122
+
1123
+ def __init__(self, forms: Sequence[str]):
1124
+ # Each form as a reader sees it, split into its words before
1125
+ # anything is stripped (`_detector_parts`), and never an empty
1126
+ # alternative: a form made entirely of invisible characters
1127
+ # (validation lets a zero-width space through, and two of them
1128
+ # pass the length rule) would otherwise match every string.
1129
+ if not any(_detector_alternative(form) for form in forms):
1130
+ self._direct = self._folded = _NEVER
1131
+ return
1132
+ self._direct = re.compile(_detector_pattern(forms), re.IGNORECASE)
1133
+ # A second reading, because `re.IGNORECASE` and `str.casefold` do
1134
+ # not agree on every code point (U+00DF casefolds to "ss" but
1135
+ # does not match "SS" under IGNORECASE). Two searches over one
1136
+ # short string cost nothing, and here the wider answer is the
1137
+ # safe one.
1138
+ self._folded = re.compile(_detector_pattern(forms, fold=True))
1139
+
1140
+ def contains(self, value: Any) -> bool:
1141
+ """True when a reader of `value` would see any surface form.
1142
+
1143
+ Anything that is not a non-empty string is False: a NULL label
1144
+ carries no name, and nothing downstream should have to know that
1145
+ a missing value and a clean one are different answers.
1146
+ """
1147
+ if not isinstance(value, str) or not value:
1148
+ return False
1149
+ text = _reader_sees(value)
1150
+ return bool(self._direct.search(text)
1151
+ or self._folded.search(_fold(text)))
1152
+
1153
+
1154
+ # `re.IGNORECASE` compares a literal character by character: a text
1155
+ # character matches a cased pattern character when the two have the same
1156
+ # simple lower case, or when that lower case is one of a few groups the
1157
+ # module treats as one letter (i and dotless i, s and long s, the Greek
1158
+ # letters with two lower cases). `str.casefold` is a different mapping
1159
+ # (U+0130 casefolds to two code points; U+00DF to two letters), which is
1160
+ # why a match the pattern accepts can miss the casefolded dictionary
1161
+ # (the re-verification's B-3, Turkish capitals under an insensitive
1162
+ # mode). The key below reproduces the module's own comparison, from the
1163
+ # same functions its compiler calls, so the forms a match can be sharing
1164
+ # a key with are found by one dictionary lookup. It is a short cut, never
1165
+ # the authority: every candidate is confirmed by a full match of its
1166
+ # compiled pattern, and a key the table cannot build (an interpreter
1167
+ # without these private functions) finds nothing and the exact scan
1168
+ # below it runs instead (fix round 2).
1169
+ try: # pragma: no cover - import
1170
+ import _sre
1171
+ _SRE_TOLOWER = getattr(_sre, "unicode_tolower", None)
1172
+ _SRE_ISCASED = getattr(_sre, "unicode_iscased", None)
1173
+ except ImportError: # pragma: no cover
1174
+ _SRE_TOLOWER = _SRE_ISCASED = None
1175
+
1176
+
1177
+ def _ignorecase_groups() -> Dict[int, Tuple[int, ...]]:
1178
+ """The module's extra case groups, keyed by lower case."""
1179
+ try:
1180
+ from re import _casefix # Python 3.11 and later
1181
+ return dict(_casefix._EXTRA_CASES)
1182
+ except (ImportError, AttributeError):
1183
+ pass
1184
+ try: # pragma: no cover - 3.10
1185
+ import warnings
1186
+ with warnings.catch_warnings():
1187
+ warnings.simplefilter("ignore")
1188
+ import sre_compile
1189
+ return dict(getattr(sre_compile, "_ignorecase_fixes", {}))
1190
+ except ImportError: # pragma: no cover
1191
+ return {}
1192
+
1193
+
1194
+ class _IgnoreCaseTable(dict):
1195
+ """A `str.translate` table that fills itself: each code point to the
1196
+ one standing for every character `re.IGNORECASE` treats as it."""
1197
+
1198
+ groups: Dict[int, Tuple[int, ...]] = {}
1199
+
1200
+ def __missing__(self, code_point: int) -> int:
1201
+ key = code_point
1202
+ if _SRE_TOLOWER is not None and _SRE_ISCASED(code_point):
1203
+ lower = _SRE_TOLOWER(code_point)
1204
+ key = min((lower,) + tuple(self.groups.get(lower, ())))
1205
+ self[code_point] = key
1206
+ return key
1207
+
1208
+
1209
+ _IGNORECASE_TABLE = _IgnoreCaseTable()
1210
+ _IgnoreCaseTable.groups = _ignorecase_groups()
1211
+
1212
+
1213
+ def ignorecase_key(value: str) -> str:
1214
+ """`value` with each character replaced by its `re.IGNORECASE` key."""
1215
+ return value.translate(_IGNORECASE_TABLE)
1216
+
1217
+
1218
+ def _word_key(value: str) -> str:
1219
+ """How the file-text count compares words: `ignorecase_key`, or
1220
+ casefolding where the interpreter lacks what that key is built from."""
1221
+ if _SRE_TOLOWER is not None:
1222
+ return ignorecase_key(value)
1223
+ return _fold(value) # pragma: no cover
1224
+
1225
+
1226
+ class Compiled:
1227
+ """A mapping compiled into one pattern, plus the lookups it needs."""
1228
+
1229
+ __slots__ = ("mapping", "case_mode", "flags", "pattern", "forms",
1230
+ "_exact", "_folded", "_by_first", "max_form_len",
1231
+ "pseudonym_pattern", "_pseudonym_entries", "detector",
1232
+ "_text_lookup", "_entry_memo", "_form_patterns",
1233
+ "_by_key", "_by_length", "_form_units")
1234
+
1235
+ def __init__(self, mapping: Mapping):
1236
+ self.mapping = mapping
1237
+ self.case_mode = mapping.case_mode
1238
+ self.flags = 0 if mapping.case_mode == "exact" else re.IGNORECASE
1239
+ forms: List[Tuple[str, int]] = []
1240
+ for entry in mapping.entries:
1241
+ for form in entry.forms:
1242
+ forms.append((form, entry.index))
1243
+ forms.sort(key=_form_sort_key)
1244
+ self.forms = tuple(forms)
1245
+ self.max_form_len = max(len(form) for form, _ in forms)
1246
+ # The master file-import form, verbatim: a Unicode-aware word
1247
+ # boundary on both sides and every surface form escaped, so no
1248
+ # caller string ever reaches the regex engine as syntax
1249
+ # (manage_files.py:3348; the survey importer at :134-140 still
1250
+ # interpolates unescaped, which is the bug this parity does not
1251
+ # inherit).
1252
+ self.pattern = re.compile(
1253
+ "(?<!\\w)(?:" + "|".join(re.escape(form) for form, _ in forms)
1254
+ + ")(?!\\w)", self.flags)
1255
+ # The other matcher, with the opposite contract: never used to
1256
+ # rewrite, always used to decide whether a string is reported or
1257
+ # withheld as carrying a name. See `NameDetector`.
1258
+ self.detector = NameDetector([form for form, _ in forms])
1259
+ self._exact = {form: index for form, index in forms}
1260
+ self._folded: Dict[str, int] = {}
1261
+ for form, index in forms:
1262
+ self._folded.setdefault(_fold(form), index)
1263
+ self._by_first: Dict[str, List[Tuple[str, int]]] = {}
1264
+ for form, index in forms:
1265
+ self._by_first.setdefault(_fold(form[0]), []).append((form, index))
1266
+
1267
+ pseudonyms = sorted({entry.pseudonym for entry in mapping.entries},
1268
+ key=lambda p: (-len(p), p))
1269
+ self.pseudonym_pattern = re.compile(
1270
+ "(?<!\\w)(?:" + "|".join(re.escape(p) for p in pseudonyms)
1271
+ + ")(?!\\w)", self.flags)
1272
+ self._pseudonym_entries: Dict[str, List[int]] = {}
1273
+ for entry in mapping.entries:
1274
+ key = _fold(entry.pseudonym) if self.flags else entry.pseudonym
1275
+ self._pseudonym_entries.setdefault(key, []).append(entry.index)
1276
+ # Built on first use by the file-text count, never here: nothing
1277
+ # on the rewrite path needs it (v0.13).
1278
+ self._text_lookup: Optional["_TextLookup"] = None
1279
+ # Fix round 2 (B-3): which entry a matched spelling belongs to is
1280
+ # decided once per spelling, and the per-form patterns the exact
1281
+ # fall-back full-matches are compiled once each and held here,
1282
+ # rather than recompiled through `re`'s cache of 512, which a
1283
+ # mapping of 2,000 forms overflows on every call.
1284
+ self._entry_memo: Dict[str, int] = {}
1285
+ self._form_patterns: Dict[str, "re.Pattern[str]"] = {}
1286
+ self._by_key: Optional[Dict[str, List[Tuple[str, int]]]] = None
1287
+ self._by_length: Optional[Dict[int, List[Tuple[str, int]]]] = None
1288
+ self._form_units: Optional[int] = None
1289
+
1290
+ @property
1291
+ def form_units(self) -> int:
1292
+ """The forms' part of `residue_work`'s price for one character,
1293
+ by their total length rather than their number (fix round 6, the
1294
+ fourth re-verification's B4-2): each form is a unit for every
1295
+ `FORM_UNIT_CHARS` characters it reads as, or part of them. Where
1296
+ the text runs along a prefix many forms share, the detector's
1297
+ alternation compares each of them as far as it goes, so a
1298
+ position costs up to the forms' total length, not their number; a
1299
+ name of up to ten characters is one unit, as every form was."""
1300
+ units = self._form_units
1301
+ if units is None:
1302
+ units = self._form_units = sum(
1303
+ -(-reading_length(form) // FORM_UNIT_CHARS)
1304
+ for form, _ in self.forms)
1305
+ return units
1306
+
1307
+ def text_lookup(self) -> "_TextLookup":
1308
+ """What `names_left_in_text` needs from these forms, built once."""
1309
+ if self._text_lookup is None:
1310
+ self._text_lookup = _TextLookup(self)
1311
+ return self._text_lookup
1312
+
1313
+ def carries_a_name(self, value: Any) -> bool:
1314
+ """Whether `value` shows a name to a reader OR to this run's rule.
1315
+
1316
+ The union of the two matchers, and the one predicate for every
1317
+ question of the form "is a name here": the wide count of a note,
1318
+ a label or an attribute value, whether a file shows a name, and
1319
+ whether a file name, a path or a pseudonym is withheld from a
1320
+ record. The detector alone reads NFKC, which composes a combining
1321
+ mark onto the letter before it, so a name followed by one ("Rene"
1322
+ typed decomposed as "René") is matched by the whole-word rule and
1323
+ not seen by the detector; the union keeps the wide reading at
1324
+ least as wide as the rewrite, which is what v0.12 founded it on,
1325
+ and makes every withholding withhold more, never less (lead's
1326
+ ruling of 2026-09-23 on QA-1).
1327
+ """
1328
+ if not isinstance(value, str) or not value:
1329
+ return False
1330
+ return bool(self.detector.contains(value)
1331
+ or self.pattern.search(value))
1332
+
1333
+ def entry_for(self, matched: str) -> int:
1334
+ """Which entry produced `matched`.
1335
+
1336
+ Three steps, fastest first, and the last is exact by
1337
+ construction: the combined pattern matched, so at least one
1338
+ escaped alternative full-matches the matched text under the same
1339
+ flags, and the alternatives are tested in the pattern's own
1340
+ order. The dictionaries are a short cut, not the authority,
1341
+ because `re.IGNORECASE` and `str.casefold` do not agree on every
1342
+ code point (U+00DF folds to "ss" but does not match "SS" under
1343
+ IGNORECASE).
1344
+
1345
+ Decided once per spelling (fix round 2, B-3): a transcript of a
1346
+ thousand speaker labels asks the same question a thousand times,
1347
+ and the last step, which the dictionaries miss under an
1348
+ insensitive mode for a spelling such as `ALİ`, cost 12 ms a match
1349
+ at 2,000 forms before it was memoised and indexed.
1350
+ """
1351
+ found = self._entry_memo.get(matched)
1352
+ if found is not None:
1353
+ return found
1354
+ found = self._exact.get(matched)
1355
+ if found is None and self.flags:
1356
+ found = self._folded.get(_fold(matched))
1357
+ if found is None:
1358
+ found = self.first_full_match(matched)[1]
1359
+ self._entry_memo[matched] = found
1360
+ return found
1361
+
1362
+ def form_pattern(self, form: str) -> "re.Pattern[str]":
1363
+ """One surface form, escaped and compiled under this case mode,
1364
+ once per form."""
1365
+ pattern = self._form_patterns.get(form)
1366
+ if pattern is None:
1367
+ pattern = self._form_patterns[form] = re.compile(
1368
+ re.escape(form), self.flags)
1369
+ return pattern
1370
+
1371
+ def first_full_match(self, matched: str) -> Tuple[str, int]:
1372
+ """The first form, in the pattern's own order, whose escaped
1373
+ alternative full-matches `matched` under this case mode.
1374
+
1375
+ Exact, and the answer the combined pattern's alternation gives:
1376
+ a literal matches character for character under `re.IGNORECASE`
1377
+ as without it, so only the forms of `matched`'s length can match,
1378
+ and among the forms sharing its `ignorecase_key` the first that
1379
+ full-matches is the first of all (the key is the module's own
1380
+ comparison). The key is tried first; the forms of the same length
1381
+ are the fall-back, in the same order, so a key that found
1382
+ nothing never changes an answer.
1383
+ """
1384
+ if self.flags:
1385
+ by_key = self._by_key
1386
+ if by_key is None:
1387
+ by_key = self._by_key = {}
1388
+ for form, index in self.forms:
1389
+ by_key.setdefault(ignorecase_key(form), []).append(
1390
+ (form, index))
1391
+ for form, index in by_key.get(ignorecase_key(matched), ()):
1392
+ if self.form_pattern(form).fullmatch(matched):
1393
+ return form, index
1394
+ by_length = self._by_length
1395
+ if by_length is None:
1396
+ by_length = self._by_length = {}
1397
+ for form, index in self.forms:
1398
+ by_length.setdefault(len(form), []).append((form, index))
1399
+ for form, index in by_length.get(len(matched), ()):
1400
+ if self.form_pattern(form).fullmatch(matched):
1401
+ return form, index
1402
+ raise AssertionError( # pragma: no cover - impossible
1403
+ "a matched span belongs to no surface form")
1404
+
1405
+ def forms_at(self, text: str, position: int) -> List[Tuple[str, int]]:
1406
+ """Every surface form that matches, standalone, at `position`.
1407
+
1408
+ Used only by the overlap diagnostic, which asks what the combined
1409
+ pass did NOT choose. Bucketed by first character so the scan is
1410
+ proportional to the forms that could start here rather than to
1411
+ the whole mapping.
1412
+ """
1413
+ if position >= len(text):
1414
+ return []
1415
+ found: List[Tuple[str, int]] = []
1416
+ for form, index in self._by_first.get(_fold(text[position]), ()):
1417
+ end = position + len(form)
1418
+ if end > len(text):
1419
+ continue
1420
+ candidate = text[position:end]
1421
+ if self.flags:
1422
+ # The regex, not a case-folded comparison: this decides
1423
+ # whether a form is REPORTED as having lost, so it has
1424
+ # to be the same test the matching pass itself applies.
1425
+ if not self.form_pattern(form).fullmatch(candidate):
1426
+ continue
1427
+ elif candidate != form:
1428
+ continue
1429
+ if position > 0 and _is_word_char(text[position - 1]):
1430
+ continue
1431
+ if end < len(text) and _is_word_char(text[end]):
1432
+ continue
1433
+ found.append((form, index))
1434
+ return found
1435
+
1436
+ def replacement_for(self, entry_index: int, matched: str) -> str:
1437
+ """The text that replaces one match, under this case mode.
1438
+
1439
+ Under `insensitive_preserve` this is a HEURISTIC, and the tool
1440
+ description says so in the same word: it copies the shape of the
1441
+ matched text onto the pseudonym, which gets SHOUTED and
1442
+ lower-case passages right because that is what transcripts
1443
+ actually contain, and takes the pseudonym as written for
1444
+ anything else.
1445
+ """
1446
+ pseudonym = self.mapping.entries[entry_index].pseudonym
1447
+ if self.case_mode != "insensitive_preserve":
1448
+ return pseudonym
1449
+ # Anything that is not all upper or all lower (mixed case, a
1450
+ # capitalised word) takes the pseudonym as the researcher wrote
1451
+ # it. The length guard is upstream-shaped rather than reachable:
1452
+ # no surface form is shorter than two characters.
1453
+ if len(matched) > 1 and matched.isupper():
1454
+ return pseudonym.upper()
1455
+ if matched.islower():
1456
+ return pseudonym.lower()
1457
+ return pseudonym
1458
+
1459
+
1460
+ _WORD_RE = re.compile(r"\w")
1461
+
1462
+
1463
+ def _is_word_char(char: str) -> bool:
1464
+ """`\\w` for one character, the way the boundary assertions read it."""
1465
+ return _WORD_RE.match(char) is not None
1466
+
1467
+
1468
+ class Replacement:
1469
+ """One edit: the span it covers and the text that takes its place."""
1470
+
1471
+ __slots__ = ("start", "end", "entry", "matched", "text")
1472
+
1473
+ def __init__(self, start: int, end: int, entry: int, matched: str,
1474
+ text: str):
1475
+ self.start = start
1476
+ self.end = end
1477
+ self.entry = entry
1478
+ self.matched = matched
1479
+ self.text = text
1480
+
1481
+ @property
1482
+ def delta(self) -> int:
1483
+ return len(self.text) - (self.end - self.start)
1484
+
1485
+ def __repr__(self) -> str: # pragma: no cover - debugging
1486
+ return f"Replacement({self.start}, {self.end}, entry={self.entry})"
1487
+
1488
+
1489
+ def find_replacements(compiled: Compiled, text: str) -> List[Replacement]:
1490
+ """Every edit this mapping makes to `text`, left to right.
1491
+
1492
+ One `finditer` over one pattern, so the matches are non-overlapping
1493
+ and leftmost, and at each start the longest viable surface form wins.
1494
+ Nothing this pass produces is ever matched again, which is the
1495
+ single-pass guarantee.
1496
+ """
1497
+ found: List[Replacement] = []
1498
+ for match in compiled.pattern.finditer(text):
1499
+ matched = match.group(0)
1500
+ entry = compiled.entry_for(matched)
1501
+ found.append(Replacement(match.start(), match.end(), entry, matched,
1502
+ compiled.replacement_for(entry, matched)))
1503
+ return found
1504
+
1505
+
1506
+ def apply_replacements(text: str, replacements: Sequence[Replacement]) -> str:
1507
+ """The rewritten text."""
1508
+ if not replacements:
1509
+ return text
1510
+ pieces: List[str] = []
1511
+ cursor = 0
1512
+ for item in replacements:
1513
+ pieces.append(text[cursor:item.start])
1514
+ pieces.append(item.text)
1515
+ cursor = item.end
1516
+ pieces.append(text[cursor:])
1517
+ return "".join(pieces)
1518
+
1519
+
1520
+ # --------------------------------------------------------------------------
1521
+ # Diagnostics (computed from the same pass)
1522
+ # --------------------------------------------------------------------------
1523
+
1524
+ def pre_existing_pseudonym_occurrences(
1525
+ compiled: Compiled, text: str,
1526
+ max_spans: Optional[int] = None) -> List[Dict[str, Any]]:
1527
+ """Where each pseudonym ALREADY occurs in the text (D1 3.4).
1528
+
1529
+ A warning, never a refusal (owner ruling Q3): some researchers reuse
1530
+ common given names deliberately. What it costs is told plainly: after
1531
+ the run the rewritten text cannot distinguish a pseudonym this tool
1532
+ wrote from one that was always there, so a reversal driven by text
1533
+ would be unsafe. The run manifest records spans rather than strings
1534
+ for exactly this reason.
1535
+ """
1536
+ counts: Dict[int, List[Tuple[int, int]]] = {}
1537
+ for match in compiled.pseudonym_pattern.finditer(text):
1538
+ matched = match.group(0)
1539
+ key = _fold(matched) if compiled.flags else matched
1540
+ for index in compiled._pseudonym_entries.get(key, ()):
1541
+ counts.setdefault(index, []).append((match.start(), match.end()))
1542
+ report: List[Dict[str, Any]] = []
1543
+ for index in sorted(counts):
1544
+ spans = counts[index]
1545
+ shown = spans if max_spans is None else spans[:max_spans]
1546
+ item: Dict[str, Any] = {
1547
+ "entry": index,
1548
+ "pseudonym": compiled.mapping.entries[index].pseudonym,
1549
+ "count": len(spans),
1550
+ "spans": [[s, e] for s, e in shown],
1551
+ }
1552
+ if max_spans is not None and len(spans) > len(shown):
1553
+ item["spans_truncated"] = True
1554
+ report.append(item)
1555
+ return report
1556
+
1557
+
1558
+ def overlap_conflicts(compiled: Compiled, text: str,
1559
+ replacements: Sequence[Replacement]
1560
+ ) -> Tuple[List[Dict[str, Any]], bool, bool]:
1561
+ """Surface forms that lost the competition for the same characters.
1562
+
1563
+ "Ann Marie" and "Marie Curie" over "Ann Marie Curie": leftmost and
1564
+ longest chooses "Ann Marie" and leaves "Curie" behind, so the second
1565
+ entry never fires there. The researcher has to see that, because
1566
+ nothing else in the preview would show it.
1567
+
1568
+ Only the neighbourhood of each chosen match is examined: a form can
1569
+ only conflict with a match it overlaps, so the scan is bounded by the
1570
+ number of matches times the longest surface form, never by the text.
1571
+ That can still be a great deal (the fourth re-verification's B4-3),
1572
+ so the forms examined are counted, and past
1573
+ `MAX_OVERLAP_CANDIDATES` the scan stops with what it has found.
1574
+
1575
+ Returns (conflicts, truncated, capped): truncated when
1576
+ `MAX_OVERLAP_CONFLICTS` conflicts were found, capped when the scan
1577
+ stopped at `MAX_OVERLAP_CANDIDATES` forms examined first.
1578
+ """
1579
+ conflicts: List[Dict[str, Any]] = []
1580
+ truncated = False
1581
+ examined = 0
1582
+ by_first = compiled._by_first
1583
+ for item in replacements:
1584
+ first = max(0, item.start - compiled.max_form_len + 1)
1585
+ for position in range(first, item.end):
1586
+ examined += 1 + len(by_first.get(_fold(text[position]), ()))
1587
+ if examined > MAX_OVERLAP_CANDIDATES:
1588
+ return conflicts, truncated, True
1589
+ for form, index in compiled.forms_at(text, position):
1590
+ end = position + len(form)
1591
+ if end <= item.start:
1592
+ continue # ends before this match
1593
+ if index == item.entry:
1594
+ continue # the same entry, not a clash
1595
+ if position == item.start and end == item.end:
1596
+ continue # the very match that was chosen
1597
+ if len(conflicts) >= MAX_OVERLAP_CONFLICTS:
1598
+ return conflicts, True, False
1599
+ conflicts.append({
1600
+ "entry": index,
1601
+ "form": form,
1602
+ "span": [position, end],
1603
+ "loses_to_entry": item.entry,
1604
+ "chosen_span": [item.start, item.end],
1605
+ })
1606
+ return conflicts, truncated, False
1607
+
1608
+
1609
+ def short_forms(mapping: Mapping) -> List[Dict[str, Any]]:
1610
+ """The surface forms shorter than `SHORT_FORM_CHARS` (a warning).
1611
+
1612
+ Entry index and length only, never the form: this block travels on
1613
+ the sidecar path too, where a surface form is the researcher's
1614
+ reverse key. Length is in code points, the same measure the
1615
+ validator's minimums use.
1616
+ """
1617
+ found: List[Dict[str, Any]] = []
1618
+ for entry in mapping.entries:
1619
+ for form in entry.forms:
1620
+ if len(form) < SHORT_FORM_CHARS:
1621
+ found.append({"entry": entry.index, "length": len(form)})
1622
+ return found
1623
+
1624
+
1625
+ def case_variants_seen(compiled: Compiled, text: str) -> List[Dict[str, Any]]:
1626
+ """Occurrences that differ from every form only in case (D1 3.4).
1627
+
1628
+ Reported under `case_mode="exact"` only, where they are precisely the
1629
+ occurrences this run does NOT replace. Counts, never spans: the point
1630
+ is "TOM appears twelve times and is not being touched", and a span
1631
+ list would not add to it.
1632
+ """
1633
+ if compiled.case_mode != "exact":
1634
+ return []
1635
+ insensitive = re.compile(compiled.pattern.pattern, re.IGNORECASE)
1636
+ pools: Dict[str, Dict[str, int]] = {}
1637
+ for match in insensitive.finditer(text):
1638
+ matched = match.group(0)
1639
+ pool = pools.setdefault(_fold(matched), {})
1640
+ pool[matched] = pool.get(matched, 0) + 1
1641
+ exact_forms = {form for form, _ in compiled.forms}
1642
+ report: List[Dict[str, Any]] = []
1643
+ for form, index in compiled.forms:
1644
+ pool = pools.get(_fold(form), {})
1645
+ other = sum(count for spelling, count in pool.items()
1646
+ if spelling not in exact_forms)
1647
+ if other:
1648
+ report.append({"entry": index, "form": form,
1649
+ "other_case_count": other})
1650
+ return report
1651
+
1652
+
1653
+ # --------------------------------------------------------------------------
1654
+ # Names left in the file text (v0.13, Brief 1 item 4)
1655
+ # --------------------------------------------------------------------------
1656
+
1657
+ # The kinds a wide occurrence in the file text is split into, in the
1658
+ # order the report lists them. `unattributed` is not a kind: it is the
1659
+ # occurrences no entry could be charged to, counted beside them.
1660
+ TEXT_RESIDUE_REASONS = (
1661
+ "inside_a_longer_word", "case_only", "joined_differently",
1662
+ "normalisation_variants", "put_back_by_a_pseudonym",
1663
+ "whole_word_in_a_file_not_rewritten")
1664
+
1665
+ # What `names_left_in_text` returns for one file, JSON-ready:
1666
+ # `occurrences` ({"wide": N, "whole_word": M}), `unattributed`,
1667
+ # `entries` (one row per entry with anything to report), and the two
1668
+ # spelling diagnostics `case_variants_seen` and
1669
+ # `normalisation_variants_seen`.
1670
+ TextResidue = Dict[str, Any]
1671
+
1672
+ _UNPLACED = object()
1673
+
1674
+
1675
+ def _residue_key(value: str) -> Tuple[str, str]:
1676
+ """The attribution key of `value`: its reader words, joined, casefolded.
1677
+
1678
+ `Mary Ann`, `MaryAnn`, `Mary_Ann` and `MARY ANN` all key to
1679
+ `maryann`, which is exactly the equivalence the detector's
1680
+ alternation encodes (`_detector_parts` splits on separators after
1681
+ NFKC and strips the unseen characters inside each word). The key
1682
+ JOINS the words: one that kept the word split disagreed with the
1683
+ capture-group reference in 792 of 5,479 random runs in the counts
1684
+ study, and this one in none, so the joined key is the right one. A
1685
+ value with no letters or digits keys by its literal reading, in a
1686
+ namespace of its own, so it never collides with a word key.
1687
+
1688
+ Since fix round 2 the words are keyed the way `re.IGNORECASE`
1689
+ compares them (`ignorecase_key`), which is the comparison the direct
1690
+ pattern made, rather than casefolded: the two disagree on a few
1691
+ letters (a capital I with a dot above, a dotless i, a long s), and a
1692
+ match the pattern accepted and the casefolded key could not place was
1693
+ charged to no entry. Where the interpreter lacks the functions that
1694
+ key is built from, casefolding stands in, as before.
1695
+ """
1696
+ parts = _detector_parts(value)
1697
+ if parts:
1698
+ return ("words", _word_key("".join(parts)))
1699
+ return ("literal", _word_key(_reader_sees(value)))
1700
+
1701
+
1702
+ _NOT_A_WORD_CHARACTER = re.compile(r"[\W_]+")
1703
+
1704
+
1705
+ def _direct_key(matched: str) -> Tuple[str, str]:
1706
+ """The attribution key of a match of the DIRECT pattern.
1707
+
1708
+ The direct pattern runs over the reader's reading, already normalised
1709
+ and stripped, and each alternative is a form's words escaped and
1710
+ joined by separator runs, so a match is those words, in any case the
1711
+ pattern accepts, with separators between. Keyed as it stands with the
1712
+ separators taken out, never normalised again: `_residue_key` of the
1713
+ match composed a mark the reading had left apart (an invisible
1714
+ character between a letter and a combining accent stops the
1715
+ composition when the text is read; the strip then removes it, and
1716
+ NFKC of the match composes the two), so the key matched no form and
1717
+ the occurrence was charged to no entry (the second re-verification's
1718
+ CORR2-2, "Mary" U+200B U+0301 "Ann" under {"Mary Ann"}; fix round 2
1719
+ fixed the same mechanism in the folded pass). The key of each form,
1720
+ `_residue_key`, is the same words joined, so this is its key.
1721
+ """
1722
+ words = _NOT_A_WORD_CHARACTER.sub("", matched)
1723
+ if words:
1724
+ return ("words", _word_key(words))
1725
+ return ("literal", _word_key(matched))
1726
+
1727
+
1728
+ def _folded_key(folded: str) -> Tuple[str, str]:
1729
+ """The attribution key of a match of the FOLDED pattern, and of the
1730
+ folded reading of a form.
1731
+
1732
+ The folded pattern is the forms' words casefolded, escaped and
1733
+ joined by separator runs, compiled without flags, so a match of it
1734
+ is those casefolded words exactly, with separators between. Keyed
1735
+ as it stands with the separators taken out, never normalised again:
1736
+ fix round 1 keyed it through `_residue_key`, whose NFKC composed a
1737
+ combining mark in the text onto the casefolded letter before it
1738
+ ("ß" followed by U+0301 folds to "ss" and then reads "sś"), so the
1739
+ key matched no form and the occurrence was charged to no entry (the
1740
+ re-verification's CORR-1; the property that asserted it never
1741
+ happens failed on about one random seed in twenty). A form's own
1742
+ folded words keyed the same way are the key of every match of its
1743
+ alternative, so this pass charges every match to an entry.
1744
+ """
1745
+ words = _NOT_A_WORD_CHARACTER.sub("", folded)
1746
+ if words:
1747
+ return ("words", words)
1748
+ return ("literal", folded)
1749
+
1750
+
1751
+ def _form_folded_key(form: str) -> Tuple[str, str]:
1752
+ """`_folded_key` of what a form's folded alternative matches."""
1753
+ parts = _detector_parts(form)
1754
+ if parts:
1755
+ return _folded_key("".join(_fold(part) for part in parts))
1756
+ return ("literal", _fold(_reader_sees(form)))
1757
+
1758
+
1759
+ class _TextLookup:
1760
+ """What the file-text count needs from one compiled mapping.
1761
+
1762
+ Built once per `Compiled`, on first use, from the forms in the
1763
+ pattern's own order (`_form_sort_key`: longest first). Each key maps
1764
+ to every form that shares it, in that order; `_pick` breaks a tie
1765
+ the way the combined pattern breaks it.
1766
+ """
1767
+
1768
+ __slots__ = ("direct", "folded", "reader_form", "all_ascii",
1769
+ "folded_form", "entry_forms", "rewriter_form",
1770
+ "seen_alone", "own_patterns")
1771
+
1772
+ def __init__(self, compiled: "Compiled"):
1773
+ self.direct: Dict[Tuple[str, str], List[Tuple[str, int, Any]]] = {}
1774
+ self.folded: Dict[Tuple[str, str], List[Tuple[str, int, Any]]] = {}
1775
+ self.reader_form: Dict[Tuple[int, str], str] = {}
1776
+ self.folded_form: Dict[Tuple[int, str], str] = {}
1777
+ self.entry_forms: Dict[int, List[str]] = {}
1778
+ # Per matched spelling, across every file of one preview (fix
1779
+ # round 2, B-3): the rewriter's form, and whether the detector
1780
+ # sees a whole word read with its trailing marks.
1781
+ self.rewriter_form: Dict[str, Tuple[int, str]] = {}
1782
+ self.seen_alone: Dict[str, bool] = {}
1783
+ # Each form's own two detector alternatives, direct and folded:
1784
+ # the cheap first question about a whole word the rewriter
1785
+ # matched, before the whole detector is asked.
1786
+ self.own_patterns: Dict[Tuple[int, str], Tuple[Any, Any]] = {}
1787
+ for form, index in compiled.forms:
1788
+ self.entry_forms.setdefault(index, []).append(form)
1789
+ self.folded_form.setdefault((index, _fold(form)), form)
1790
+ if not _detector_alternative(form):
1791
+ continue # a form no reader can see matches nothing
1792
+ # Each pattern built from the escaping builder at the site
1793
+ # that compiles it, as every pattern in this module is.
1794
+ direct = re.compile(_detector_alternative(form), re.IGNORECASE)
1795
+ folded = re.compile(_detector_alternative(form, fold=True))
1796
+ self.own_patterns[(index, form)] = (direct, folded)
1797
+ self.direct.setdefault(_residue_key(form), []).append(
1798
+ (form, index, direct))
1799
+ self.folded.setdefault(_form_folded_key(form), []).append(
1800
+ (form, index, folded))
1801
+ self.reader_form[(index, form)] = _reader_sees(form)
1802
+ self.all_ascii = all(form.isascii() for form, _ in compiled.forms)
1803
+
1804
+
1805
+ def _pick(candidates: Sequence[Tuple[str, int, Any]], matched: str
1806
+ ) -> Tuple[str, int, Any]:
1807
+ """The form the combined pattern chose for `matched`, among those
1808
+ sharing its key.
1809
+
1810
+ The alternation takes the first branch that matches at a position,
1811
+ so the first form, in the pattern's own order, whose alternative
1812
+ matches the whole of `matched` is the one it chose. One candidate
1813
+ is the common case and needs no regex at all.
1814
+ """
1815
+ if len(candidates) == 1:
1816
+ return candidates[0]
1817
+ for candidate in candidates:
1818
+ if candidate[2].fullmatch(matched):
1819
+ return candidate
1820
+ return candidates[0]
1821
+
1822
+
1823
+ def _attributed(matches, lookup_table, key=_direct_key):
1824
+ """Each match of a detector pattern, with the form it is charged to.
1825
+
1826
+ Yields `(match, (form, entry) or None, charge)`, `charge` being what
1827
+ the match budget is charged for it: one match for a spelling placed
1828
+ before, and `RESIDUE_FIRST_SIGHT_MATCHES` plus one for each further
1829
+ candidate sharing its key for the first sight of a spelling, which
1830
+ is what placing it costs (fix round 2, B-3). The entry is recovered
1831
+ from the matched TEXT by key, never from the regex engine: one
1832
+ capture group per alternative made the count cost 82 seconds per
1833
+ 1.27 MB at the documented ceiling of 2,000 forms, which is the shape
1834
+ of the uncapped-form defect `MAX_FORM_CHARS` exists for. A match
1835
+ the lookup cannot place is yielded with None: it is still counted,
1836
+ and charged to no entry. Since fix round 3 both passes key a match
1837
+ as it stands (`_direct_key`, `_folded_key`) and none is known to
1838
+ reach None; the two causes found before, `re.IGNORECASE` against
1839
+ `str.casefold` (fix round 2) and a match normalised again (CORR-1 in
1840
+ the folded pass, CORR2-2 in the direct one), are gone. It stays
1841
+ counted if one is ever found.
1842
+ """
1843
+ placed: Dict[str, Any] = {}
1844
+ for match in matches:
1845
+ matched = match.group(0)
1846
+ candidate = placed.get(matched, _UNPLACED)
1847
+ charge = 1
1848
+ if candidate is _UNPLACED:
1849
+ candidates = lookup_table.get(key(matched))
1850
+ candidate = (_pick(candidates, matched)[:2] if candidates
1851
+ else None)
1852
+ placed[matched] = candidate
1853
+ charge = RESIDUE_FIRST_SIGHT_MATCHES + (
1854
+ len(candidates) - 1 if candidates else 0)
1855
+ yield match, candidate, charge
1856
+
1857
+
1858
+ def direct_attribution(compiled: "Compiled", seen: str
1859
+ ) -> Tuple[Dict[Tuple[int, str], int], int]:
1860
+ """The wide reading's direct pass over `seen`, charged by form.
1861
+
1862
+ One UNGROUPED `finditer` of the detector's direct pattern over the
1863
+ reader's text, the entry recovered by key (`_attributed`). Returns
1864
+ ({(entry, form): count}, unattributed). The counterpart the tests
1865
+ compare it with, one capture group per alternative, lives in the
1866
+ test file and never here.
1867
+ """
1868
+ lookup = compiled.text_lookup()
1869
+ counts: Dict[Tuple[int, str], int] = {}
1870
+ unattributed = 0
1871
+ for _match, candidate, _charge in _attributed(
1872
+ compiled.detector._direct.finditer(seen), lookup.direct):
1873
+ if candidate is None:
1874
+ unattributed += 1
1875
+ continue
1876
+ key = (candidate[1], candidate[0])
1877
+ counts[key] = counts.get(key, 0) + 1
1878
+ return counts, unattributed
1879
+
1880
+
1881
+ def _rewriter_form(compiled: "Compiled", lookup: _TextLookup, matched: str
1882
+ ) -> Tuple[int, str]:
1883
+ """(entry, form) for one match of the REWRITER's pattern.
1884
+
1885
+ The entry is `Compiled.entry_for`'s, which is exact by construction;
1886
+ the form is the mapping's own spelling the match stands for, looked
1887
+ up exactly, then case-folded, then by the same full match
1888
+ `entry_for` falls back to, against the patterns `Compiled` holds
1889
+ compiled. Decided once per spelling (fix round 2, B-3).
1890
+ """
1891
+ found = lookup.rewriter_form.get(matched)
1892
+ if found is not None:
1893
+ return found
1894
+ index = compiled.entry_for(matched)
1895
+ if compiled._exact.get(matched) == index:
1896
+ found = (index, matched)
1897
+ else:
1898
+ form = lookup.folded_form.get((index, _fold(matched)))
1899
+ if form is None:
1900
+ form = next((candidate for candidate in lookup.entry_forms[index]
1901
+ if compiled.form_pattern(candidate).fullmatch(
1902
+ matched)),
1903
+ compiled.mapping.entries[index].original)
1904
+ found = (index, form)
1905
+ lookup.rewriter_form[matched] = found
1906
+ return found
1907
+
1908
+
1909
+ # Every combining mark (categories Mn, Mc and Me of the running
1910
+ # interpreter), as one pattern for a run of them, built on first use.
1911
+ _MARK_RUN_PATTERN: Optional["re.Pattern[str]"] = None
1912
+
1913
+
1914
+ def _mark_run_pattern() -> "re.Pattern[str]":
1915
+ """A pattern for a run of zero or more combining marks, built once."""
1916
+ global _MARK_RUN_PATTERN
1917
+ pattern = _MARK_RUN_PATTERN
1918
+ if pattern is None:
1919
+ if _MARK_RANGES is None:
1920
+ _sweep_code_points()
1921
+ ranges = _MARK_RANGES
1922
+ pattern = _MARK_RUN_PATTERN = re.compile(
1923
+ "[" + "".join(
1924
+ re.escape(chr(low)) if low == high
1925
+ else re.escape(chr(low)) + "-" + re.escape(chr(high))
1926
+ for low, high in ranges) + "]*")
1927
+ return pattern
1928
+
1929
+
1930
+ def _with_trailing_marks(text: str, start: int, end: int) -> str:
1931
+ """A whole-word match with the combining marks that follow it.
1932
+
1933
+ The rewriter's `(?!\\w)` treats a combining mark as a boundary, so
1934
+ "Rene" followed by U+0301 is a whole word to it; a reader, and NFKC,
1935
+ join the mark to the letter and read "René". Reading the match
1936
+ together with its trailing marks is how the detector is asked about
1937
+ the occurrence as a reader sees it.
1938
+
1939
+ The WHOLE run of marks is read (fix round 2, the re-verification's
1940
+ CORR-2): fix round 1 read sixteen, and a seventeenth that composes
1941
+ onto the name's last letter across sixteen that do not (a mark of a
1942
+ lower combining class) brought QA-1's reading back. A run belongs to
1943
+ the one match before it, so reading it whole keeps the cost linear in
1944
+ the text; it is found with one compiled pattern, and no mark comes
1945
+ before U+0300, so an ASCII text never builds it.
1946
+ """
1947
+ if end < len(text) and text[end] >= "̀":
1948
+ end = _mark_run_pattern().match(text, end).end()
1949
+ return text[start:end]
1950
+
1951
+
1952
+ def _reading_extra_pattern() -> "re.Pattern[str]":
1953
+ """The code points of `_READING_EXTRA` as one character class, built
1954
+ once, as `_unseen_pattern` builds its own."""
1955
+ global _READING_EXTRA_PATTERN
1956
+ pattern = _READING_EXTRA_PATTERN
1957
+ if pattern is None:
1958
+ if _READING_EXTRA is None:
1959
+ _sweep_code_points()
1960
+ ranges: List[List[int]] = []
1961
+ for code_point in sorted(_READING_EXTRA or {}):
1962
+ _add_to_ranges(ranges, code_point)
1963
+ pattern = _READING_EXTRA_PATTERN = re.compile(
1964
+ "[" + "".join(
1965
+ re.escape(chr(low)) if low == high
1966
+ else re.escape(chr(low)) + "-" + re.escape(chr(high))
1967
+ for low, high in ranges) + "]")
1968
+ return pattern
1969
+
1970
+
1971
+ def reading_length(text: str, stop_past_work: Optional[int] = None) -> int:
1972
+ """The length the count is priced at: its reading, not its store.
1973
+
1974
+ The count and the cheap question read `_reader_sees(text)` (NFKC) and
1975
+ its casefolded form, and one character's NFKC can be eighteen (U+FDFA,
1976
+ the Arabic ligature of the honorific), so the stored length
1977
+ under-prices such a text (fix round 6, the fourth re-verification's
1978
+ B4-1). Returned: an upper bound on the length of both readings, and
1979
+ never less than the stored length, which the normalisation itself
1980
+ reads. Linear, and never normalising (the normalisation of U+FDFA
1981
+ costs what the count does, which is what the estimate must not):
1982
+ ASCII text reads as it stands; text already in NFKC reads no longer
1983
+ than its casefolded form (the reading only removes invisible
1984
+ characters, and casefolding maps each character on its own), which
1985
+ is measured exactly; any other text is bounded character by
1986
+ character, each adding what `_READING_EXTRA` says (its NFKC,
1987
+ casefolded, less one). The sum bounds the reading because composing
1988
+ only shortens a text, and a composition that folds longer than its
1989
+ parts is charged to the letter it is built on; both are checked on
1990
+ every code point and every composition of the interpreter's tables,
1991
+ and on random texts, in the tests.
1992
+
1993
+ With `stop_past_work`, the sum stops once the text would cost more
1994
+ than that under a mapping of one form, the least any mapping costs:
1995
+ the caller needs no more to know the file is too large for any
1996
+ mapping, and a text of U+FDFA is not read to its end for it.
1997
+ """
1998
+ if text.isascii():
1999
+ return len(text)
2000
+ if unicodedata.is_normalized("NFKC", text):
2001
+ return len(text.casefold())
2002
+ pattern = _reading_extra_pattern()
2003
+ extra = _READING_EXTRA or {}
2004
+ length = len(text)
2005
+ limit = (None if stop_past_work is None else
2006
+ stop_past_work // (1 + RESIDUE_WORK_PER_CHARACTER_NON_ASCII))
2007
+ for start in range(0, len(text), _READING_CHUNK):
2008
+ length += sum(extra[ord(char)] for char in pattern.findall(
2009
+ text, start, start + _READING_CHUNK))
2010
+ if limit is not None and length > limit:
2011
+ break
2012
+ return length
2013
+
2014
+
2015
+ # The stretch `reading_length` sums at a time between two looks at its
2016
+ # limit: small beside a file, large beside the pattern's own start-up.
2017
+ _READING_CHUNK = 65536
2018
+
2019
+
2020
+ def residue_work(compiled: "Compiled", text: str,
2021
+ length: Optional[int] = None) -> int:
2022
+ """What counting (or checking) `text` costs, in the budgets' units.
2023
+
2024
+ `reading_length(text) * (the forms' units + the per-character term
2025
+ of the text's class)`: `RESIDUE_WORK_PER_CHARACTER` when the text is
2026
+ ASCII, `RESIDUE_WORK_PER_CHARACTER_NON_ASCII` when it is not (fix
2027
+ round 2, B-1). Since fix round 6 the length is the reading's (B4-1),
2028
+ and `length` passes it when the caller has it already, and the forms
2029
+ are priced by their length, `Compiled.form_units` (B4-2): a unit for
2030
+ each ten characters of a form, so a name is one unit, as before. The same units
2031
+ price the count, the cheap question and the whole detector asked
2032
+ about one word.
2033
+ """
2034
+ term = (RESIDUE_WORK_PER_CHARACTER if text.isascii()
2035
+ else RESIDUE_WORK_PER_CHARACTER_NON_ASCII)
2036
+ if length is None:
2037
+ length = reading_length(text)
2038
+ return length * (compiled.form_units + term)
2039
+
2040
+
2041
+ def residue_work_at_one_form(text: str,
2042
+ length: Optional[int] = None) -> int:
2043
+ """`residue_work` of `text` under a mapping of one surface form, the
2044
+ least any mapping costs: a file whose work passes the budget even so
2045
+ cannot be counted in a preview with any mapping, and fewer names
2046
+ cannot help it (the lead's follow-on to fix round 5)."""
2047
+ term = (RESIDUE_WORK_PER_CHARACTER if text.isascii()
2048
+ else RESIDUE_WORK_PER_CHARACTER_NON_ASCII)
2049
+ if length is None:
2050
+ length = reading_length(text)
2051
+ return length * (1 + term)
2052
+
2053
+
2054
+ def in_a_no_separator_script(char: str) -> bool:
2055
+ """Whether `char` belongs to one of `NO_SEPARATOR_RANGES`."""
2056
+ code_point = ord(char)
2057
+ index = bisect_right(_NO_SEPARATOR_STARTS, code_point) - 1
2058
+ return index >= 0 and code_point <= NO_SEPARATOR_RANGES[index][1]
2059
+
2060
+
2061
+ def _longer_word(seen: str, start: int, end: int) -> Optional[str]:
2062
+ """The run of word characters around a match, as the reader sees it,
2063
+ when it is a word.
2064
+
2065
+ `Thomas_P01` is one word and `Thomas_Smith.txt` gives
2066
+ `Thomas_Smith`. None, and the occurrence is counted and not listed,
2067
+ when the run extends the name by more than
2068
+ `MAX_LONGER_WORD_EXTENSION` characters, or when what extends it
2069
+ carries a character of a script written without separators
2070
+ (`NO_SEPARATOR_RANGES`), where the run is a clause. The walk stops
2071
+ one character past the bound on each side, so a match inside a long
2072
+ run costs a fixed handful of steps, never a walk of the run.
2073
+ """
2074
+ budget = MAX_LONGER_WORD_EXTENSION
2075
+ left = start
2076
+ while left > 0 and _is_word_char(seen[left - 1]):
2077
+ left -= 1
2078
+ budget -= 1
2079
+ if budget < 0:
2080
+ return None
2081
+ right = end
2082
+ while right < len(seen) and _is_word_char(seen[right]):
2083
+ right += 1
2084
+ budget -= 1
2085
+ if budget < 0:
2086
+ return None
2087
+ extension = seen[left:start] + seen[end:right]
2088
+ if any(in_a_no_separator_script(char) for char in extension):
2089
+ return None
2090
+ return seen[left:right]
2091
+
2092
+
2093
+ def names_left_in_text(compiled: "Compiled", text: str,
2094
+ list_longer_words: bool,
2095
+ rewritten_by_this_run: bool,
2096
+ max_matches: Optional[int] = None,
2097
+ max_extra_work: Optional[int] = None,
2098
+ stopped: Optional[Dict[str, Any]] = None
2099
+ ) -> Optional[TextResidue]:
2100
+ """Where this mapping's names are left in one file's text.
2101
+
2102
+ `text` is the file as it will be AFTER the run: the rewritten text
2103
+ for the one file the run rewrites, the stored text for every other.
2104
+ Counts, never spans and never surrounding text; with
2105
+ `list_longer_words`, the longer words a name sits inside (the typed
2106
+ path only, decided by the caller).
2107
+
2108
+ Two readings of every count, in one object `{"wide", "whole_word"}`:
2109
+
2110
+ - wide: the detector's reading of the reader's text, and beside it
2111
+ every whole word the rewriter matches that the reader's reading
2112
+ does not see (a name followed by a combining mark, which NFKC
2113
+ composes onto its last letter): an occurrence counts in the wide
2114
+ reading when either matcher finds it. Every wide occurrence is
2115
+ charged to exactly one kind (`TEXT_RESIDUE_REASONS`) or to
2116
+ `unattributed`, so the kinds and `unattributed` add up to the wide
2117
+ count on every row; the split is a heuristic and the total is
2118
+ not.
2119
+ - whole_word: the rewriter's own pattern over the RAW text, reported
2120
+ as it stands. On the file this run rewrote it is normally zero,
2121
+ and non-zero means a pseudonym put a name back.
2122
+
2123
+ The kinds, for each wide match in the reader's text: a word
2124
+ character touches either end (inside_a_longer_word); otherwise the
2125
+ match casefolds differently from the form as the reader sees it
2126
+ (joined_differently); otherwise it differs from that form and the
2127
+ case mode is exact (case_only); otherwise it IS the form, whole
2128
+ word, and the rewriter left it. That last class has two causes,
2129
+ split per form by the whole-word count of the raw text: on the row
2130
+ of the file this run rewrote, `min(whole_word, class)` is charged to
2131
+ put_back_by_a_pseudonym and the rest to normalisation_variants; on
2132
+ every other row nothing was put back by this run, so the same share
2133
+ is charged to whole_word_in_a_file_not_rewritten instead. The `min`
2134
+ and `max` are a declared HEURISTIC about which cause an occurrence
2135
+ is charged to when both are present; their sum is exact.
2136
+
2137
+ `max_matches` bounds the work a hostile text can cost: every match of
2138
+ the three passes counts against it, the first sight of each spelling
2139
+ in a pass as `RESIDUE_FIRST_SIGHT_MATCHES` (placing a new spelling
2140
+ costs several matches; a spelling met before is looked up; fix round
2141
+ 2), and past it the count stops and the function returns None, which the
2142
+ caller reads as "not counted in full" (the match budget, fix round
2143
+ 1, S-4). The result's `matches` is what the count spent, in those
2144
+ units.
2145
+
2146
+ `max_extra_work` bounds the one cost the caller's work model does not
2147
+ see (fix round 2): asking the whole detector about a whole word the
2148
+ rewriter matched and the word's own alternatives do not find in its
2149
+ reading (a name before a composing mark). Such a question costs what
2150
+ the count costs per character of the word and its marks, and is
2151
+ asked once per distinct spelling in a preview, so a text written to
2152
+ make every spelling distinct could otherwise cost the count's time
2153
+ again. Each such question is charged `residue_work` of the word and
2154
+ its marks, and one match; past the allowance the
2155
+ function returns None, as past the match budget. The result's
2156
+ `extra_work` is what it charged. When the count stops, `stopped` (a
2157
+ dict, if given) is told what the count had spent, so the caller can
2158
+ charge it (fix round 5, the lead's rule 1: everything spent is
2159
+ charged): `cause` ("matches" or "work", the allowance it passed),
2160
+ `matches` and `extra_work` as the result would give them, and
2161
+ `at_least`, how many occurrences the count had found by then (rule
2162
+ 3). Every stop comes on a match or on a question about one, so
2163
+ `at_least` is 1 or more and the file certainly shows a name. It is a
2164
+ lower bound on the larger of the two readings: the wide count is
2165
+ exactly the larger of the direct and folded passes' matches plus the
2166
+ whole words only the rewriter sees, so each pass's matches so far
2167
+ bound it, and the whole words so far bound the whole-word count.
2168
+ """
2169
+ lookup = compiled.text_lookup()
2170
+ spent = 0
2171
+ limit = max_matches if max_matches is not None else -1
2172
+ extra_work = 0
2173
+ extra_limit = max_extra_work if max_extra_work is not None else -1
2174
+ seen = _reader_sees(text)
2175
+ rows: Dict[Tuple[int, str], Dict[str, Any]] = {}
2176
+ direct_total = 0
2177
+ folded_seen = 0
2178
+ whole_word_total = 0
2179
+ only_seen = 0
2180
+
2181
+ def stop(cause: str) -> None:
2182
+ if stopped is not None:
2183
+ stopped.update(
2184
+ cause=cause, matches=spent, extra_work=extra_work,
2185
+ at_least=max(max(direct_total, folded_seen) + only_seen,
2186
+ whole_word_total))
2187
+
2188
+ def row_for(key: Tuple[int, str]) -> Dict[str, Any]:
2189
+ row = rows.get(key)
2190
+ if row is None:
2191
+ row = rows[key] = {"direct": 0, "inside_a_longer_word": 0,
2192
+ "case_only": 0, "joined_differently": 0,
2193
+ "left_whole": 0, "folded_excess": 0,
2194
+ "whole_word": 0, "rewriter_only": 0,
2195
+ "words": {}, "words_withheld": False}
2196
+ return row
2197
+
2198
+ unattributed = 0
2199
+ exact_mode = compiled.case_mode == "exact"
2200
+ for match, candidate, charge in _attributed(
2201
+ compiled.detector._direct.finditer(seen), lookup.direct):
2202
+ direct_total += 1
2203
+ spent += charge
2204
+ if spent > limit >= 0:
2205
+ stop("matches")
2206
+ return None
2207
+ if candidate is None:
2208
+ unattributed += 1
2209
+ continue
2210
+ form, index = candidate
2211
+ row = row_for((index, form))
2212
+ row["direct"] += 1
2213
+ matched = match.group(0)
2214
+ start, end = match.span()
2215
+ if ((start > 0 and _is_word_char(seen[start - 1]))
2216
+ or (end < len(seen) and _is_word_char(seen[end]))):
2217
+ row["inside_a_longer_word"] += 1
2218
+ if list_longer_words:
2219
+ word = _longer_word(seen, start, end)
2220
+ if word is None:
2221
+ row["words_withheld"] = True
2222
+ else:
2223
+ row["words"][word] = row["words"].get(word, 0) + 1
2224
+ continue
2225
+ reader_form = lookup.reader_form[(index, form)]
2226
+ # Compared the way the pattern compared them (fix round 2): a
2227
+ # capital I with a dot above is the same letter to it, and to
2228
+ # casefolding two code points, which read as joined differently.
2229
+ if _word_key(matched) != _word_key(reader_form):
2230
+ row["joined_differently"] += 1
2231
+ elif matched != reader_form and exact_mode:
2232
+ row["case_only"] += 1
2233
+ else:
2234
+ row["left_whole"] += 1
2235
+
2236
+ # The casefolded second reading, which `NameDetector.contains` makes
2237
+ # too: `re.IGNORECASE` and `str.casefold` disagree on some code
2238
+ # points (U+00DF casefolds to "ss" and does not match "SS" under
2239
+ # IGNORECASE), so a spelling only case-folding reaches is missed by
2240
+ # the direct pass above. Where the folded pass finds more for a form
2241
+ # than the direct pass did, the excess is added to that form's wide
2242
+ # count and charged to normalisation_variants. This is a HEURISTIC,
2243
+ # declared as one: a spelling only case-folding reaches is one the
2244
+ # rewriter cannot see under any case mode, so it is charged there,
2245
+ # and it is what keeps `wide` from being zero on a file where
2246
+ # `detector.contains` says a name shows. The excess is capped at the
2247
+ # folded pass's net surplus over the direct one, so an occurrence the
2248
+ # direct pass could not place (dotted capital I, which IGNORECASE
2249
+ # matches and casefolding spells as two code points) and the folded
2250
+ # pass could is not counted twice. On ASCII text with ASCII forms
2251
+ # the two passes cannot disagree, so the second is skipped.
2252
+ if not (seen.isascii() and lookup.all_ascii):
2253
+ folded_counts: Dict[Tuple[int, str], int] = {}
2254
+ folded_unattributed = 0
2255
+ for _match, candidate, charge in _attributed(
2256
+ compiled.detector._folded.finditer(_fold(seen)),
2257
+ lookup.folded, key=_folded_key):
2258
+ folded_seen += 1
2259
+ spent += charge
2260
+ if spent > limit >= 0:
2261
+ stop("matches")
2262
+ return None
2263
+ if candidate is None:
2264
+ folded_unattributed += 1
2265
+ continue
2266
+ key = (candidate[1], candidate[0])
2267
+ folded_counts[key] = folded_counts.get(key, 0) + 1
2268
+ surplus = (sum(folded_counts.values()) + folded_unattributed
2269
+ - direct_total)
2270
+ for form, index in compiled.forms:
2271
+ if surplus <= 0:
2272
+ break
2273
+ key = (index, form)
2274
+ excess = folded_counts.get(key, 0) - (
2275
+ rows[key]["direct"] if key in rows else 0)
2276
+ if excess > 0:
2277
+ excess = min(excess, surplus)
2278
+ row_for(key)["folded_excess"] += excess
2279
+ surplus -= excess
2280
+ if surplus > 0:
2281
+ unattributed += min(surplus, max(
2282
+ 0, folded_unattributed - unattributed))
2283
+
2284
+ # The whole-word half, over the RAW text. A whole word the rewriter
2285
+ # matches that the detector does not see, read with the combining
2286
+ # marks after it as a reader reads it, is an occurrence of the wide
2287
+ # reading too (the union; lead's ruling on QA-1), charged below to
2288
+ # the whole-word kind of its row. Asked once per distinct spelling in
2289
+ # the preview: first of the form's own two alternatives, which find
2290
+ # it in all but the composing-mark case, then, only when they do not,
2291
+ # of the whole detector, whose cost is charged (`max_extra_work`).
2292
+ detector_sees = lookup.seen_alone
2293
+ for match in compiled.pattern.finditer(text):
2294
+ whole_word_total += 1
2295
+ matched = match.group(0)
2296
+ spent += (1 if matched in lookup.rewriter_form
2297
+ else RESIDUE_FIRST_SIGHT_MATCHES)
2298
+ if spent > limit >= 0:
2299
+ stop("matches")
2300
+ return None
2301
+ key = _rewriter_form(compiled, lookup, matched)
2302
+ row = row_for(key)
2303
+ row["whole_word"] += 1
2304
+ snippet = _with_trailing_marks(text, match.start(), match.end())
2305
+ seen_alone = detector_sees.get(snippet)
2306
+ if seen_alone is None:
2307
+ reading = _reader_sees(snippet)
2308
+ own = lookup.own_patterns.get(key)
2309
+ seen_alone = bool(own and (own[0].search(reading)
2310
+ or own[1].search(_fold(reading))))
2311
+ if not seen_alone:
2312
+ spent += 1
2313
+ extra_work += residue_work(compiled, snippet)
2314
+ if spent > limit >= 0 or (
2315
+ extra_limit >= 0 and extra_work > extra_limit):
2316
+ stop("matches" if spent > limit >= 0 else "work")
2317
+ return None
2318
+ seen_alone = compiled.detector.contains(snippet)
2319
+ detector_sees[snippet] = seen_alone
2320
+ if not seen_alone:
2321
+ row["rewriter_only"] += 1
2322
+ only_seen += 1
2323
+
2324
+ entries: Dict[int, Dict[str, Any]] = {}
2325
+ case_variants: List[Dict[str, Any]] = []
2326
+ normalisation_variants: List[Dict[str, Any]] = []
2327
+ wide_total = unattributed
2328
+ for form, index in compiled.forms:
2329
+ row = rows.get((index, form))
2330
+ if row is None:
2331
+ continue
2332
+ # The heuristic split of the class the rewriter left whole, per
2333
+ # form (see the docstring); the sum of the two is exact. The
2334
+ # whole words only the rewriter sees are whole words of that
2335
+ # kind outright, and are added to the wide count.
2336
+ only = row["rewriter_only"]
2337
+ seen_whole = row["whole_word"] - only
2338
+ shared = min(seen_whole, row["left_whole"]) + only
2339
+ spelling = (max(0, row["left_whole"] - seen_whole)
2340
+ + row["folded_excess"])
2341
+ wide = row["direct"] + row["folded_excess"] + only
2342
+ wide_total += wide
2343
+ entry = entries.get(index)
2344
+ if entry is None:
2345
+ entry = entries[index] = {
2346
+ "entry": index,
2347
+ "form": compiled.mapping.entries[index].original,
2348
+ "occurrences": {"wide": 0, "whole_word": 0},
2349
+ "inside_a_longer_word": 0, "case_only": 0,
2350
+ "joined_differently": 0, "normalisation_variants": 0,
2351
+ "put_back_by_a_pseudonym": 0,
2352
+ "whole_word_in_a_file_not_rewritten": 0,
2353
+ "_words": {}, "_withheld": False}
2354
+ entry["occurrences"]["wide"] += wide
2355
+ entry["occurrences"]["whole_word"] += row["whole_word"]
2356
+ entry["inside_a_longer_word"] += row["inside_a_longer_word"]
2357
+ entry["case_only"] += row["case_only"]
2358
+ entry["joined_differently"] += row["joined_differently"]
2359
+ entry["normalisation_variants"] += spelling
2360
+ if rewritten_by_this_run:
2361
+ entry["put_back_by_a_pseudonym"] += shared
2362
+ else:
2363
+ entry["whole_word_in_a_file_not_rewritten"] += shared
2364
+ for word, count in row["words"].items():
2365
+ entry["_words"][word] = entry["_words"].get(word, 0) + count
2366
+ entry["_withheld"] = entry["_withheld"] or row["words_withheld"]
2367
+ if row["case_only"]:
2368
+ case_variants.append({"entry": index, "form": form,
2369
+ "other_case_count": row["case_only"]})
2370
+ if spelling:
2371
+ normalisation_variants.append({"entry": index, "form": form,
2372
+ "count": spelling})
2373
+
2374
+ listed: List[Dict[str, Any]] = []
2375
+ # In wide-count order, the entry index breaking ties, so a cap on the
2376
+ # rows (the caller's) keeps the entries with the most left.
2377
+ for index in sorted(entries, key=lambda i: (
2378
+ -entries[i]["occurrences"]["wide"],
2379
+ -entries[i]["occurrences"]["whole_word"], i)):
2380
+ entry = entries[index]
2381
+ words = entry.pop("_words")
2382
+ withheld = entry.pop("_withheld")
2383
+ if not (entry["occurrences"]["wide"]
2384
+ or entry["occurrences"]["whole_word"]):
2385
+ continue
2386
+ if list_longer_words:
2387
+ ranked = sorted(words.items(), key=lambda kv: (-kv[1], kv[0]))
2388
+ entry["longer_words"] = [
2389
+ {"word": word, "count": count}
2390
+ for word, count in ranked[:MAX_LONGER_WORDS_PER_ENTRY]]
2391
+ if withheld or len(ranked) > MAX_LONGER_WORDS_PER_ENTRY:
2392
+ entry["longer_words_truncated"] = True
2393
+ listed.append(entry)
2394
+ return {"occurrences": {"wide": wide_total,
2395
+ "whole_word": whole_word_total},
2396
+ "unattributed": unattributed,
2397
+ "entries": listed,
2398
+ "case_variants_seen": case_variants,
2399
+ "normalisation_variants_seen": normalisation_variants,
2400
+ "matches": spent,
2401
+ "extra_work": extra_work}
2402
+
2403
+
2404
+ # The owner's ruling of 2026-09-23 on Brief 1's finding F-1: a pseudonym
2405
+ # that contains a name from the mapping is withheld, by the rule that
2406
+ # withholds a file name carrying one, from the run record and the journal
2407
+ # entry on both mapping paths and from the preview on the
2408
+ # `use_project_pseudonyms` path. The one sentence every such place says.
2409
+ PSEUDONYMS_WITHHELD_NOTE = (
2410
+ "A pseudonym that contains a name from this mapping is withheld here, "
2411
+ "by the rule that withholds a file name that carries one; its entry "
2412
+ "number identifies it. The run still writes it into the text.")
2413
+
2414
+
2415
+ def pseudonyms_withheld(compiled: "Compiled") -> Tuple[int, ...]:
2416
+ """The entries whose pseudonym carries a name from the mapping.
2417
+
2418
+ `Compiled.carries_a_name` over each pseudonym: the union of the two
2419
+ matchers, so "Alex Smith" (a whole word of `Smith`), "Thomas Jr" (its
2420
+ own original) and "Thomasina" (inside a longer word, which the static
2421
+ check of `pseudonyms_containing_a_name` cannot see) are all withheld.
2422
+ The price is the wide reading's: a short name withholds a pseudonym
2423
+ that merely contains its letters ("Ed" in "Fred"), and the entry
2424
+ number still identifies it.
2425
+ """
2426
+ return tuple(entry.index for entry in compiled.mapping.entries
2427
+ if compiled.carries_a_name(entry.pseudonym))
2428
+
2429
+
2430
+ def pseudonyms_containing_a_name(compiled: "Compiled"
2431
+ ) -> List[Dict[str, Any]]:
2432
+ """Pseudonyms the rewriter's own rule finds a name in (v0.13, item 5).
2433
+
2434
+ A static check, before any text is read: the rewriter's compiled
2435
+ pattern run over each pseudonym. `{"Smith": "Jones", "Thomas Smith":
2436
+ "Alex Smith"}` puts the real surname back wherever the second entry
2437
+ fires, and a pseudonym that contains its OWN original ("Thomas" to
2438
+ "Thomas Jr") does the same. What it cannot see, and says so where
2439
+ it is reported: a pseudonym that puts a name back inside a longer
2440
+ word (`xThomasx`), which no whole-word rule matches, and a
2441
+ pseudonym that forms a name with the words around it.
2442
+
2443
+ Each finding once per (entry, contained form): the entry whose
2444
+ pseudonym it is, the pseudonym, the entry it contains and that
2445
+ entry's form as the mapping spells it.
2446
+ """
2447
+ lookup = compiled.text_lookup()
2448
+ found: List[Dict[str, Any]] = []
2449
+ for entry in compiled.mapping.entries:
2450
+ reported = set()
2451
+ for match in compiled.pattern.finditer(entry.pseudonym):
2452
+ index, form = _rewriter_form(compiled, lookup, match.group(0))
2453
+ if (index, form) in reported:
2454
+ continue
2455
+ reported.add((index, form))
2456
+ found.append({"entry": entry.index,
2457
+ "pseudonym": entry.pseudonym,
2458
+ "contains_entry": index, "form": form})
2459
+ return found
2460
+
2461
+
2462
+ # --------------------------------------------------------------------------
2463
+ # The remap
2464
+ # --------------------------------------------------------------------------
2465
+
2466
+ class RemappedSpan:
2467
+ """What the run would do to one stored span."""
2468
+
2469
+ __slots__ = ("pos0", "pos1", "change", "clamped", "touched")
2470
+
2471
+ def __init__(self, pos0: Optional[int], pos1: Optional[int],
2472
+ change: str, clamped: bool, touched: bool):
2473
+ self.pos0 = pos0
2474
+ self.pos1 = pos1
2475
+ self.change = change
2476
+ self.clamped = clamped
2477
+ # Whether the old span met an edit at all. A row whose positions
2478
+ # do not move can still need its `seltext` refreshed, because a
2479
+ # pseudonym of the same length changes the text inside it.
2480
+ self.touched = touched
2481
+
2482
+ def __repr__(self) -> str: # pragma: no cover - debugging
2483
+ return (f"RemappedSpan({self.pos0}, {self.pos1}, {self.change}"
2484
+ f"{', clamped' if self.clamped else ''})")
2485
+
2486
+
2487
+ class SpanMapper:
2488
+ """Maps old offsets to new ones for one text and one edit list.
2489
+
2490
+ `snap_to_pseudonym` is arithmetic on a prefix sum and needs no walk;
2491
+ `qualcoder_edit_parity` walks the edits per row, which is the same
2492
+ semantics as upstream's walk over all rows per edit but the opposite
2493
+ iteration order, so the two are a real cross-check of each other.
2494
+ """
2495
+
2496
+ def __init__(self, replacements: Sequence[Replacement], old_len: int):
2497
+ self.old_len = old_len
2498
+ self.starts = [item.start for item in replacements]
2499
+ self.ends = [item.end for item in replacements]
2500
+ self.new_lengths = [len(item.text) for item in replacements]
2501
+ self.replacements = list(replacements)
2502
+ cumulative = [0]
2503
+ for item in replacements:
2504
+ cumulative.append(cumulative[-1] + item.delta)
2505
+ self.cumulative = cumulative
2506
+ self.new_len = old_len + cumulative[-1]
2507
+
2508
+ # -- snap_to_pseudonym ------------------------------------------------
2509
+
2510
+ def _edit_containing(self, position: int) -> Optional[int]:
2511
+ """The edit `position` falls strictly inside, or None."""
2512
+ index = bisect_right(self.ends, position)
2513
+ if index < len(self.starts) and self.starts[index] < position:
2514
+ return index
2515
+ return None
2516
+
2517
+ def _delta_before(self, position: int) -> int:
2518
+ return self.cumulative[bisect_right(self.ends, position)]
2519
+
2520
+ def map_start(self, position: int) -> int:
2521
+ """Where a span's START goes under `snap_to_pseudonym`.
2522
+
2523
+ Strictly inside a replaced name, the start snaps OUTWARD to the
2524
+ pseudonym's first character, so a coding that began in the middle
2525
+ of the name now begins at the pseudonym. Anywhere else it moves
2526
+ by the cumulative length change of the edits that finished before
2527
+ it.
2528
+ """
2529
+ inside = self._edit_containing(position)
2530
+ if inside is not None:
2531
+ return self.starts[inside] + self.cumulative[inside]
2532
+ return position + self._delta_before(position)
2533
+
2534
+ def map_end(self, position: int) -> int:
2535
+ """Where a span's END goes under `snap_to_pseudonym`.
2536
+
2537
+ The mirror of `map_start`: strictly inside a replaced name the
2538
+ end snaps outward to just past the pseudonym, so a coding that
2539
+ stopped in the middle of the name now contains the whole
2540
+ pseudonym. An end exactly at the name's end lands exactly at the
2541
+ pseudonym's end.
2542
+ """
2543
+ inside = self._edit_containing(position)
2544
+ if inside is not None:
2545
+ return (self.starts[inside] + self.new_lengths[inside]
2546
+ + self.cumulative[inside])
2547
+ return position + self._delta_before(position)
2548
+
2549
+ def _intersects_edit(self, pos0: int, pos1: int) -> bool:
2550
+ index = bisect_right(self.ends, pos0)
2551
+ return index < len(self.starts) and self.starts[index] < pos1
2552
+
2553
+ def _snap(self, pos0: int, pos1: int) -> RemappedSpan:
2554
+ cut = (self._edit_containing(pos0) is not None
2555
+ or self._edit_containing(pos1) is not None)
2556
+ new0 = self.map_start(pos0)
2557
+ new1 = self.map_end(pos1)
2558
+ return self._classify(pos0, pos1, new0, new1, cut)
2559
+
2560
+ # -- qualcoder_edit_parity --------------------------------------------
2561
+
2562
+ def _parity(self, pos0: int, pos1: int,
2563
+ keep_start_anchor: bool) -> RemappedSpan:
2564
+ """Master's `apply_delete` then `apply_insert`, per edit.
2565
+
2566
+ Transcribed from `code_text.py:5906-5927` and `:5893-5904` at pin
2567
+ 9bddf17 and driven by the running offset of `:5933-5950`: each
2568
+ replacement is the deletion of the name followed by the insertion
2569
+ of the pseudonym at the same evolving offset. The consequences
2570
+ are upstream's own and are the reason this is the OPTION and not
2571
+ the default: a span that sits exactly on a name is deleted, and a
2572
+ span that merely touches one is trimmed to exclude the pseudonym
2573
+ entirely.
2574
+ """
2575
+ original_pos0 = pos0
2576
+ new0: Optional[int] = pos0
2577
+ new1 = pos1
2578
+ shift = 0
2579
+ head_cut = False
2580
+ tail_cut = False
2581
+ for index, item in enumerate(self.replacements):
2582
+ at = item.start + shift
2583
+ length = item.end - item.start
2584
+ inserted = self.new_lengths[index]
2585
+ if new0 is not None:
2586
+ # apply_delete (code_text.py:5906-5927)
2587
+ if new0 >= at + length:
2588
+ new0 -= length
2589
+ new1 -= length
2590
+ elif new0 >= at:
2591
+ if new1 <= at + length:
2592
+ new0 = None # span fully inside the name
2593
+ else:
2594
+ # The span's head held the name, whether it began
2595
+ # inside it or exactly on it; either way the
2596
+ # pseudonym ends up outside the span.
2597
+ new0 = at
2598
+ new1 -= length
2599
+ head_cut = True
2600
+ elif new1 > at:
2601
+ # This branch also fires for a span that STRICTLY
2602
+ # CONTAINS the name, where nothing is cut at all: the
2603
+ # span shrinks by the name and grows by the pseudonym
2604
+ # through the insert below. Only an end at or inside
2605
+ # the name's end really loses the pseudonym, so only
2606
+ # that is recorded as a cut.
2607
+ if new1 <= at + length:
2608
+ tail_cut = True
2609
+ new1 -= min(new1, at + length) - at
2610
+ if new1 <= new0:
2611
+ new0 = None
2612
+ if new0 is not None:
2613
+ # apply_insert (code_text.py:5893-5904)
2614
+ if new0 >= at:
2615
+ new0 += inserted
2616
+ new1 += inserted
2617
+ if keep_start_anchor and original_pos0 == 0:
2618
+ new0 = 0
2619
+ elif new0 < at < new1:
2620
+ new1 += inserted
2621
+ shift += inserted - length
2622
+ if new0 is None:
2623
+ return RemappedSpan(None, None, DELETED, False, True)
2624
+ # A span anchored at the file start never moves its start, so a
2625
+ # head cut on it was undone in the same step and is not a cut.
2626
+ anchored = keep_start_anchor and original_pos0 == 0
2627
+ cut = tail_cut or (head_cut and not anchored)
2628
+ return self._classify(pos0, pos1, new0, new1, cut)
2629
+
2630
+ # -- shared ------------------------------------------------------------
2631
+
2632
+ def _substitutes(self, pos0: int, pos1: int, new0: int, new1: int
2633
+ ) -> bool:
2634
+ """Whether the old span WAS one replaced name and the new span
2635
+ IS that name's pseudonym, both boundaries on the edit's own.
2636
+
2637
+ Under `snap_to_pseudonym` the second half follows from the first
2638
+ by the arithmetic of `map_start` and `map_end`; under
2639
+ `qualcoder_edit_parity` such a span is deleted before it reaches
2640
+ `_classify`. Both halves are checked all the same, because the
2641
+ class is defined by both (owner ruling 7.3(3)) and a third
2642
+ policy would inherit the definition rather than the coincidence.
2643
+ """
2644
+ index = bisect_left(self.starts, pos0)
2645
+ if index >= len(self.starts) or self.starts[index] != pos0 \
2646
+ or self.ends[index] != pos1:
2647
+ return False
2648
+ start = self.starts[index] + self.cumulative[index]
2649
+ return (new0, new1) == (start, start + self.new_lengths[index])
2650
+
2651
+ def _classify(self, pos0: int, pos1: int, new0: int, new1: int,
2652
+ cut: bool) -> RemappedSpan:
2653
+ """One vocabulary for both policies.
2654
+
2655
+ `snapped` means the run had to MOVE a boundary because it met a
2656
+ replaced name: under `snap_to_pseudonym` a boundary that lay
2657
+ strictly inside a name snapped outward to contain the whole
2658
+ pseudonym; under `qualcoder_edit_parity` a head or tail cut
2659
+ excluded the pseudonym from the span. `substituted` is a span
2660
+ that sat exactly on one replaced name and now sits exactly on
2661
+ that name's pseudonym, whatever the two lengths: the coder
2662
+ marked the name and still marks it under its new spelling, so
2663
+ no coding decision changed (owner ruling 7.3(3) of 2026-09-15,
2664
+ refining X1; before it this span was classified by length, a
2665
+ shift when the lengths matched and a resize when they did not).
2666
+ `resized` is any other length change, which is what a span that
2667
+ contains a name without being the name gets when the pseudonym
2668
+ is a different length: "Mr Thomas said" grows or shrinks with the
2669
+ pseudonym and records the same decision about the same words, so
2670
+ ruling 7.4 of 2026-09-16 exempts it from the hidden-coder
2671
+ override as well. `shifted` is a span that kept its length and
2672
+ moved, which includes one containing a name whose pseudonym is
2673
+ exactly as long: the coder still marks the same passage.
2674
+
2675
+ The order matters. A cut is a snap before anything else, so no
2676
+ span that had to grow can read as a substitution. `clamped` is
2677
+ decided in `map_row`, which alone holds the flag, and it comes
2678
+ before every test here but the two whose classes are gated
2679
+ already.
2680
+ """
2681
+ touched = self._intersects_edit(pos0, pos1) or cut
2682
+ if cut:
2683
+ change = SNAPPED
2684
+ elif self._substitutes(pos0, pos1, new0, new1):
2685
+ change = SUBSTITUTED
2686
+ elif (new1 - new0) != (pos1 - pos0):
2687
+ change = RESIZED
2688
+ elif (new0, new1) != (pos0, pos1):
2689
+ change = SHIFTED
2690
+ else:
2691
+ change = UNCHANGED
2692
+ return RemappedSpan(new0, new1, change, False, touched)
2693
+
2694
+ def map_row(self, pos0: Any, pos1: Any, policy: str,
2695
+ keep_start_anchor: bool) -> Optional[RemappedSpan]:
2696
+ """Map one stored row, or None when the row is not mappable.
2697
+
2698
+ Returning None is the "leave it exactly as it is" answer for a
2699
+ row whose stored positions are not a span: NULL, not an integer,
2700
+ NEGATIVE, or `pos0 >= pos1`. None refuses the run; all are
2701
+ listed. A negative position is a damaged row QualCoder never
2702
+ writes, and mapping one would write it back negative and could
2703
+ collide with the parked values the write uses; left alone and
2704
+ reported, it is a row the researcher can go and look at (QA F-9).
2705
+
2706
+ A `pos1` past the end of the text is CLAMPED on the old side
2707
+ before mapping and reported, which is master's own rule applied a
2708
+ step earlier (`code_text.py:6232-6233`, "clamp, never delete a
2709
+ valid code"). Clamping before rather than after is what makes
2710
+ the guarantee that no row is written with an end past the new
2711
+ text hold under both policies rather than only under one.
2712
+
2713
+ A clamped row has a class of its own, `clamped`, which nothing
2714
+ else produces. The classes describe what happens to the STORED
2715
+ span, and a clamp truncates it: the annotation stored at
2716
+ (76, 200) on an 81-character text is written as (66, 71), which
2717
+ is a 119-character truncation however the rest of the run moved
2718
+ it. Classifying that as `shifted` put it under ruling X1's
2719
+ exemption and let a hidden coder's row be cut back with no
2720
+ override asked for (Security S5); classifying it as `resized`
2721
+ kept it gated until ruling 7.4 of 2026-09-16 exempted the
2722
+ ordinary resize, which is why the clamp now has a class instead
2723
+ of borrowing one. The same holds when the clamp lands the
2724
+ truncated span exactly on a name that ends the text: the coder
2725
+ never marked that name alone, so the row is `clamped` and not a
2726
+ substitution. Only a snap and a deletion keep their own class
2727
+ through a clamp, both being on the override side already; the
2728
+ test is written that way round so that a class added later
2729
+ stays gated rather than exempt by omission.
2730
+ """
2731
+ if isinstance(pos0, bool) or isinstance(pos1, bool):
2732
+ return None
2733
+ if not isinstance(pos0, int) or not isinstance(pos1, int):
2734
+ return None
2735
+ if pos0 < 0 or pos1 < 0:
2736
+ return None
2737
+ clamped = False
2738
+ if pos1 > self.old_len:
2739
+ pos1 = self.old_len
2740
+ clamped = True
2741
+ if pos0 >= pos1:
2742
+ return None
2743
+ if policy == "qualcoder_edit_parity":
2744
+ mapped = self._parity(pos0, pos1, keep_start_anchor)
2745
+ elif policy == "snap_to_pseudonym":
2746
+ mapped = self._snap(pos0, pos1)
2747
+ else:
2748
+ raise ValueError(
2749
+ f"overlap_policy must be one of "
2750
+ f"{', '.join(OVERLAP_POLICIES)}.")
2751
+ mapped.clamped = clamped
2752
+ if clamped and mapped.change not in (SNAPPED, DELETED):
2753
+ mapped.change = CLAMPED
2754
+ return mapped
2755
+
2756
+
2757
+ def unique_constraint_collisions(keys: Sequence[Tuple[Any, ...]],
2758
+ row_ids: Sequence[Any],
2759
+ shown=None) -> List[Dict[str, Any]]:
2760
+ """Rows that would land on the same unique key after the remap.
2761
+
2762
+ `code_text` is unique on (cid, fid, pos0, pos1, owner) and
2763
+ `annotation` on (fid, pos0, pos1, owner) in QualCoder's own schema
2764
+ (`__main__.py:1819-1821` and `:1800-1801` at master, in that order,
2765
+ identical at the 3.8.2 tag). Two rows can collapse onto one span when both were cut by
2766
+ the same name: one marking "Thomas" and one marking "Thom" both snap
2767
+ to the pseudonym. Rare enough that the right answer is a human
2768
+ decision, so the preview lists it and the execute refuses.
2769
+
2770
+ Rows are GROUPED on the whole key, because that is the constraint,
2771
+ and REPORTED under `shown(key)` when a caller gives one. The
2772
+ database layer uses it to leave the owner column out of what the
2773
+ preview carries: the key's owner can be a coder the project hides,
2774
+ and two rows of one hidden coder cut by one name is an ordinary
2775
+ shape (one marking "Thomas", one marking "Thom"), so the owner in
2776
+ the reported key named a hidden coder in a preview whose every
2777
+ other field withholds the name (X1). `row_ids` identify the rows
2778
+ on their own.
2779
+ """
2780
+ groups: Dict[Tuple[Any, ...], List[Any]] = {}
2781
+ for key, row_id in zip(keys, row_ids):
2782
+ groups.setdefault(key, []).append(row_id)
2783
+ return [{"key": list(key if shown is None else shown(key)),
2784
+ "row_ids": sorted(ids)}
2785
+ for key, ids in sorted(groups.items(), key=lambda kv: str(kv[0]))
2786
+ if len(ids) > 1]
2787
+
2788
+
2789
+ def context_for(text: str, start: int, end: int, chars: int) -> str:
2790
+ """The text around one match, for `include_context`.
2791
+
2792
+ This returns FILE CONTENT, which is why the argument is off by
2793
+ default and why the tool description says so in as many words.
2794
+ """
2795
+ chars = max(0, min(int(chars), MAX_CONTEXT_CHARS))
2796
+ return text[max(0, start - chars):min(len(text), end + chars)]