exegete 0.14.1a0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exegete/__init__.py +16 -0
- exegete/coder_comparison.py +208 -0
- exegete/cursors.py +218 -0
- exegete/database.py +12621 -0
- exegete/env_settings.py +174 -0
- exegete/memo_privacy.py +215 -0
- exegete/names.py +68 -0
- exegete/new_project.py +1027 -0
- exegete/preview_tokens.py +613 -0
- exegete/project_settings.py +1082 -0
- exegete/pseudonymise.py +2796 -0
- exegete/refi_export.py +645 -0
- exegete/server.py +18899 -0
- exegete/sessions.py +1153 -0
- exegete/state_folder.py +360 -0
- exegete/transition.py +1049 -0
- exegete-0.14.1a0.dev1.dist-info/METADATA +515 -0
- exegete-0.14.1a0.dev1.dist-info/RECORD +24 -0
- exegete-0.14.1a0.dev1.dist-info/WHEEL +5 -0
- exegete-0.14.1a0.dev1.dist-info/entry_points.txt +2 -0
- exegete-0.14.1a0.dev1.dist-info/licenses/COPYING.LESSER +165 -0
- exegete-0.14.1a0.dev1.dist-info/licenses/NOTICE +773 -0
- exegete-0.14.1a0.dev1.dist-info/licenses/legal/GPL-3.0.txt +674 -0
- exegete-0.14.1a0.dev1.dist-info/top_level.txt +1 -0
exegete/pseudonymise.py
ADDED
|
@@ -0,0 +1,2796 @@
|
|
|
1
|
+
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
"""The pseudonymisation engine (v0.12 flagship, D1).
|
|
3
|
+
|
|
4
|
+
Pure functions over strings and integers. No database, no server, no new
|
|
5
|
+
dependency: the edit list is computed here and is exact, so every stored
|
|
6
|
+
offset is remapped arithmetically rather than by diffing two texts. The
|
|
7
|
+
parity oracles in the test suite drive this module directly, which is
|
|
8
|
+
why it imports nothing of ours.
|
|
9
|
+
|
|
10
|
+
What it does, in one paragraph. A mapping of names to pseudonyms is
|
|
11
|
+
validated, compiled into ONE whole-word, escaped, longest-first
|
|
12
|
+
alternation, and run over a text in a single non-chaining pass. Each
|
|
13
|
+
match becomes an edit. Every `code_text`, `annotation` and `case_text`
|
|
14
|
+
span of that text is then remapped under one of two policies:
|
|
15
|
+
`snap_to_pseudonym`, which treats the pseudonym as the same token as the
|
|
16
|
+
name it replaces and never empties or deletes a span, and
|
|
17
|
+
`qualcoder_edit_parity`, which reproduces the semantics QualCoder's own
|
|
18
|
+
coding-view editor applies when a whole token is deleted and retyped.
|
|
19
|
+
|
|
20
|
+
Upstream reference at pin 9bddf17 (QualCoder master), read-only:
|
|
21
|
+
|
|
22
|
+
- the boundary regex and the escaping: `manage_files.py:3344-3349`,
|
|
23
|
+
`re.sub(rf"(?<!\\w){re.escape(original)}(?!\\w)", pseudonym, text_)`,
|
|
24
|
+
applied entry by entry over the whole text at import;
|
|
25
|
+
- the minimum lengths: `pseudonyms.py:72-74` (original, 2) and
|
|
26
|
+
`pseudonyms.py:76-78` (pseudonym, 3);
|
|
27
|
+
- the span walk `qualcoder_edit_parity` reproduces: `apply_insert` at
|
|
28
|
+
`code_text.py:5893-5904`, `apply_delete` at `:5906-5927`, the running
|
|
29
|
+
offset at `:5933-5950`, the end clamp at `:6208-6209` (case links) and
|
|
30
|
+
`:6232-6233` (codings), and the absence of a clamp for annotations at
|
|
31
|
+
`:6213-6222`.
|
|
32
|
+
|
|
33
|
+
Two deliberate deviations from upstream, both documented in the tool
|
|
34
|
+
description and pinned by tests:
|
|
35
|
+
|
|
36
|
+
1. ONE pass, not a chain. Upstream applies each entry as its own
|
|
37
|
+
`re.sub` over the whole text, so a later entry rewrites what an
|
|
38
|
+
earlier one produced (`[Sam -> Alex, Alex -> Pat]` turns every "Sam"
|
|
39
|
+
into "Pat"). This engine refuses a mapping whose pseudonym is also a
|
|
40
|
+
name in the mapping and matches everything in one leftmost-longest
|
|
41
|
+
pass, so no replacement is ever replaced again.
|
|
42
|
+
2. `qualcoder_edit_parity` reproduces the SEMANTICS of master's walk for
|
|
43
|
+
a whole-token replacement (delete the name, insert the pseudonym at
|
|
44
|
+
the same offset), not the output of diff-match-patch. Upstream feeds
|
|
45
|
+
its walk whatever `diff_main` produced, and `diff_main` may factor a
|
|
46
|
+
common prefix or suffix out of a replacement, which would move the
|
|
47
|
+
edit boundaries. That factoring is a property of the diff library and
|
|
48
|
+
is not reproduced here; the claim this module makes is the narrower,
|
|
49
|
+
true one.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
import re
|
|
53
|
+
import unicodedata
|
|
54
|
+
from bisect import bisect_left, bisect_right
|
|
55
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
56
|
+
|
|
57
|
+
# --------------------------------------------------------------------------
|
|
58
|
+
# Limits and vocabularies
|
|
59
|
+
# --------------------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
# Upstream's own minimum for an original (pseudonyms.py:72-74). Two code
|
|
62
|
+
# points, so a one-letter initial cannot be turned into a rule that
|
|
63
|
+
# rewrites half the transcript.
|
|
64
|
+
MIN_ORIGINAL_CHARS = 2
|
|
65
|
+
# Upstream's own minimum for a pseudonym (pseudonyms.py:76-78).
|
|
66
|
+
MIN_PSEUDONYM_CHARS = 3
|
|
67
|
+
# Ours. A pseudonym is written into the researcher's text once per match,
|
|
68
|
+
# so its length is what bounds how far this tool can grow a file.
|
|
69
|
+
MAX_PSEUDONYM_CHARS = 200
|
|
70
|
+
# Ours. A surface form is a NAME, and 200 characters is already absurdly
|
|
71
|
+
# generous for one. The cap is not about the text it matches, it is about
|
|
72
|
+
# `max_form_len`: the overlap diagnostic scans a window of that width
|
|
73
|
+
# around every match, so an uncapped form makes a READ-ONLY preview, with
|
|
74
|
+
# no token and no approval, cost O(matches x form length). Measured
|
|
75
|
+
# before the cap: a 200,000-character original over a 99 KB file took 91
|
|
76
|
+
# seconds, and `validate_mapping` accepted a 1,000,000-character one
|
|
77
|
+
# (Security S4).
|
|
78
|
+
MAX_FORM_CHARS = 200
|
|
79
|
+
# Ours. The same amplification from the other side: entries were capped
|
|
80
|
+
# and variants were not, so one entry could carry 200,000 of them.
|
|
81
|
+
MAX_VARIANTS_PER_ENTRY = 50
|
|
82
|
+
# Ours. What the alternation, `forms_at`'s buckets and the token payload
|
|
83
|
+
# are actually sized by. 500 entries of 4 forms each is far past any real
|
|
84
|
+
# mapping.
|
|
85
|
+
MAX_TOTAL_FORMS = 2000
|
|
86
|
+
# Ours. Five hundred entries is far past any real mapping and keeps the
|
|
87
|
+
# alternation, the diagnostics and the token payload bounded.
|
|
88
|
+
MAX_ENTRIES = 500
|
|
89
|
+
# Ours. `include_context` returns file text; this caps one window.
|
|
90
|
+
MAX_CONTEXT_CHARS = 120
|
|
91
|
+
# Ours, and this is the cap that actually stops a preview turning into a
|
|
92
|
+
# transcript dump. Per-window was never enough: the number of windows was
|
|
93
|
+
# uncapped, so 120 characters each side of 400 shown spans returned
|
|
94
|
+
# 98,100 characters of a 20,800-character transcript, 4.7 times the file,
|
|
95
|
+
# because the windows overlap (Security S8). The budget is per preview
|
|
96
|
+
# call, across every file and every entry, and a block that runs out of
|
|
97
|
+
# it says `context_truncated`.
|
|
98
|
+
MAX_CONTEXT_TOTAL_CHARS = 10000
|
|
99
|
+
# Ours. A paging cap in the house shape (`validate_limit`): silently
|
|
100
|
+
# capped rather than refused, because a caller asking for more than this
|
|
101
|
+
# many span positions per entry is asking for "all of them".
|
|
102
|
+
MAX_SPANS_PER_ENTRY = 500
|
|
103
|
+
# Ours, and deliberately NOT driven by a tool argument: `overlap_conflicts`
|
|
104
|
+
# is part of the signed effect block, so its size must be the same on the
|
|
105
|
+
# preview call and on the execute call.
|
|
106
|
+
MAX_OVERLAP_CONFLICTS = 200
|
|
107
|
+
# Ours (fix round 6, the fourth re-verification's B4-3). The overlap
|
|
108
|
+
# diagnostic examines, for every match in the file a call rewrites, every
|
|
109
|
+
# position within the longest form's reach and there every form starting
|
|
110
|
+
# with that position's letter: about 0.2 microseconds a form examined on
|
|
111
|
+
# the development Mac, so at 2,000 names a transcript with a speaker's
|
|
112
|
+
# label on every line examined 17 million a megabyte (3.4 s), and a
|
|
113
|
+
# mapping of long forms sharing a prefix 200 ms a match. Past this many
|
|
114
|
+
# the diagnostic stops, lists what it found and says it was capped
|
|
115
|
+
# (`overlap_conflicts_capped`); about half a second here. Counted, not
|
|
116
|
+
# timed, so the preview and the execute call cap it at the same place:
|
|
117
|
+
# the diagnostic is part of the signed effect. The rewrite is not
|
|
118
|
+
# affected: it never reads the diagnostic.
|
|
119
|
+
MAX_OVERLAP_CANDIDATES = 2_000_000
|
|
120
|
+
# Ours, a WARNING threshold and not a limit. The residue block reads a
|
|
121
|
+
# name as a bare substring, and QualCoder's own two-character minimum
|
|
122
|
+
# is well below the length at which that reading stops being exact: a
|
|
123
|
+
# form of fewer than four characters makes the residue counts generous
|
|
124
|
+
# (re-verification 6.2 measured the turn from zero false hits to one in
|
|
125
|
+
# five at four). The rewrite is whole-word and is not affected.
|
|
126
|
+
SHORT_FORM_CHARS = 4
|
|
127
|
+
# Ours (v0.13, ruling 7; re-derived in fix round 1 from S-4). The
|
|
128
|
+
# file-text count's work budget, deterministic rather than a clock so the
|
|
129
|
+
# same preview always counts the same files. A file's work is
|
|
130
|
+
# `len(text) * (surface forms + RESIDUE_WORK_PER_CHARACTER)`: the count
|
|
131
|
+
# costs a little per character whatever the mapping (the reader's reading,
|
|
132
|
+
# the two passes' own scan), and that term is what a one-form mapping
|
|
133
|
+
# spends most of, the worst case a rate taken at a hundred forms cannot
|
|
134
|
+
# see (the Security gate measured 6 seconds at one form under the old
|
|
135
|
+
# model). Fix round 1 measured 1 MB of English prose at 1, 10 and 100
|
|
136
|
+
# forms, best of five: 4.24 ms per MB per unit of work with the
|
|
137
|
+
# per-character term at 2.45 on Python 3.13.5, 3.98 and 2.32 on 3.11.13,
|
|
138
|
+
# rounded the term up to 3 and set the budget at 350 million. Past the
|
|
139
|
+
# budget a file is still read, with the one cheap question "does any
|
|
140
|
+
# name show here", and the report says which files those are; it is
|
|
141
|
+
# never omitted and never reported clean when it is not.
|
|
142
|
+
#
|
|
143
|
+
# Fix round 2 (the re-verification's B-1, B-2 and B-3) re-derived all
|
|
144
|
+
# three budgets from the WORST of: one form on ASCII text, one form on
|
|
145
|
+
# text that is not ASCII, the hostile dense run, and 1,997 forms with
|
|
146
|
+
# Turkish capitals under an insensitive mode, on Python 3.13.5 and
|
|
147
|
+
# 3.11.13 (BRIEF1_FIX2_REPORT.md has the figures). The term above was
|
|
148
|
+
# measured on text stripped to ASCII; on any other text (one curly
|
|
149
|
+
# apostrophe is enough) the reader's reading costs more per character, so
|
|
150
|
+
# the term is priced by the text's own class (`residue_work`): measured
|
|
151
|
+
# 2.2 to 2.7 on ASCII, rounded up to 3, and 3.4 to 6.7 on curly quotes,
|
|
152
|
+
# Cyrillic, Japanese and accented Latin composed or decomposed, rounded
|
|
153
|
+
# up to 7 (after `_strip_unseen` stopped costing 37 to 74 ms per MB). The
|
|
154
|
+
# unit rate is 3.9 to 5.6 ms per MB per unit in exact mode and 4.4 to 6.0
|
|
155
|
+
# in the insensitive ones, the worst 5.97 (3.11.13, curly quotes).
|
|
156
|
+
#
|
|
157
|
+
# Frozen under ruling 7 from the six CI platforms (run 35920173126 at
|
|
158
|
+
# 15b0859, the rate line's worst case at the budgets of the time, 200
|
|
159
|
+
# million, 60 million and 150,000): macOS 3.10 3.15 s, macOS 3.13 3.34,
|
|
160
|
+
# Ubuntu 3.13 4.00, Windows 3.13 4.05, Ubuntu 3.10 4.41, Windows 3.10
|
|
161
|
+
# 4.43. All three budgets were scaled by 0.45 to bring the slowest to
|
|
162
|
+
# about two seconds (the six rate lines then read 1.51 to 1.98 s). The
|
|
163
|
+
# file a call names is read first from the same budgets (ruling 1 as
|
|
164
|
+
# amended, fix round 4), and every count is charged all it spent, one
|
|
165
|
+
# that stops part-way too (fix round 5, the lead's rule 1), so nothing is
|
|
166
|
+
# spent outside them. A unit costs what the rate line measured only if
|
|
167
|
+
# the estimate prices what the count reads: since fix round 6 a file is
|
|
168
|
+
# priced at the length of its reading (`reading_length`, B4-1) and a
|
|
169
|
+
# form by its length (`FORM_UNIT_CHARS`, B4-2). The lead's scope of
|
|
170
|
+
# ruling 7: on realistic input (natural text in any script, mappings of
|
|
171
|
+
# up to 2,000 forms of up to 100 characters) the worst case is the
|
|
172
|
+
# three budgets' cost, the figure the rate line prints (0.93 to 0.96 s
|
|
173
|
+
# on the development Mac; through the tool the dearest shape measured,
|
|
174
|
+
# filling all three, 0.80 to 0.86 s at fix round 6). Crafted input is
|
|
175
|
+
# bounded, and linear in the text and the mapping, but may take longer:
|
|
176
|
+
# the dearest measured, a text of U+3316 (NFKC reads it as six
|
|
177
|
+
# characters and recomposes them, at about 3.6 times a unit of prose),
|
|
178
|
+
# fills the budgets in 1.9 to 2.1 s there; long forms sharing a prefix,
|
|
179
|
+
# 1.2 to 1.3 s. A file whose estimate passes the EMPTY budget is not
|
|
180
|
+
# counted at all, and is too large for this mapping (rule 2). A run of
|
|
181
|
+
# combining marks out of order no longer costs more than its length
|
|
182
|
+
# (`_ordered_runs`).
|
|
183
|
+
RESIDUE_WORK_PER_CHARACTER = 3
|
|
184
|
+
RESIDUE_WORK_PER_CHARACTER_NON_ASCII = 7
|
|
185
|
+
# Ours (fix round 6, the fourth re-verification's B4-2). A form is priced
|
|
186
|
+
# one unit for each started block of this many characters of its reading,
|
|
187
|
+
# not one unit whatever its length: over text that runs along a prefix
|
|
188
|
+
# many forms share, the detector's alternation compares every one of them
|
|
189
|
+
# as far as the prefix goes. Measured on this Mac: 100 names of up to ten
|
|
190
|
+
# characters over prose cost 5.4 ns a unit for the count and 6.2 for the
|
|
191
|
+
# cheap question, as before; 2,000 forms of 200 characters sharing 197
|
|
192
|
+
# over a run of that letter, 7.6 and 12.5 ns a unit priced by blocks of
|
|
193
|
+
# ten (152 and 250 priced one unit a form). Ten keeps every name of up to
|
|
194
|
+
# ten characters at one unit; a full name of eleven to twenty is two.
|
|
195
|
+
FORM_UNIT_CHARS = 10
|
|
196
|
+
MAX_RESIDUE_SCAN_WORK = 90_000_000
|
|
197
|
+
# Ours (fix round 2, the lead's ruling on B-2). Past the work or the match
|
|
198
|
+
# budget a file is asked the cheap question "does any name show here", and
|
|
199
|
+
# on a file where none shows that question reads the whole text: charged
|
|
200
|
+
# to this budget of its own, in the same units. It costs a little more
|
|
201
|
+
# per unit than the count (three scans where the count on ASCII makes
|
|
202
|
+
# two): 2.8 to 4.2 ms per MB per unit at one form, 4.5 to 6.9 at a
|
|
203
|
+
# hundred, so this budget's worst case is about 27 x 6.91 ms, 0.19
|
|
204
|
+
# seconds here (scaled with the others under ruling 7, above). A file
|
|
205
|
+
# whose question passes this budget on its own is not asked and closes
|
|
206
|
+
# nothing (rule 2); past it a file is not checked at all: it is listed in
|
|
207
|
+
# `files_not_checked`, the warning says how to get it checked, and it is
|
|
208
|
+
# never reported clean.
|
|
209
|
+
MAX_RESIDUE_CHECK_WORK = 27_000_000
|
|
210
|
+
# Ours (fix round 1, S-4; re-derived in fix round 2). Matches cost work
|
|
211
|
+
# the character model does not see: a text that is nothing but the name
|
|
212
|
+
# repeated has a match every few characters. Every match of the three
|
|
213
|
+
# passes counts against this budget, a first sight of a spelling as
|
|
214
|
+
# `RESIDUE_FIRST_SIGHT_MATCHES`. A count that passes it stops, is charged
|
|
215
|
+
# what it spent, and is reported counted in part, with a lower bound: a
|
|
216
|
+
# name certainly shows (fix round 5, the lead's rule 3). Every file after
|
|
217
|
+
# it drops to the cheap question, as past the work budget.
|
|
218
|
+
# The dearest unit measured is a first sight of a Turkish capital at 1,997
|
|
219
|
+
# forms under an insensitive mode, 2.2 microseconds on 3.11.13 (the dense
|
|
220
|
+
# typed run, a name inside a long run of word characters, 1.3 to 1.6), so
|
|
221
|
+
# this budget's worst case is about 0.15 seconds here. The three
|
|
222
|
+
# together, and the 0.08 seconds a 2,000-form mapping costs to prepare
|
|
223
|
+
# once per preview, are under one second on the development Mac and
|
|
224
|
+
# about two seconds on the slowest CI platform (ruling 7, above).
|
|
225
|
+
MAX_RESIDUE_SCAN_MATCHES = 67_500
|
|
226
|
+
# Ours (fix round 2, the re-verification's B-3). The first sight of a
|
|
227
|
+
# spelling in a pass is placed (the entry it belongs to, the rewriter's
|
|
228
|
+
# form, whether the reader's reading sees it) and every later sight is
|
|
229
|
+
# looked up, so a text written to make every sight a new spelling (case
|
|
230
|
+
# variants, Turkish capitals under an insensitive mode) costs more per
|
|
231
|
+
# match than the dense run the budget above was priced on. Measured at
|
|
232
|
+
# 1,997 forms: a first sight costs up to 6.9 microseconds more than a
|
|
233
|
+
# repeated one on 3.11.13 (4.5 on 3.13.5), against 1.4 to 1.7 for an
|
|
234
|
+
# ordinary match, so a first sight is charged five matches, and one more
|
|
235
|
+
# for each further form sharing its key that has to be tried.
|
|
236
|
+
RESIDUE_FIRST_SIGHT_MATCHES = 5
|
|
237
|
+
# Ours (fix round 1, S-3). Entry rows in one file row, in wide-count
|
|
238
|
+
# order; the row's own totals stay complete and the row says
|
|
239
|
+
# `entries_truncated`. The same cap bounds each of the row's two
|
|
240
|
+
# spelling lists.
|
|
241
|
+
MAX_RESIDUE_ENTRY_ROWS = 50
|
|
242
|
+
# Ours (v0.13). Full rows in the residue's file-text block. The totals are
|
|
243
|
+
# always complete, and every further file that shows a name has a compact
|
|
244
|
+
# row or, past that cap too, is listed by id: the house shape of
|
|
245
|
+
# `max_spans_per_entry` and `MAX_OVERLAP_CONFLICTS`.
|
|
246
|
+
MAX_RESIDUE_FILE_ROWS = 200
|
|
247
|
+
# Ours (fix round 2, the lead's ruling on CORR-4). Compact rows (id, name,
|
|
248
|
+
# the two readings' counts) have a cap of their own, so the owner's
|
|
249
|
+
# "compact by default" promise of a row for each file still showing a name
|
|
250
|
+
# holds up to a thousand files; past it a file is listed by id only in
|
|
251
|
+
# `more_files_showing_a_name`, and the detail note says so.
|
|
252
|
+
MAX_RESIDUE_COMPACT_ROWS = 1000
|
|
253
|
+
# Ours (v0.13, decision B). The longer words listed per entry per file
|
|
254
|
+
# on the typed path, most frequent first; the list says when it stopped.
|
|
255
|
+
MAX_LONGER_WORDS_PER_ENTRY = 20
|
|
256
|
+
# Ours (v0.13, the lead's ruling on S-1). A longer word is listed only
|
|
257
|
+
# when it extends the matched name by at most this many characters, left
|
|
258
|
+
# and right together: `Thomasson` (+3), `Thomas_P01` (+4), `Thomasin`
|
|
259
|
+
# (+2) and `Thomas_Smith` (+6) are words; a run of word characters that
|
|
260
|
+
# carries a date of birth and a record number joined to the name is not,
|
|
261
|
+
# and is counted and not listed, the list saying it is incomplete.
|
|
262
|
+
MAX_LONGER_WORD_EXTENSION = 8
|
|
263
|
+
# Ours (v0.13, the same ruling). All the longer words one preview lists
|
|
264
|
+
# share this many characters; past it an occurrence is counted and not
|
|
265
|
+
# listed. On by default on the typed path, so it sits well under the
|
|
266
|
+
# 10,000 characters `include_context` may spend when the caller opts in.
|
|
267
|
+
MAX_LONGER_WORDS_TOTAL_CHARS = 4000
|
|
268
|
+
# Scripts written without separators between words, where one character
|
|
269
|
+
# can be a word of its own and a run of "word characters" is a clause:
|
|
270
|
+
# Han, Hiragana, Katakana, Thai, Lao, Khmer and Myanmar (the lead's ruling
|
|
271
|
+
# on S-1). A longer word whose extension carries any of these is never
|
|
272
|
+
# listed. Explicit ranges, because `unicodedata` does not expose the
|
|
273
|
+
# Script property; each range is pinned.
|
|
274
|
+
NO_SEPARATOR_RANGES = (
|
|
275
|
+
(0x0E00, 0x0E7F, "Thai"),
|
|
276
|
+
(0x0E80, 0x0EFF, "Lao"),
|
|
277
|
+
(0x1000, 0x109F, "Myanmar"),
|
|
278
|
+
(0x1780, 0x17FF, "Khmer"),
|
|
279
|
+
(0x19E0, 0x19FF, "Khmer Symbols"),
|
|
280
|
+
(0x2E80, 0x2EFF, "CJK Radicals Supplement"),
|
|
281
|
+
(0x2F00, 0x2FDF, "Kangxi Radicals"),
|
|
282
|
+
(0x3005, 0x3007, "CJK iteration and zero marks"),
|
|
283
|
+
(0x3021, 0x3029, "Hangzhou numerals"),
|
|
284
|
+
(0x3038, 0x303B, "CJK iteration marks"),
|
|
285
|
+
(0x3040, 0x309F, "Hiragana"),
|
|
286
|
+
(0x30A0, 0x30FF, "Katakana"),
|
|
287
|
+
(0x31F0, 0x31FF, "Katakana Phonetic Extensions"),
|
|
288
|
+
(0x3400, 0x4DBF, "CJK Unified Ideographs Extension A"),
|
|
289
|
+
(0x4E00, 0x9FFF, "CJK Unified Ideographs"),
|
|
290
|
+
(0xA9E0, 0xA9FF, "Myanmar Extended-B"),
|
|
291
|
+
(0xAA60, 0xAA7F, "Myanmar Extended-A"),
|
|
292
|
+
(0xF900, 0xFAFF, "CJK Compatibility Ideographs"),
|
|
293
|
+
(0xFF66, 0xFF9F, "Halfwidth Katakana"),
|
|
294
|
+
(0x1AFF0, 0x1B16F, "Kana Extended-B to Small Kana Extension"),
|
|
295
|
+
(0x20000, 0x2FA1F, "CJK Unified Ideographs Extensions B to F, "
|
|
296
|
+
"and the Compatibility Supplement"),
|
|
297
|
+
(0x30000, 0x323AF, "CJK Unified Ideographs Extensions G and H"),
|
|
298
|
+
)
|
|
299
|
+
_NO_SEPARATOR_STARTS = tuple(low for low, _, _ in NO_SEPARATOR_RANGES)
|
|
300
|
+
|
|
301
|
+
CASE_MODES = ("exact", "insensitive", "insensitive_preserve")
|
|
302
|
+
OVERLAP_POLICIES = ("snap_to_pseudonym", "qualcoder_edit_parity")
|
|
303
|
+
|
|
304
|
+
# How a row's span changed. One vocabulary for both policies, because the
|
|
305
|
+
# preview shape and the hidden-coder rule key on it. `substituted` is the
|
|
306
|
+
# owner's ruling of 2026-09-15 (QC40_PLATFORM 7.3(3), refining X1): a
|
|
307
|
+
# span that sat exactly on a replaced name and now sits exactly on its
|
|
308
|
+
# pseudonym is a pure substitution, whatever the two lengths, and is
|
|
309
|
+
# exempt from the hidden-coder override like a pure shift. `clamped` is
|
|
310
|
+
# the owner's ruling of 2026-09-16 (QC40_PLATFORM 7.4, decision 7): a
|
|
311
|
+
# span that CONTAINED a whole replaced name and changed length only
|
|
312
|
+
# because that name did is `resized` and is exempt too, so the one row
|
|
313
|
+
# whose extent changed for a reason the rewrite does not explain, the
|
|
314
|
+
# damaged row whose stored end lay past the end of the text, needs a
|
|
315
|
+
# class of its own to stay on the override side.
|
|
316
|
+
UNCHANGED = "unchanged"
|
|
317
|
+
SHIFTED = "shifted"
|
|
318
|
+
SUBSTITUTED = "substituted"
|
|
319
|
+
RESIZED = "resized"
|
|
320
|
+
SNAPPED = "snapped"
|
|
321
|
+
DELETED = "deleted"
|
|
322
|
+
CLAMPED = "clamped"
|
|
323
|
+
|
|
324
|
+
# Characters refused in any mapping string.
|
|
325
|
+
#
|
|
326
|
+
# D1 3.2 names line breaks, U+2029 and control characters. This pattern is
|
|
327
|
+
# a superset, and the additions are deliberate:
|
|
328
|
+
#
|
|
329
|
+
# - U+2028 as well as U+2029, because both are line separators Qt's
|
|
330
|
+
# document model treats as breaks;
|
|
331
|
+
# - the bidirectional formatting controls, all of them: Unicode's
|
|
332
|
+
# Bidi_Control set is U+061C, U+200E, U+200F, U+202A to U+202E and
|
|
333
|
+
# U+2066 to U+2069, because a pseudonym is written INTO the
|
|
334
|
+
# researcher's text and one of these would visually reorder every line
|
|
335
|
+
# after it while leaving the stored offsets untouched. U+061C is the
|
|
336
|
+
# weakest of them (an invisible strong-AL character rather than a
|
|
337
|
+
# range control) and was the one this list missed; no pseudonym needs
|
|
338
|
+
# it, so the set is now the whole property rather than an enumeration;
|
|
339
|
+
# - U+FEFF, because QualCoder strips a leading one on import
|
|
340
|
+
# (manage_files.py:3340-3341), so a pseudonym carrying one would behave
|
|
341
|
+
# differently depending on where it landed.
|
|
342
|
+
#
|
|
343
|
+
# The zero-width joiner and non-joiner (U+200C, U+200D) are NOT refused:
|
|
344
|
+
# they are needed to spell ordinary words in several scripts.
|
|
345
|
+
# The ranges are built with chr() rather than written as escapes so that
|
|
346
|
+
# this source file itself contains no control characters.
|
|
347
|
+
_FORBIDDEN_RANGES = (
|
|
348
|
+
(0x0000, 0x001F), # C0 controls, which includes every line break
|
|
349
|
+
(0x007F, 0x009F), # DEL and the C1 controls
|
|
350
|
+
(0x061C, 0x061C), # ARABIC LETTER MARK, the fifth bidi control
|
|
351
|
+
(0x2028, 0x2029), # LINE SEPARATOR and PARAGRAPH SEPARATOR
|
|
352
|
+
(0x200E, 0x200F), # LEFT-TO-RIGHT and RIGHT-TO-LEFT MARK
|
|
353
|
+
(0x202A, 0x202E), # the bidirectional embeddings and overrides
|
|
354
|
+
(0x2066, 0x2069), # the bidirectional isolates
|
|
355
|
+
(0xFEFF, 0xFEFF), # ZERO WIDTH NO-BREAK SPACE, the byte-order mark
|
|
356
|
+
)
|
|
357
|
+
_FORBIDDEN_CHARS_RE = re.compile(
|
|
358
|
+
"[" + "".join(chr(lo) + "-" + chr(hi)
|
|
359
|
+
for lo, hi in _FORBIDDEN_RANGES) + "]")
|
|
360
|
+
|
|
361
|
+
# QualCoder 4.0's private-memo marker. A mapping string carrying one
|
|
362
|
+
# could plant a private zone in any memo a later release rewrites, and
|
|
363
|
+
# there is no reading of a name that needs five hashes in it.
|
|
364
|
+
_MARKER_RUN_RE = re.compile(r"#{5,}")
|
|
365
|
+
|
|
366
|
+
# Beyond the Basic Multilingual Plane. QualCoder's editor counts UTF-16
|
|
367
|
+
# code units, so a single astral code point makes Qt's offsets and the
|
|
368
|
+
# code-point offsets every report uses diverge from that point on
|
|
369
|
+
# (database.py `position_safe`). A pseudonym is ours to choose, so this is
|
|
370
|
+
# refused rather than warned about.
|
|
371
|
+
_ASTRAL_MINIMUM = 0x10000
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _has_astral(value: str) -> bool:
|
|
375
|
+
"""Whether `value` carries a code point outside the BMP."""
|
|
376
|
+
return any(ord(char) >= _ASTRAL_MINIMUM for char in value)
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
class MappingError(ValueError):
|
|
380
|
+
"""A mapping this engine refuses, with the caller-facing text.
|
|
381
|
+
|
|
382
|
+
A ValueError subclass so the server's tool guard turns it into the
|
|
383
|
+
ordinary sanitised error envelope without a branch of its own.
|
|
384
|
+
"""
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
# --------------------------------------------------------------------------
|
|
388
|
+
# Validation
|
|
389
|
+
# --------------------------------------------------------------------------
|
|
390
|
+
|
|
391
|
+
def _describe_bad_characters(value: str) -> Optional[str]:
|
|
392
|
+
"""Which character rule `value` breaks, or None."""
|
|
393
|
+
if _FORBIDDEN_CHARS_RE.search(value):
|
|
394
|
+
return "control"
|
|
395
|
+
if _MARKER_RUN_RE.search(value):
|
|
396
|
+
return "marker"
|
|
397
|
+
return None
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _check_common(value: Any, entry_index: int, label: str) -> str:
|
|
401
|
+
"""The rules every mapping string shares. Returns the string."""
|
|
402
|
+
if not isinstance(value, str):
|
|
403
|
+
raise MappingError(
|
|
404
|
+
f"mapping entry {entry_index}: {label} must be a string.")
|
|
405
|
+
if value != value.strip():
|
|
406
|
+
raise MappingError(
|
|
407
|
+
f"mapping entry {entry_index}: {label} must not begin or end "
|
|
408
|
+
f"with whitespace.")
|
|
409
|
+
if _describe_bad_characters(value) is not None:
|
|
410
|
+
raise MappingError(
|
|
411
|
+
f"mapping entry {entry_index}: original/pseudonym/variant must "
|
|
412
|
+
f"not contain line breaks, control characters, bidirectional "
|
|
413
|
+
f"formatting characters or a run of five or more '#' "
|
|
414
|
+
f"characters.")
|
|
415
|
+
return value
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _check_name(value: Any, entry_index: int, label: str) -> str:
|
|
419
|
+
"""An `original` or a `variant`: a surface form to look for."""
|
|
420
|
+
value = _check_common(value, entry_index, label)
|
|
421
|
+
if len(value) < MIN_ORIGINAL_CHARS:
|
|
422
|
+
raise MappingError(
|
|
423
|
+
f"mapping entry {entry_index}: {label} must be at least "
|
|
424
|
+
f"{MIN_ORIGINAL_CHARS} characters.")
|
|
425
|
+
if len(value) > MAX_FORM_CHARS:
|
|
426
|
+
raise MappingError(
|
|
427
|
+
f"mapping entry {entry_index}: {label} must be at most "
|
|
428
|
+
f"{MAX_FORM_CHARS} characters.")
|
|
429
|
+
return value
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _check_pseudonym(value: Any, entry_index: int,
|
|
433
|
+
may_echo_names: bool = True) -> str:
|
|
434
|
+
"""A `pseudonym`: a string this engine writes into the file."""
|
|
435
|
+
value = _check_common(value, entry_index, "pseudonym")
|
|
436
|
+
if len(value) < MIN_PSEUDONYM_CHARS:
|
|
437
|
+
raise MappingError(
|
|
438
|
+
f"mapping entry {entry_index}: pseudonym must be at least "
|
|
439
|
+
f"{MIN_PSEUDONYM_CHARS} characters (QualCoder's own minimum).")
|
|
440
|
+
if len(value) > MAX_PSEUDONYM_CHARS:
|
|
441
|
+
raise MappingError(
|
|
442
|
+
f"mapping entry {entry_index}: pseudonym must be at most "
|
|
443
|
+
f"{MAX_PSEUDONYM_CHARS} characters.")
|
|
444
|
+
if _has_astral(value) or "\r" in value:
|
|
445
|
+
shown = f"pseudonym '{value}'" if may_echo_names else "the pseudonym"
|
|
446
|
+
raise MappingError(
|
|
447
|
+
f"mapping entry {entry_index}: {shown} contains "
|
|
448
|
+
f"characters beyond U+FFFF (emoji or similar) or a carriage "
|
|
449
|
+
f"return; QualCoder's editor would misplace positions in the "
|
|
450
|
+
f"rewritten file. Choose a pseudonym in the Basic Multilingual "
|
|
451
|
+
f"Plane.")
|
|
452
|
+
return value
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
class Entry:
|
|
456
|
+
"""One validated mapping entry."""
|
|
457
|
+
|
|
458
|
+
__slots__ = ("index", "original", "pseudonym", "variants")
|
|
459
|
+
|
|
460
|
+
def __init__(self, index: int, original: str, pseudonym: str,
|
|
461
|
+
variants: Tuple[str, ...]):
|
|
462
|
+
self.index = index
|
|
463
|
+
self.original = original
|
|
464
|
+
self.pseudonym = pseudonym
|
|
465
|
+
self.variants = variants
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def forms(self) -> Tuple[str, ...]:
|
|
469
|
+
"""Every surface form this entry replaces, original first."""
|
|
470
|
+
return (self.original,) + self.variants
|
|
471
|
+
|
|
472
|
+
def __repr__(self) -> str: # pragma: no cover - debugging
|
|
473
|
+
return (f"Entry({self.index}, pseudonym={self.pseudonym!r}, "
|
|
474
|
+
f"forms={len(self.forms)})")
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
class Mapping:
|
|
478
|
+
"""A validated mapping, with the case mode it was validated under."""
|
|
479
|
+
|
|
480
|
+
__slots__ = ("entries", "case_mode", "shared_pseudonyms")
|
|
481
|
+
|
|
482
|
+
def __init__(self, entries: Tuple[Entry, ...], case_mode: str,
|
|
483
|
+
shared_pseudonyms: Tuple[Dict[str, Any], ...]):
|
|
484
|
+
self.entries = entries
|
|
485
|
+
self.case_mode = case_mode
|
|
486
|
+
# Two entries pointing at one pseudonym is a legitimate choice
|
|
487
|
+
# (two people merged into one identity), so it is reported rather
|
|
488
|
+
# than refused (D1 3.2).
|
|
489
|
+
self.shared_pseudonyms = shared_pseudonyms
|
|
490
|
+
|
|
491
|
+
def __len__(self) -> int:
|
|
492
|
+
return len(self.entries)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _fold(value: str) -> str:
|
|
496
|
+
return value.casefold()
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def validate_mapping(raw: Any, case_mode: str = "exact",
|
|
500
|
+
may_echo_names: bool = True) -> Mapping:
|
|
501
|
+
"""Validate a caller's mapping, or raise MappingError.
|
|
502
|
+
|
|
503
|
+
Refused as a whole, with nothing previewed, on any of: an empty
|
|
504
|
+
mapping, more than `MAX_ENTRIES` entries, a malformed entry, a string
|
|
505
|
+
that breaks the character rules, a duplicate surface form, or a
|
|
506
|
+
pseudonym that is also a name in the same mapping.
|
|
507
|
+
|
|
508
|
+
That last rule is where this engine parts company with upstream.
|
|
509
|
+
QualCoder applies entries sequentially (`manage_files.py:3344-3349`),
|
|
510
|
+
so `[Sam -> Alex, Alex -> Pat]` silently turns every "Sam" into "Pat";
|
|
511
|
+
the dialog checks only that the list has no duplicate originals and no
|
|
512
|
+
duplicate pseudonyms (`pseudonyms.py:84-89`). Refusing is the honest
|
|
513
|
+
answer, because there is no reading of that mapping the researcher
|
|
514
|
+
meant.
|
|
515
|
+
|
|
516
|
+
`case_mode` matters here: in the two insensitive modes two forms that
|
|
517
|
+
differ only in case ARE the same rule, so they are duplicates, and a
|
|
518
|
+
pseudonym that case-folds onto a name would chain.
|
|
519
|
+
|
|
520
|
+
`may_echo_names=False` says the mapping did NOT come from the caller:
|
|
521
|
+
it was read out of the project's own `pseudonyms.json`, which is the
|
|
522
|
+
researcher's reverse key. D1 5.1's premise ("the mapping is
|
|
523
|
+
user-supplied and therefore already in the conversation") is false on
|
|
524
|
+
that path and D1 3.10 promises the opposite, so no refusal text on it
|
|
525
|
+
quotes a value. Entry indices carry everything the reader needs, and
|
|
526
|
+
the chaining refusal is where it mattered most: the value it quotes
|
|
527
|
+
is a pseudonym that is ALSO somebody's real name, which is the whole
|
|
528
|
+
reason the entry is refused (Security S3).
|
|
529
|
+
"""
|
|
530
|
+
if case_mode not in CASE_MODES:
|
|
531
|
+
raise MappingError(
|
|
532
|
+
f"case_mode must be one of {', '.join(CASE_MODES)}.")
|
|
533
|
+
if raw is None or (isinstance(raw, (list, tuple)) and not raw):
|
|
534
|
+
raise MappingError(
|
|
535
|
+
"mapping is empty: give at least one entry with original and "
|
|
536
|
+
"pseudonym.")
|
|
537
|
+
if not isinstance(raw, (list, tuple)):
|
|
538
|
+
raise MappingError(
|
|
539
|
+
"mapping must be a list of entries, each an object with "
|
|
540
|
+
"original and pseudonym.")
|
|
541
|
+
if len(raw) > MAX_ENTRIES:
|
|
542
|
+
raise MappingError(
|
|
543
|
+
f"mapping has {len(raw)} entries; at most {MAX_ENTRIES} are "
|
|
544
|
+
f"accepted.")
|
|
545
|
+
|
|
546
|
+
insensitive = case_mode != "exact"
|
|
547
|
+
entries: List[Entry] = []
|
|
548
|
+
for index, item in enumerate(raw):
|
|
549
|
+
if not isinstance(item, dict):
|
|
550
|
+
raise MappingError(
|
|
551
|
+
f"mapping entry {index}: each entry must be an object with "
|
|
552
|
+
f"original and pseudonym.")
|
|
553
|
+
unknown = set(item) - {"original", "pseudonym", "variants"}
|
|
554
|
+
if unknown:
|
|
555
|
+
raise MappingError(
|
|
556
|
+
f"mapping entry {index}: unknown key(s) "
|
|
557
|
+
f"{', '.join(sorted(unknown))}; an entry has original, "
|
|
558
|
+
f"pseudonym and optionally variants.")
|
|
559
|
+
if "original" not in item or "pseudonym" not in item:
|
|
560
|
+
raise MappingError(
|
|
561
|
+
f"mapping entry {index}: each entry must have both original "
|
|
562
|
+
f"and pseudonym.")
|
|
563
|
+
original = _check_name(item["original"], index, "original")
|
|
564
|
+
pseudonym = _check_pseudonym(item["pseudonym"], index,
|
|
565
|
+
may_echo_names)
|
|
566
|
+
raw_variants = item.get("variants")
|
|
567
|
+
variants: List[str] = []
|
|
568
|
+
if raw_variants is not None:
|
|
569
|
+
if not isinstance(raw_variants, (list, tuple)):
|
|
570
|
+
raise MappingError(
|
|
571
|
+
f"mapping entry {index}: variants must be a list of "
|
|
572
|
+
f"strings.")
|
|
573
|
+
if len(raw_variants) > MAX_VARIANTS_PER_ENTRY:
|
|
574
|
+
raise MappingError(
|
|
575
|
+
f"mapping entry {index}: has {len(raw_variants)} "
|
|
576
|
+
f"variants; at most {MAX_VARIANTS_PER_ENTRY} are "
|
|
577
|
+
f"accepted per entry.")
|
|
578
|
+
for variant in raw_variants:
|
|
579
|
+
variants.append(_check_name(variant, index, "variant"))
|
|
580
|
+
entries.append(Entry(index, original, pseudonym, tuple(variants)))
|
|
581
|
+
|
|
582
|
+
total_forms = sum(len(entry.forms) for entry in entries)
|
|
583
|
+
if total_forms > MAX_TOTAL_FORMS:
|
|
584
|
+
raise MappingError(
|
|
585
|
+
f"mapping has {total_forms} surface forms (originals plus "
|
|
586
|
+
f"variants); at most {MAX_TOTAL_FORMS} are accepted.")
|
|
587
|
+
|
|
588
|
+
# Duplicate surface forms, across the whole mapping and within one
|
|
589
|
+
# entry. Exact spelling always; case-folded as well when the run is
|
|
590
|
+
# case-insensitive, because there the two are one rule.
|
|
591
|
+
seen: Dict[str, int] = {}
|
|
592
|
+
for entry in entries:
|
|
593
|
+
for form in entry.forms:
|
|
594
|
+
for key in ({form, _fold(form)} if insensitive else {form}):
|
|
595
|
+
previous = seen.get(key)
|
|
596
|
+
if previous is not None and previous != entry.index:
|
|
597
|
+
raise MappingError(
|
|
598
|
+
f"mapping entries {previous} and {entry.index}: the "
|
|
599
|
+
f"same original (or variant) appears twice.")
|
|
600
|
+
if previous is not None:
|
|
601
|
+
raise MappingError(
|
|
602
|
+
f"mapping entry {entry.index}: the same original (or "
|
|
603
|
+
f"variant) appears twice.")
|
|
604
|
+
seen[key] = entry.index
|
|
605
|
+
|
|
606
|
+
# A pseudonym that is also a name in this mapping would chain upstream
|
|
607
|
+
# and is refused here.
|
|
608
|
+
for entry in entries:
|
|
609
|
+
keys = {entry.pseudonym, _fold(entry.pseudonym)} if insensitive \
|
|
610
|
+
else {entry.pseudonym}
|
|
611
|
+
if any(key in seen for key in keys):
|
|
612
|
+
shown = (f"pseudonym '{entry.pseudonym}'" if may_echo_names
|
|
613
|
+
else f"the pseudonym of entry {entry.index}")
|
|
614
|
+
raise MappingError(
|
|
615
|
+
f"mapping entry {entry.index}: {shown} "
|
|
616
|
+
f"is also an original or variant in this "
|
|
617
|
+
f"mapping. QualCoder's import would chain the two "
|
|
618
|
+
f"replacements; this tool refuses instead. Choose a "
|
|
619
|
+
f"pseudonym that does not occur as a name in the mapping.")
|
|
620
|
+
|
|
621
|
+
by_pseudonym: Dict[str, List[int]] = {}
|
|
622
|
+
for entry in entries:
|
|
623
|
+
by_pseudonym.setdefault(entry.pseudonym, []).append(entry.index)
|
|
624
|
+
shared = tuple(
|
|
625
|
+
{"pseudonym": pseudonym, "entries": indices}
|
|
626
|
+
for pseudonym, indices in sorted(by_pseudonym.items())
|
|
627
|
+
if len(indices) > 1)
|
|
628
|
+
return Mapping(tuple(entries), case_mode, shared)
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def canonical_mapping(mapping: Mapping) -> List[Dict[str, Any]]:
|
|
632
|
+
"""The mapping as the authorisation token binds it (D1 3.8).
|
|
633
|
+
|
|
634
|
+
Entries sorted by original, variants sorted, every string normalised
|
|
635
|
+
to NFC, so the same mapping described in a different order binds
|
|
636
|
+
identically and a mapping read from `pseudonyms.json` binds the same
|
|
637
|
+
as the identical mapping typed out by the caller (which is what keeps
|
|
638
|
+
the source flag itself out of the binding).
|
|
639
|
+
|
|
640
|
+
Note what NFC does NOT do: the ENGINE matches the strings exactly as
|
|
641
|
+
supplied, never normalised, because normalising what we look for
|
|
642
|
+
could silently stop matching a file stored in another normal form.
|
|
643
|
+
So a mapping re-sent in a different normal form binds the same and
|
|
644
|
+
has a different effect, which the signed state catches as
|
|
645
|
+
"the project changed"; a test pins that.
|
|
646
|
+
"""
|
|
647
|
+
def nfc(value: str) -> str:
|
|
648
|
+
return unicodedata.normalize("NFC", value)
|
|
649
|
+
|
|
650
|
+
return sorted(
|
|
651
|
+
({"original": nfc(entry.original),
|
|
652
|
+
"pseudonym": nfc(entry.pseudonym),
|
|
653
|
+
"variants": sorted(nfc(v) for v in entry.variants)}
|
|
654
|
+
for entry in mapping.entries),
|
|
655
|
+
key=lambda item: (item["original"], item["pseudonym"],
|
|
656
|
+
item["variants"]))
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def canonical_entry_positions(mapping: Mapping) -> Dict[int, int]:
|
|
660
|
+
"""Each caller entry index, as its position in `canonical_mapping`.
|
|
661
|
+
|
|
662
|
+
The signed effect block records which ENTRY made each replacement,
|
|
663
|
+
and it used to record the caller's index: so the identical mapping
|
|
664
|
+
given in a different order, which `canonical_mapping` binds to the
|
|
665
|
+
same token, signed a different effect and was refused as
|
|
666
|
+
"the project changed" on a project that had not (fix round 3, S3).
|
|
667
|
+
Keyed on the canonical order, the same mapping signs the same effect
|
|
668
|
+
however it was typed, and a token issued from `pseudonyms.json`
|
|
669
|
+
executes from the identical typed mapping in any order.
|
|
670
|
+
"""
|
|
671
|
+
def nfc(value: str) -> str:
|
|
672
|
+
return unicodedata.normalize("NFC", value)
|
|
673
|
+
|
|
674
|
+
order = sorted(
|
|
675
|
+
range(len(mapping.entries)),
|
|
676
|
+
key=lambda index: (nfc(mapping.entries[index].original),
|
|
677
|
+
nfc(mapping.entries[index].pseudonym),
|
|
678
|
+
sorted(nfc(v) for v in mapping.entries[index].variants)))
|
|
679
|
+
return {caller: position for position, caller in enumerate(order)}
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
# --------------------------------------------------------------------------
|
|
683
|
+
# Compiling and matching
|
|
684
|
+
# --------------------------------------------------------------------------
|
|
685
|
+
|
|
686
|
+
def _form_sort_key(item: Tuple[str, int]) -> Tuple[int, int, str]:
|
|
687
|
+
"""Longest first, then entry order, then alphabetical (D1 3.4).
|
|
688
|
+
|
|
689
|
+
Length first is what makes "Mary Ann" win over "Mary" and
|
|
690
|
+
"Jean-Paul" over "Jean": Python's alternation takes the first branch
|
|
691
|
+
that lets the whole pattern succeed, so the longest viable form at a
|
|
692
|
+
position is chosen. The two remaining keys only make ties stable, so
|
|
693
|
+
the same mapping always compiles to the same pattern.
|
|
694
|
+
"""
|
|
695
|
+
form, index = item
|
|
696
|
+
return (-len(form), index, form)
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
# Runs of anything that is not a letter or a digit, underscore included
|
|
700
|
+
# (`\W` alone excludes it, and `_` is the separator this whole finding is
|
|
701
|
+
# about). Used to join the parts of a multi-word name, so the detector
|
|
702
|
+
# reads "Mary Ann", "Mary_Ann", "Mary-Ann" and "MaryAnn" as the same
|
|
703
|
+
# name, which is what a person reading a file list does.
|
|
704
|
+
_SEPARATOR_RUN = r"[\W_]*"
|
|
705
|
+
_SEPARATOR_SPLIT = re.compile(r"[\W_]+")
|
|
706
|
+
|
|
707
|
+
# Code points a reader does not see: Unicode's Default_Ignorable_Code_Point
|
|
708
|
+
# property, transcribed as ranges from DerivedCoreProperties.txt of the
|
|
709
|
+
# Unicode version named below (4,174 code points in seventeen ranges;
|
|
710
|
+
# the same set in 14.0.0 and 16.0.0, and one short in 13.0.0, which had
|
|
711
|
+
# not yet assigned U+180F). `unicodedata` does not expose the property,
|
|
712
|
+
# and an interpreter's own tables are that interpreter's (Python 3.10
|
|
713
|
+
# carries 13.0.0), so the table is embedded and applied as it stands on
|
|
714
|
+
# every interpreter. It holds the format characters a reader never sees
|
|
715
|
+
# (the soft hyphen, the zero-width space, joiner and non-joiner, the
|
|
716
|
+
# word joiner, the bidi marks and controls, the byte order mark, the
|
|
717
|
+
# tag characters), the marks and fillers that render as nothing (the
|
|
718
|
+
# combining grapheme joiner, the variation selectors, the Mongolian free
|
|
719
|
+
# variation selectors, the Khmer inherent vowels, the Hangul fillers)
|
|
720
|
+
# and the reserved code points the standard says a renderer must treat
|
|
721
|
+
# the same way. Fix round 3 approximated the property with category Cf
|
|
722
|
+
# plus a hand list that stopped at U+180D, and U+180F and the reserved
|
|
723
|
+
# ranges walked a name past the detector into the manifest (fix round
|
|
724
|
+
# 4, R2). Each range boundary is pinned against this table, and the
|
|
725
|
+
# count against the file's own total.
|
|
726
|
+
DEFAULT_IGNORABLE_UNICODE_VERSION = "15.1.0"
|
|
727
|
+
DEFAULT_IGNORABLE_RANGES = (
|
|
728
|
+
(0x00AD, 0x00AD), # SOFT HYPHEN
|
|
729
|
+
(0x034F, 0x034F), # COMBINING GRAPHEME JOINER
|
|
730
|
+
(0x061C, 0x061C), # ARABIC LETTER MARK
|
|
731
|
+
(0x115F, 0x1160), # HANGUL CHOSEONG FILLER, HANGUL JUNGSEONG FILLER
|
|
732
|
+
(0x17B4, 0x17B5), # KHMER VOWEL INHERENT AQ, AA
|
|
733
|
+
(0x180B, 0x180F), # MONGOLIAN FREE VARIATION SELECTORS ONE to FOUR
|
|
734
|
+
# and the MONGOLIAN VOWEL SEPARATOR between them
|
|
735
|
+
(0x200B, 0x200F), # ZERO WIDTH SPACE to RIGHT-TO-LEFT MARK
|
|
736
|
+
(0x202A, 0x202E), # the bidi embeddings and overrides
|
|
737
|
+
(0x2060, 0x206F), # WORD JOINER to NOMINAL DIGIT SHAPES, with the
|
|
738
|
+
# reserved U+2065 and the bidi isolates inside
|
|
739
|
+
(0x3164, 0x3164), # HANGUL FILLER
|
|
740
|
+
(0xFE00, 0xFE0F), # VARIATION SELECTOR-1 to -16
|
|
741
|
+
(0xFEFF, 0xFEFF), # ZERO WIDTH NO-BREAK SPACE, the byte order mark
|
|
742
|
+
(0xFFA0, 0xFFA0), # HALFWIDTH HANGUL FILLER
|
|
743
|
+
(0xFFF0, 0xFFF8), # reserved
|
|
744
|
+
(0x1BCA0, 0x1BCA3), # the shorthand format controls
|
|
745
|
+
(0x1D173, 0x1D17A), # the musical symbol beam and phrase controls
|
|
746
|
+
(0xE0000, 0xE0FFF), # LANGUAGE TAG, the tag characters, VARIATION
|
|
747
|
+
# SELECTOR-17 to -256, and the reserved code
|
|
748
|
+
# points between and after them
|
|
749
|
+
)
|
|
750
|
+
DEFAULT_IGNORABLE_COUNT = 4174
|
|
751
|
+
_DEFAULT_IGNORABLE = frozenset(
|
|
752
|
+
code_point for low, high in DEFAULT_IGNORABLE_RANGES
|
|
753
|
+
for code_point in range(low, high + 1))
|
|
754
|
+
# A pattern that matches nothing, for a mapping whose every surface form
|
|
755
|
+
# is made of characters a reader does not see.
|
|
756
|
+
_NEVER = re.compile(r"(?!x)x")
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
def is_default_ignorable(code_point: int) -> bool:
|
|
760
|
+
"""Whether Unicode's Default_Ignorable_Code_Point property holds."""
|
|
761
|
+
return code_point in _DEFAULT_IGNORABLE
|
|
762
|
+
|
|
763
|
+
|
|
764
|
+
# The sweep below, as one `str.translate` table: every code point in
|
|
765
|
+
# `_DEFAULT_IGNORABLE` and every code point the RUNNING interpreter puts
|
|
766
|
+
# in category Cf, each mapped to None. Built on the first non-ASCII call
|
|
767
|
+
# and held here, never at import: enumerating Cf over the whole code
|
|
768
|
+
# point space costs tens of milliseconds once, which a server that only
|
|
769
|
+
# ever sees ASCII should not pay (v0.13). Two threads that race to build
|
|
770
|
+
# it build the same table and the second assignment replaces the first
|
|
771
|
+
# with an equal one, so no lock is needed.
|
|
772
|
+
_UNSEEN_TABLE: Optional[Dict[int, None]] = None
|
|
773
|
+
# The same code points as one compiled character class, which is what
|
|
774
|
+
# `_strip_unseen` applies since fix round 2 (B-1): `str.translate` with a
|
|
775
|
+
# table of four thousand entries looks every character of a non-ASCII
|
|
776
|
+
# text up in a dictionary, 36 to 42 ms per MB on Python 3.13.5, while one
|
|
777
|
+
# character class applied with `re.sub` costs about 6 and gives the same
|
|
778
|
+
# string (pinned). Built with the table, on first use.
|
|
779
|
+
_UNSEEN_PATTERN: Optional["re.Pattern[str]"] = None
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
# The combining marks (categories Mn, Mc and Me of the running
|
|
783
|
+
# interpreter), as ranges; found in the same one sweep of the code points
|
|
784
|
+
# as the Cf characters above, and made into a pattern on first use.
|
|
785
|
+
_MARK_RANGES: Optional[List[List[int]]] = None
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
# The non-starters (fix round 3, the second re-verification's B2-1): the
|
|
789
|
+
# code points that decompose, under NFKD, to combining marks with a
|
|
790
|
+
# non-zero combining class only, which canonical reordering sorts among
|
|
791
|
+
# themselves. Found in the same one sweep, as ranges.
|
|
792
|
+
_NON_STARTER_RANGES: Optional[List[List[int]]] = None
|
|
793
|
+
|
|
794
|
+
# What each code point adds to the length the count reads (fix round 6,
|
|
795
|
+
# the fourth re-verification's B4-1), found in the same one sweep: for a
|
|
796
|
+
# code point whose NFKC, casefolded, is longer than one character, that
|
|
797
|
+
# length less one (U+FDFA adds 17). And where a composition with a mark
|
|
798
|
+
# folds longer than its two parts (six Greek letters with dialytika and
|
|
799
|
+
# an accent: U+03CA and U+03CB with an acute, a grave or a perispomeni),
|
|
800
|
+
# the letter it is built on adds the difference, so the sum over a text
|
|
801
|
+
# bounds its reading, composed or not. Made into a pattern on first use.
|
|
802
|
+
_READING_EXTRA: Optional[Dict[int, int]] = None
|
|
803
|
+
_READING_EXTRA_PATTERN: Optional["re.Pattern[str]"] = None
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _add_to_ranges(ranges: List[List[int]], code_point: int) -> None:
|
|
807
|
+
"""Append `code_point` to sorted `ranges`, merging a neighbour."""
|
|
808
|
+
if ranges and code_point == ranges[-1][1] + 1:
|
|
809
|
+
ranges[-1][1] = code_point
|
|
810
|
+
else:
|
|
811
|
+
ranges.append([code_point, code_point])
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def _sweep_code_points() -> None:
|
|
815
|
+
"""One pass over every code point: the Cf characters for the unseen
|
|
816
|
+
table, the combining marks for `_mark_run_pattern`, and the
|
|
817
|
+
non-starters for `_ordered_runs`. About a tenth of a second once per
|
|
818
|
+
process, paid on the first text that is not ASCII and never at
|
|
819
|
+
import."""
|
|
820
|
+
global _UNSEEN_TABLE, _MARK_RANGES, _NON_STARTER_RANGES, _READING_EXTRA
|
|
821
|
+
table = dict.fromkeys(_DEFAULT_IGNORABLE)
|
|
822
|
+
marks: List[List[int]] = []
|
|
823
|
+
non_starters: List[List[int]] = []
|
|
824
|
+
readings: Dict[int, str] = {}
|
|
825
|
+
deficit: Dict[str, int] = {}
|
|
826
|
+
category = unicodedata.category
|
|
827
|
+
combining = unicodedata.combining
|
|
828
|
+
decomposition = unicodedata.decomposition
|
|
829
|
+
for code_point in range(0x110000):
|
|
830
|
+
char = chr(code_point)
|
|
831
|
+
kind = category(char)
|
|
832
|
+
if kind == "Cf":
|
|
833
|
+
table[code_point] = None
|
|
834
|
+
elif kind[0] == "M":
|
|
835
|
+
_add_to_ranges(marks, code_point)
|
|
836
|
+
parts = decomposition(char)
|
|
837
|
+
if combining(char) or (parts and all(
|
|
838
|
+
combining(part) for part in
|
|
839
|
+
unicodedata.normalize("NFKD", char))):
|
|
840
|
+
_add_to_ranges(non_starters, code_point)
|
|
841
|
+
# Only a letter folds to more than one character, and only a
|
|
842
|
+
# character with a decomposition normalises to more than one.
|
|
843
|
+
if parts or kind in ("Ll", "Lu", "Lt"):
|
|
844
|
+
readings[code_point] = unicodedata.normalize(
|
|
845
|
+
"NFKC", char).casefold()
|
|
846
|
+
if parts and not parts.startswith("<") and " " in parts:
|
|
847
|
+
# Any number of parts: a canonical decomposition is two in
|
|
848
|
+
# the Unicode data, but some interpreters' `decomposition()`
|
|
849
|
+
# also spells a Hangul syllable as its two or three jamo
|
|
850
|
+
# (Python 3.13.15 on CI; 3.13.5 and 3.13.7 return nothing).
|
|
851
|
+
pieces = [chr(int(part, 16)) for part in parts.split()]
|
|
852
|
+
short = len(char.casefold()) - sum(len(piece.casefold())
|
|
853
|
+
for piece in pieces)
|
|
854
|
+
if short > 0:
|
|
855
|
+
deficit[pieces[0]] = max(deficit.get(pieces[0], 0), short)
|
|
856
|
+
extra: Dict[int, int] = {}
|
|
857
|
+
for code_point, reading in readings.items():
|
|
858
|
+
added = len(reading) - 1 + (deficit.get(reading[-1], 0)
|
|
859
|
+
if reading else 0)
|
|
860
|
+
if added > 0:
|
|
861
|
+
extra[code_point] = added
|
|
862
|
+
_MARK_RANGES = marks
|
|
863
|
+
_NON_STARTER_RANGES = non_starters
|
|
864
|
+
_READING_EXTRA = extra
|
|
865
|
+
_UNSEEN_TABLE = table
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
# A run of non-starters longer than this is put in canonical order before
|
|
869
|
+
# the normalisation (fix round 3, B2-1); UAX #15's stream-safe bound.
|
|
870
|
+
LONG_RUN = 30
|
|
871
|
+
_LONG_RUN_PATTERN: Optional[Tuple["re.Pattern[str]", "re.Pattern[str]"]] = None
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _ordered_runs(text: str) -> str:
|
|
875
|
+
"""`text` with every run of more than 30 non-starters decomposed and
|
|
876
|
+
put in canonical order in advance, so NFKC reads it in one pass.
|
|
877
|
+
|
|
878
|
+
Fix round 3 (the second re-verification's B2-1). Canonical reordering
|
|
879
|
+
inside NFKC is quadratic in the length of a run of combining marks
|
|
880
|
+
out of order, and no budget saw it: one crafted file of 320,000 such
|
|
881
|
+
marks cost a preview 36 seconds for 2.8 per cent of the work budget.
|
|
882
|
+
Each such run is replaced by its own NFKD, character by character,
|
|
883
|
+
sorted by combining class with a stable sort, which is what NFKD's
|
|
884
|
+
reordering does to it; so NFKC of the result is NFKC of the text,
|
|
885
|
+
exactly (pinned), and only the cost changes. A text already in NFKD
|
|
886
|
+
has every run in order and is returned as it stands; ordinary text
|
|
887
|
+
never has a run of more than a few marks.
|
|
888
|
+
"""
|
|
889
|
+
global _LONG_RUN_PATTERN
|
|
890
|
+
if text.isascii() or unicodedata.is_normalized("NFKD", text):
|
|
891
|
+
return text
|
|
892
|
+
patterns = _LONG_RUN_PATTERN
|
|
893
|
+
if patterns is None:
|
|
894
|
+
if _NON_STARTER_RANGES is None:
|
|
895
|
+
_sweep_code_points()
|
|
896
|
+
# The BMP non-starters exactly, and every astral code point: a
|
|
897
|
+
# character class with seventy astral ranges is tested range by
|
|
898
|
+
# range, twenty times dearer per character than this one, and the
|
|
899
|
+
# few astral code points are sorted out exactly below. More than
|
|
900
|
+
# `LONG_RUN` (30) in a row: 31 or more.
|
|
901
|
+
patterns = _LONG_RUN_PATTERN = (
|
|
902
|
+
re.compile("[" + "".join(
|
|
903
|
+
re.escape(chr(low)) if low == high
|
|
904
|
+
else re.escape(chr(low)) + "-" + re.escape(chr(high))
|
|
905
|
+
for low, high in _NON_STARTER_RANGES
|
|
906
|
+
if high <= 0xFFFF) + "\U00010000-\U0010ffff]"),
|
|
907
|
+
re.compile("[" + "".join(
|
|
908
|
+
re.escape(chr(low)) if low == high
|
|
909
|
+
else re.escape(chr(low)) + "-" + re.escape(chr(high))
|
|
910
|
+
for low, high in _NON_STARTER_RANGES
|
|
911
|
+
if high <= 0xFFFF) + "\U00010000-\U0010ffff]{31,}"))
|
|
912
|
+
first = patterns[0].search(text)
|
|
913
|
+
if first is None:
|
|
914
|
+
return text
|
|
915
|
+
start = first.start()
|
|
916
|
+
return text[:start] + patterns[1].sub(_ordered_run, text[start:])
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def _is_non_starter(char: str) -> bool:
|
|
920
|
+
"""Whether `char` is one of the non-starters of the sweep."""
|
|
921
|
+
ranges = _NON_STARTER_RANGES or []
|
|
922
|
+
code_point = ord(char)
|
|
923
|
+
index = bisect_right(ranges, [code_point, 0x110000]) - 1
|
|
924
|
+
return index >= 0 and ranges[index][0] <= code_point <= ranges[index][1]
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
def _ordered_run(match: "re.Match[str]") -> str:
|
|
928
|
+
"""Each maximal run of true non-starters in `match` (the superset may
|
|
929
|
+
have run on across astral code points that are not), when longer
|
|
930
|
+
than `LONG_RUN`, decomposed character by character and sorted by
|
|
931
|
+
combining class, stably."""
|
|
932
|
+
out: List[str] = []
|
|
933
|
+
segment: List[str] = []
|
|
934
|
+
|
|
935
|
+
def flush() -> None:
|
|
936
|
+
if len(segment) > LONG_RUN:
|
|
937
|
+
parts = [part for char in segment
|
|
938
|
+
for part in unicodedata.normalize("NFKD", char)]
|
|
939
|
+
parts.sort(key=unicodedata.combining)
|
|
940
|
+
out.append("".join(parts))
|
|
941
|
+
else:
|
|
942
|
+
out.append("".join(segment))
|
|
943
|
+
segment.clear()
|
|
944
|
+
|
|
945
|
+
for char in match.group(0):
|
|
946
|
+
if _is_non_starter(char):
|
|
947
|
+
segment.append(char)
|
|
948
|
+
else:
|
|
949
|
+
flush()
|
|
950
|
+
out.append(char)
|
|
951
|
+
flush()
|
|
952
|
+
return "".join(out)
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def _unseen_table() -> Dict[int, None]:
|
|
956
|
+
"""The code points `_strip_unseen` removes, as a translation table,
|
|
957
|
+
built once."""
|
|
958
|
+
table = _UNSEEN_TABLE
|
|
959
|
+
if table is None:
|
|
960
|
+
_sweep_code_points()
|
|
961
|
+
table = _UNSEEN_TABLE
|
|
962
|
+
return table
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def _unseen_pattern() -> "re.Pattern[str]":
|
|
966
|
+
"""`_unseen_table()`'s code points as one character class, built once.
|
|
967
|
+
|
|
968
|
+
Every run of consecutive code points becomes one range, each end
|
|
969
|
+
through `re.escape`, so nothing in the class is ever read as
|
|
970
|
+
syntax.
|
|
971
|
+
"""
|
|
972
|
+
global _UNSEEN_PATTERN
|
|
973
|
+
pattern = _UNSEEN_PATTERN
|
|
974
|
+
if pattern is None:
|
|
975
|
+
ranges: List[List[int]] = []
|
|
976
|
+
for code_point in sorted(_unseen_table()):
|
|
977
|
+
if ranges and code_point == ranges[-1][1] + 1:
|
|
978
|
+
ranges[-1][1] = code_point
|
|
979
|
+
else:
|
|
980
|
+
ranges.append([code_point, code_point])
|
|
981
|
+
pattern = _UNSEEN_PATTERN = re.compile(
|
|
982
|
+
"[" + "".join(
|
|
983
|
+
re.escape(chr(low)) if low == high
|
|
984
|
+
else re.escape(chr(low)) + "-" + re.escape(chr(high))
|
|
985
|
+
for low, high in ranges) + "]")
|
|
986
|
+
return pattern
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
def _strip_unseen(text: str) -> str:
|
|
990
|
+
"""`text` less every character a reader does not see, unnormalised.
|
|
991
|
+
|
|
992
|
+
The property above, and category Cf beside it. The format
|
|
993
|
+
characters outside the property (the Arabic, Syriac and Kaithi
|
|
994
|
+
number signs, the interlinear annotation controls, the Egyptian
|
|
995
|
+
hieroglyph format controls) are visible marks by Unicode's own
|
|
996
|
+
account, so stripping them is a widening rather than the property;
|
|
997
|
+
it is kept because a wider reading here is the safe one and the
|
|
998
|
+
round before this one already read that wide. Unlike the table,
|
|
999
|
+
the sweep reads the interpreter's own Unicode tables, so which
|
|
1000
|
+
visible format marks it removes beyond the property varies with
|
|
1001
|
+
the interpreter, in the over-detect direction only (fix round 5,
|
|
1002
|
+
D-1 note); the table does not move.
|
|
1003
|
+
|
|
1004
|
+
How the sweep is applied, since v0.13, and nothing about what it
|
|
1005
|
+
removes: pure ASCII is returned as it stands, because no ASCII code
|
|
1006
|
+
point is default-ignorable or in category Cf; anything else goes
|
|
1007
|
+
through `_unseen_pattern()`, the property and the interpreter's own
|
|
1008
|
+
Cf sweep united in one character class (fix round 2, B-1; it was a
|
|
1009
|
+
`str.translate` table, which cost six times as much per character).
|
|
1010
|
+
The output is the per-character generator's, byte for byte (pinned
|
|
1011
|
+
against a copy of that generator kept in the tests), at a fraction
|
|
1012
|
+
of its cost.
|
|
1013
|
+
"""
|
|
1014
|
+
if text.isascii():
|
|
1015
|
+
return text
|
|
1016
|
+
return _unseen_pattern().sub("", text)
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def _reader_sees(value: str) -> str:
|
|
1020
|
+
"""`value` as a reader sees it.
|
|
1021
|
+
|
|
1022
|
+
Compatibility normalisation (NFKC), so a fullwidth or otherwise
|
|
1023
|
+
compatibility-equivalent spelling reads as the plain one, and the
|
|
1024
|
+
default-ignorable code points removed, so an invisible character
|
|
1025
|
+
inside a word does not split it. Applied to every VALUE the detector
|
|
1026
|
+
is asked about; the forms go through `_detector_parts`, which
|
|
1027
|
+
splits before it strips. What this does NOT do, stated because the
|
|
1028
|
+
residue prose says so too: a look-alike letter from another script
|
|
1029
|
+
(a Cyrillic "о" for a Latin "o") is a different code point after
|
|
1030
|
+
every normalisation there is, and is out of scope. Spaces of every
|
|
1031
|
+
width are spaces: NFKC turns a thin or hair space into a plain one,
|
|
1032
|
+
and a plain one is a separator, not an invisible character.
|
|
1033
|
+
"""
|
|
1034
|
+
return _strip_unseen(unicodedata.normalize("NFKC", _ordered_runs(value)))
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
def _detector_parts(form: str) -> List[str]:
|
|
1038
|
+
"""The words of one surface form, as a reader sees them.
|
|
1039
|
+
|
|
1040
|
+
Split FIRST, on the normalised form, and strip inside each word
|
|
1041
|
+
afterwards. The order matters and was wrong in fix round 3, which
|
|
1042
|
+
stripped before splitting: `Mary<ZWSP>Ann` fused into the one word
|
|
1043
|
+
`MaryAnn`, which no longer found "Mary Ann" or "Mary_Ann.txt", a
|
|
1044
|
+
narrower reading than the round before it had (fix round 4, R1). An
|
|
1045
|
+
invisible character between two words is a separator to whoever
|
|
1046
|
+
put it there (a soft hyphen at a line break, a zero-width space
|
|
1047
|
+
from a web page); one inside a word that `\\w` counts as a letter (a
|
|
1048
|
+
Hangul filler) is nothing at all, and stripping it inside the word
|
|
1049
|
+
reads the word whole. Either way the reading is at least as wide as
|
|
1050
|
+
the split alone gave.
|
|
1051
|
+
"""
|
|
1052
|
+
text = unicodedata.normalize("NFKC", form)
|
|
1053
|
+
return [part for part in (_strip_unseen(piece)
|
|
1054
|
+
for piece in _SEPARATOR_SPLIT.split(text))
|
|
1055
|
+
if part]
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def _detector_alternative(form: str, fold: bool = False) -> str:
|
|
1059
|
+
"""One surface form as a pattern that survives a changed separator.
|
|
1060
|
+
|
|
1061
|
+
A form of one word is itself, escaped. A form of several is its
|
|
1062
|
+
words escaped and joined by "any run of separators, or none", so a
|
|
1063
|
+
transcript called `Mary_Ann.txt` and a case called `MaryAnn` are
|
|
1064
|
+
both found. A form with no letters or digits at all (which
|
|
1065
|
+
validation permits: two punctuation marks pass the length rule) has
|
|
1066
|
+
no words to join and is used literally; one made entirely of
|
|
1067
|
+
characters a reader does not see comes back empty, and the caller
|
|
1068
|
+
drops it. With `fold`, the casefolded reading of the same words.
|
|
1069
|
+
"""
|
|
1070
|
+
parts = _detector_parts(form)
|
|
1071
|
+
if fold:
|
|
1072
|
+
parts = [_fold(part) for part in parts]
|
|
1073
|
+
if parts:
|
|
1074
|
+
return _SEPARATOR_RUN.join(re.escape(part) for part in parts)
|
|
1075
|
+
literal = _reader_sees(form)
|
|
1076
|
+
return re.escape(_fold(literal) if fold else literal)
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def _detector_pattern(forms: Sequence[str], fold: bool = False) -> str:
|
|
1080
|
+
"""The alternation over `forms`, empty alternatives left out."""
|
|
1081
|
+
return "|".join(alternative for alternative in
|
|
1082
|
+
(_detector_alternative(form, fold) for form in forms)
|
|
1083
|
+
if alternative)
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
class NameDetector:
|
|
1087
|
+
"""Would a human reading this string see one of the mapping's names?
|
|
1088
|
+
|
|
1089
|
+
The DETECTOR, and it is deliberately NOT the rewriter's matcher. The
|
|
1090
|
+
two answer different questions under opposite correctness conditions
|
|
1091
|
+
and must never be swapped for one another:
|
|
1092
|
+
|
|
1093
|
+
- `Compiled.pattern` is the REWRITER's matcher: master's file-import
|
|
1094
|
+
form, whole-word on both sides, so "Ann" is never replaced inside
|
|
1095
|
+
"Anna" and an unrelated word is never corrupted. Conservative is
|
|
1096
|
+
right there, and nothing about it changes.
|
|
1097
|
+
- This class decides whether a string is REPORTED as still carrying
|
|
1098
|
+
a name, or WITHHELD from a durable record because it carries one.
|
|
1099
|
+
Both are questions about what a reader sees, not about what the
|
|
1100
|
+
rewrite would fire on, so it is liberal by design: a bare
|
|
1101
|
+
substring with no word boundary at all, because `_` is a word
|
|
1102
|
+
character and `Thomas_interview.txt` is the commonest transcript
|
|
1103
|
+
file name there is; case-insensitive under EVERY `case_mode`,
|
|
1104
|
+
because `THOMAS_P01` names the participant whether or not this run
|
|
1105
|
+
would rewrite it; over originals and variants alike.
|
|
1106
|
+
|
|
1107
|
+
The asymmetry is the whole point. Over-detecting costs a withheld
|
|
1108
|
+
file name, which the file id replaces, or a residue count that is
|
|
1109
|
+
one too high, which sends the researcher to look at a label that
|
|
1110
|
+
does mention the name. Under-detecting costs a participant's real
|
|
1111
|
+
name in a journal entry that ships inside the project.
|
|
1112
|
+
|
|
1113
|
+
The cost is real and is stated rather than hidden: an entry for
|
|
1114
|
+
"Tom" makes a memo that says "tomorrow" count as carrying a name,
|
|
1115
|
+
because a rule that catches `ThomasB.txt` cannot also spare
|
|
1116
|
+
`tomorrow`, and of the two mistakes only one of them ships a
|
|
1117
|
+
participant's name. The tool description says the residue counts
|
|
1118
|
+
read wider than the rewrite.
|
|
1119
|
+
"""
|
|
1120
|
+
|
|
1121
|
+
__slots__ = ("_direct", "_folded")
|
|
1122
|
+
|
|
1123
|
+
def __init__(self, forms: Sequence[str]):
|
|
1124
|
+
# Each form as a reader sees it, split into its words before
|
|
1125
|
+
# anything is stripped (`_detector_parts`), and never an empty
|
|
1126
|
+
# alternative: a form made entirely of invisible characters
|
|
1127
|
+
# (validation lets a zero-width space through, and two of them
|
|
1128
|
+
# pass the length rule) would otherwise match every string.
|
|
1129
|
+
if not any(_detector_alternative(form) for form in forms):
|
|
1130
|
+
self._direct = self._folded = _NEVER
|
|
1131
|
+
return
|
|
1132
|
+
self._direct = re.compile(_detector_pattern(forms), re.IGNORECASE)
|
|
1133
|
+
# A second reading, because `re.IGNORECASE` and `str.casefold` do
|
|
1134
|
+
# not agree on every code point (U+00DF casefolds to "ss" but
|
|
1135
|
+
# does not match "SS" under IGNORECASE). Two searches over one
|
|
1136
|
+
# short string cost nothing, and here the wider answer is the
|
|
1137
|
+
# safe one.
|
|
1138
|
+
self._folded = re.compile(_detector_pattern(forms, fold=True))
|
|
1139
|
+
|
|
1140
|
+
def contains(self, value: Any) -> bool:
|
|
1141
|
+
"""True when a reader of `value` would see any surface form.
|
|
1142
|
+
|
|
1143
|
+
Anything that is not a non-empty string is False: a NULL label
|
|
1144
|
+
carries no name, and nothing downstream should have to know that
|
|
1145
|
+
a missing value and a clean one are different answers.
|
|
1146
|
+
"""
|
|
1147
|
+
if not isinstance(value, str) or not value:
|
|
1148
|
+
return False
|
|
1149
|
+
text = _reader_sees(value)
|
|
1150
|
+
return bool(self._direct.search(text)
|
|
1151
|
+
or self._folded.search(_fold(text)))
|
|
1152
|
+
|
|
1153
|
+
|
|
1154
|
+
# `re.IGNORECASE` compares a literal character by character: a text
|
|
1155
|
+
# character matches a cased pattern character when the two have the same
|
|
1156
|
+
# simple lower case, or when that lower case is one of a few groups the
|
|
1157
|
+
# module treats as one letter (i and dotless i, s and long s, the Greek
|
|
1158
|
+
# letters with two lower cases). `str.casefold` is a different mapping
|
|
1159
|
+
# (U+0130 casefolds to two code points; U+00DF to two letters), which is
|
|
1160
|
+
# why a match the pattern accepts can miss the casefolded dictionary
|
|
1161
|
+
# (the re-verification's B-3, Turkish capitals under an insensitive
|
|
1162
|
+
# mode). The key below reproduces the module's own comparison, from the
|
|
1163
|
+
# same functions its compiler calls, so the forms a match can be sharing
|
|
1164
|
+
# a key with are found by one dictionary lookup. It is a short cut, never
|
|
1165
|
+
# the authority: every candidate is confirmed by a full match of its
|
|
1166
|
+
# compiled pattern, and a key the table cannot build (an interpreter
|
|
1167
|
+
# without these private functions) finds nothing and the exact scan
|
|
1168
|
+
# below it runs instead (fix round 2).
|
|
1169
|
+
try: # pragma: no cover - import
|
|
1170
|
+
import _sre
|
|
1171
|
+
_SRE_TOLOWER = getattr(_sre, "unicode_tolower", None)
|
|
1172
|
+
_SRE_ISCASED = getattr(_sre, "unicode_iscased", None)
|
|
1173
|
+
except ImportError: # pragma: no cover
|
|
1174
|
+
_SRE_TOLOWER = _SRE_ISCASED = None
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def _ignorecase_groups() -> Dict[int, Tuple[int, ...]]:
|
|
1178
|
+
"""The module's extra case groups, keyed by lower case."""
|
|
1179
|
+
try:
|
|
1180
|
+
from re import _casefix # Python 3.11 and later
|
|
1181
|
+
return dict(_casefix._EXTRA_CASES)
|
|
1182
|
+
except (ImportError, AttributeError):
|
|
1183
|
+
pass
|
|
1184
|
+
try: # pragma: no cover - 3.10
|
|
1185
|
+
import warnings
|
|
1186
|
+
with warnings.catch_warnings():
|
|
1187
|
+
warnings.simplefilter("ignore")
|
|
1188
|
+
import sre_compile
|
|
1189
|
+
return dict(getattr(sre_compile, "_ignorecase_fixes", {}))
|
|
1190
|
+
except ImportError: # pragma: no cover
|
|
1191
|
+
return {}
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
class _IgnoreCaseTable(dict):
|
|
1195
|
+
"""A `str.translate` table that fills itself: each code point to the
|
|
1196
|
+
one standing for every character `re.IGNORECASE` treats as it."""
|
|
1197
|
+
|
|
1198
|
+
groups: Dict[int, Tuple[int, ...]] = {}
|
|
1199
|
+
|
|
1200
|
+
def __missing__(self, code_point: int) -> int:
|
|
1201
|
+
key = code_point
|
|
1202
|
+
if _SRE_TOLOWER is not None and _SRE_ISCASED(code_point):
|
|
1203
|
+
lower = _SRE_TOLOWER(code_point)
|
|
1204
|
+
key = min((lower,) + tuple(self.groups.get(lower, ())))
|
|
1205
|
+
self[code_point] = key
|
|
1206
|
+
return key
|
|
1207
|
+
|
|
1208
|
+
|
|
1209
|
+
_IGNORECASE_TABLE = _IgnoreCaseTable()
|
|
1210
|
+
_IgnoreCaseTable.groups = _ignorecase_groups()
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
def ignorecase_key(value: str) -> str:
|
|
1214
|
+
"""`value` with each character replaced by its `re.IGNORECASE` key."""
|
|
1215
|
+
return value.translate(_IGNORECASE_TABLE)
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
def _word_key(value: str) -> str:
|
|
1219
|
+
"""How the file-text count compares words: `ignorecase_key`, or
|
|
1220
|
+
casefolding where the interpreter lacks what that key is built from."""
|
|
1221
|
+
if _SRE_TOLOWER is not None:
|
|
1222
|
+
return ignorecase_key(value)
|
|
1223
|
+
return _fold(value) # pragma: no cover
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
class Compiled:
|
|
1227
|
+
"""A mapping compiled into one pattern, plus the lookups it needs."""
|
|
1228
|
+
|
|
1229
|
+
__slots__ = ("mapping", "case_mode", "flags", "pattern", "forms",
|
|
1230
|
+
"_exact", "_folded", "_by_first", "max_form_len",
|
|
1231
|
+
"pseudonym_pattern", "_pseudonym_entries", "detector",
|
|
1232
|
+
"_text_lookup", "_entry_memo", "_form_patterns",
|
|
1233
|
+
"_by_key", "_by_length", "_form_units")
|
|
1234
|
+
|
|
1235
|
+
def __init__(self, mapping: Mapping):
|
|
1236
|
+
self.mapping = mapping
|
|
1237
|
+
self.case_mode = mapping.case_mode
|
|
1238
|
+
self.flags = 0 if mapping.case_mode == "exact" else re.IGNORECASE
|
|
1239
|
+
forms: List[Tuple[str, int]] = []
|
|
1240
|
+
for entry in mapping.entries:
|
|
1241
|
+
for form in entry.forms:
|
|
1242
|
+
forms.append((form, entry.index))
|
|
1243
|
+
forms.sort(key=_form_sort_key)
|
|
1244
|
+
self.forms = tuple(forms)
|
|
1245
|
+
self.max_form_len = max(len(form) for form, _ in forms)
|
|
1246
|
+
# The master file-import form, verbatim: a Unicode-aware word
|
|
1247
|
+
# boundary on both sides and every surface form escaped, so no
|
|
1248
|
+
# caller string ever reaches the regex engine as syntax
|
|
1249
|
+
# (manage_files.py:3348; the survey importer at :134-140 still
|
|
1250
|
+
# interpolates unescaped, which is the bug this parity does not
|
|
1251
|
+
# inherit).
|
|
1252
|
+
self.pattern = re.compile(
|
|
1253
|
+
"(?<!\\w)(?:" + "|".join(re.escape(form) for form, _ in forms)
|
|
1254
|
+
+ ")(?!\\w)", self.flags)
|
|
1255
|
+
# The other matcher, with the opposite contract: never used to
|
|
1256
|
+
# rewrite, always used to decide whether a string is reported or
|
|
1257
|
+
# withheld as carrying a name. See `NameDetector`.
|
|
1258
|
+
self.detector = NameDetector([form for form, _ in forms])
|
|
1259
|
+
self._exact = {form: index for form, index in forms}
|
|
1260
|
+
self._folded: Dict[str, int] = {}
|
|
1261
|
+
for form, index in forms:
|
|
1262
|
+
self._folded.setdefault(_fold(form), index)
|
|
1263
|
+
self._by_first: Dict[str, List[Tuple[str, int]]] = {}
|
|
1264
|
+
for form, index in forms:
|
|
1265
|
+
self._by_first.setdefault(_fold(form[0]), []).append((form, index))
|
|
1266
|
+
|
|
1267
|
+
pseudonyms = sorted({entry.pseudonym for entry in mapping.entries},
|
|
1268
|
+
key=lambda p: (-len(p), p))
|
|
1269
|
+
self.pseudonym_pattern = re.compile(
|
|
1270
|
+
"(?<!\\w)(?:" + "|".join(re.escape(p) for p in pseudonyms)
|
|
1271
|
+
+ ")(?!\\w)", self.flags)
|
|
1272
|
+
self._pseudonym_entries: Dict[str, List[int]] = {}
|
|
1273
|
+
for entry in mapping.entries:
|
|
1274
|
+
key = _fold(entry.pseudonym) if self.flags else entry.pseudonym
|
|
1275
|
+
self._pseudonym_entries.setdefault(key, []).append(entry.index)
|
|
1276
|
+
# Built on first use by the file-text count, never here: nothing
|
|
1277
|
+
# on the rewrite path needs it (v0.13).
|
|
1278
|
+
self._text_lookup: Optional["_TextLookup"] = None
|
|
1279
|
+
# Fix round 2 (B-3): which entry a matched spelling belongs to is
|
|
1280
|
+
# decided once per spelling, and the per-form patterns the exact
|
|
1281
|
+
# fall-back full-matches are compiled once each and held here,
|
|
1282
|
+
# rather than recompiled through `re`'s cache of 512, which a
|
|
1283
|
+
# mapping of 2,000 forms overflows on every call.
|
|
1284
|
+
self._entry_memo: Dict[str, int] = {}
|
|
1285
|
+
self._form_patterns: Dict[str, "re.Pattern[str]"] = {}
|
|
1286
|
+
self._by_key: Optional[Dict[str, List[Tuple[str, int]]]] = None
|
|
1287
|
+
self._by_length: Optional[Dict[int, List[Tuple[str, int]]]] = None
|
|
1288
|
+
self._form_units: Optional[int] = None
|
|
1289
|
+
|
|
1290
|
+
@property
|
|
1291
|
+
def form_units(self) -> int:
|
|
1292
|
+
"""The forms' part of `residue_work`'s price for one character,
|
|
1293
|
+
by their total length rather than their number (fix round 6, the
|
|
1294
|
+
fourth re-verification's B4-2): each form is a unit for every
|
|
1295
|
+
`FORM_UNIT_CHARS` characters it reads as, or part of them. Where
|
|
1296
|
+
the text runs along a prefix many forms share, the detector's
|
|
1297
|
+
alternation compares each of them as far as it goes, so a
|
|
1298
|
+
position costs up to the forms' total length, not their number; a
|
|
1299
|
+
name of up to ten characters is one unit, as every form was."""
|
|
1300
|
+
units = self._form_units
|
|
1301
|
+
if units is None:
|
|
1302
|
+
units = self._form_units = sum(
|
|
1303
|
+
-(-reading_length(form) // FORM_UNIT_CHARS)
|
|
1304
|
+
for form, _ in self.forms)
|
|
1305
|
+
return units
|
|
1306
|
+
|
|
1307
|
+
def text_lookup(self) -> "_TextLookup":
|
|
1308
|
+
"""What `names_left_in_text` needs from these forms, built once."""
|
|
1309
|
+
if self._text_lookup is None:
|
|
1310
|
+
self._text_lookup = _TextLookup(self)
|
|
1311
|
+
return self._text_lookup
|
|
1312
|
+
|
|
1313
|
+
def carries_a_name(self, value: Any) -> bool:
|
|
1314
|
+
"""Whether `value` shows a name to a reader OR to this run's rule.
|
|
1315
|
+
|
|
1316
|
+
The union of the two matchers, and the one predicate for every
|
|
1317
|
+
question of the form "is a name here": the wide count of a note,
|
|
1318
|
+
a label or an attribute value, whether a file shows a name, and
|
|
1319
|
+
whether a file name, a path or a pseudonym is withheld from a
|
|
1320
|
+
record. The detector alone reads NFKC, which composes a combining
|
|
1321
|
+
mark onto the letter before it, so a name followed by one ("Rene"
|
|
1322
|
+
typed decomposed as "René") is matched by the whole-word rule and
|
|
1323
|
+
not seen by the detector; the union keeps the wide reading at
|
|
1324
|
+
least as wide as the rewrite, which is what v0.12 founded it on,
|
|
1325
|
+
and makes every withholding withhold more, never less (lead's
|
|
1326
|
+
ruling of 2026-09-23 on QA-1).
|
|
1327
|
+
"""
|
|
1328
|
+
if not isinstance(value, str) or not value:
|
|
1329
|
+
return False
|
|
1330
|
+
return bool(self.detector.contains(value)
|
|
1331
|
+
or self.pattern.search(value))
|
|
1332
|
+
|
|
1333
|
+
def entry_for(self, matched: str) -> int:
|
|
1334
|
+
"""Which entry produced `matched`.
|
|
1335
|
+
|
|
1336
|
+
Three steps, fastest first, and the last is exact by
|
|
1337
|
+
construction: the combined pattern matched, so at least one
|
|
1338
|
+
escaped alternative full-matches the matched text under the same
|
|
1339
|
+
flags, and the alternatives are tested in the pattern's own
|
|
1340
|
+
order. The dictionaries are a short cut, not the authority,
|
|
1341
|
+
because `re.IGNORECASE` and `str.casefold` do not agree on every
|
|
1342
|
+
code point (U+00DF folds to "ss" but does not match "SS" under
|
|
1343
|
+
IGNORECASE).
|
|
1344
|
+
|
|
1345
|
+
Decided once per spelling (fix round 2, B-3): a transcript of a
|
|
1346
|
+
thousand speaker labels asks the same question a thousand times,
|
|
1347
|
+
and the last step, which the dictionaries miss under an
|
|
1348
|
+
insensitive mode for a spelling such as `ALİ`, cost 12 ms a match
|
|
1349
|
+
at 2,000 forms before it was memoised and indexed.
|
|
1350
|
+
"""
|
|
1351
|
+
found = self._entry_memo.get(matched)
|
|
1352
|
+
if found is not None:
|
|
1353
|
+
return found
|
|
1354
|
+
found = self._exact.get(matched)
|
|
1355
|
+
if found is None and self.flags:
|
|
1356
|
+
found = self._folded.get(_fold(matched))
|
|
1357
|
+
if found is None:
|
|
1358
|
+
found = self.first_full_match(matched)[1]
|
|
1359
|
+
self._entry_memo[matched] = found
|
|
1360
|
+
return found
|
|
1361
|
+
|
|
1362
|
+
def form_pattern(self, form: str) -> "re.Pattern[str]":
|
|
1363
|
+
"""One surface form, escaped and compiled under this case mode,
|
|
1364
|
+
once per form."""
|
|
1365
|
+
pattern = self._form_patterns.get(form)
|
|
1366
|
+
if pattern is None:
|
|
1367
|
+
pattern = self._form_patterns[form] = re.compile(
|
|
1368
|
+
re.escape(form), self.flags)
|
|
1369
|
+
return pattern
|
|
1370
|
+
|
|
1371
|
+
def first_full_match(self, matched: str) -> Tuple[str, int]:
|
|
1372
|
+
"""The first form, in the pattern's own order, whose escaped
|
|
1373
|
+
alternative full-matches `matched` under this case mode.
|
|
1374
|
+
|
|
1375
|
+
Exact, and the answer the combined pattern's alternation gives:
|
|
1376
|
+
a literal matches character for character under `re.IGNORECASE`
|
|
1377
|
+
as without it, so only the forms of `matched`'s length can match,
|
|
1378
|
+
and among the forms sharing its `ignorecase_key` the first that
|
|
1379
|
+
full-matches is the first of all (the key is the module's own
|
|
1380
|
+
comparison). The key is tried first; the forms of the same length
|
|
1381
|
+
are the fall-back, in the same order, so a key that found
|
|
1382
|
+
nothing never changes an answer.
|
|
1383
|
+
"""
|
|
1384
|
+
if self.flags:
|
|
1385
|
+
by_key = self._by_key
|
|
1386
|
+
if by_key is None:
|
|
1387
|
+
by_key = self._by_key = {}
|
|
1388
|
+
for form, index in self.forms:
|
|
1389
|
+
by_key.setdefault(ignorecase_key(form), []).append(
|
|
1390
|
+
(form, index))
|
|
1391
|
+
for form, index in by_key.get(ignorecase_key(matched), ()):
|
|
1392
|
+
if self.form_pattern(form).fullmatch(matched):
|
|
1393
|
+
return form, index
|
|
1394
|
+
by_length = self._by_length
|
|
1395
|
+
if by_length is None:
|
|
1396
|
+
by_length = self._by_length = {}
|
|
1397
|
+
for form, index in self.forms:
|
|
1398
|
+
by_length.setdefault(len(form), []).append((form, index))
|
|
1399
|
+
for form, index in by_length.get(len(matched), ()):
|
|
1400
|
+
if self.form_pattern(form).fullmatch(matched):
|
|
1401
|
+
return form, index
|
|
1402
|
+
raise AssertionError( # pragma: no cover - impossible
|
|
1403
|
+
"a matched span belongs to no surface form")
|
|
1404
|
+
|
|
1405
|
+
def forms_at(self, text: str, position: int) -> List[Tuple[str, int]]:
|
|
1406
|
+
"""Every surface form that matches, standalone, at `position`.
|
|
1407
|
+
|
|
1408
|
+
Used only by the overlap diagnostic, which asks what the combined
|
|
1409
|
+
pass did NOT choose. Bucketed by first character so the scan is
|
|
1410
|
+
proportional to the forms that could start here rather than to
|
|
1411
|
+
the whole mapping.
|
|
1412
|
+
"""
|
|
1413
|
+
if position >= len(text):
|
|
1414
|
+
return []
|
|
1415
|
+
found: List[Tuple[str, int]] = []
|
|
1416
|
+
for form, index in self._by_first.get(_fold(text[position]), ()):
|
|
1417
|
+
end = position + len(form)
|
|
1418
|
+
if end > len(text):
|
|
1419
|
+
continue
|
|
1420
|
+
candidate = text[position:end]
|
|
1421
|
+
if self.flags:
|
|
1422
|
+
# The regex, not a case-folded comparison: this decides
|
|
1423
|
+
# whether a form is REPORTED as having lost, so it has
|
|
1424
|
+
# to be the same test the matching pass itself applies.
|
|
1425
|
+
if not self.form_pattern(form).fullmatch(candidate):
|
|
1426
|
+
continue
|
|
1427
|
+
elif candidate != form:
|
|
1428
|
+
continue
|
|
1429
|
+
if position > 0 and _is_word_char(text[position - 1]):
|
|
1430
|
+
continue
|
|
1431
|
+
if end < len(text) and _is_word_char(text[end]):
|
|
1432
|
+
continue
|
|
1433
|
+
found.append((form, index))
|
|
1434
|
+
return found
|
|
1435
|
+
|
|
1436
|
+
def replacement_for(self, entry_index: int, matched: str) -> str:
|
|
1437
|
+
"""The text that replaces one match, under this case mode.
|
|
1438
|
+
|
|
1439
|
+
Under `insensitive_preserve` this is a HEURISTIC, and the tool
|
|
1440
|
+
description says so in the same word: it copies the shape of the
|
|
1441
|
+
matched text onto the pseudonym, which gets SHOUTED and
|
|
1442
|
+
lower-case passages right because that is what transcripts
|
|
1443
|
+
actually contain, and takes the pseudonym as written for
|
|
1444
|
+
anything else.
|
|
1445
|
+
"""
|
|
1446
|
+
pseudonym = self.mapping.entries[entry_index].pseudonym
|
|
1447
|
+
if self.case_mode != "insensitive_preserve":
|
|
1448
|
+
return pseudonym
|
|
1449
|
+
# Anything that is not all upper or all lower (mixed case, a
|
|
1450
|
+
# capitalised word) takes the pseudonym as the researcher wrote
|
|
1451
|
+
# it. The length guard is upstream-shaped rather than reachable:
|
|
1452
|
+
# no surface form is shorter than two characters.
|
|
1453
|
+
if len(matched) > 1 and matched.isupper():
|
|
1454
|
+
return pseudonym.upper()
|
|
1455
|
+
if matched.islower():
|
|
1456
|
+
return pseudonym.lower()
|
|
1457
|
+
return pseudonym
|
|
1458
|
+
|
|
1459
|
+
|
|
1460
|
+
_WORD_RE = re.compile(r"\w")
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
def _is_word_char(char: str) -> bool:
|
|
1464
|
+
"""`\\w` for one character, the way the boundary assertions read it."""
|
|
1465
|
+
return _WORD_RE.match(char) is not None
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
class Replacement:
|
|
1469
|
+
"""One edit: the span it covers and the text that takes its place."""
|
|
1470
|
+
|
|
1471
|
+
__slots__ = ("start", "end", "entry", "matched", "text")
|
|
1472
|
+
|
|
1473
|
+
def __init__(self, start: int, end: int, entry: int, matched: str,
|
|
1474
|
+
text: str):
|
|
1475
|
+
self.start = start
|
|
1476
|
+
self.end = end
|
|
1477
|
+
self.entry = entry
|
|
1478
|
+
self.matched = matched
|
|
1479
|
+
self.text = text
|
|
1480
|
+
|
|
1481
|
+
@property
|
|
1482
|
+
def delta(self) -> int:
|
|
1483
|
+
return len(self.text) - (self.end - self.start)
|
|
1484
|
+
|
|
1485
|
+
def __repr__(self) -> str: # pragma: no cover - debugging
|
|
1486
|
+
return f"Replacement({self.start}, {self.end}, entry={self.entry})"
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def find_replacements(compiled: Compiled, text: str) -> List[Replacement]:
|
|
1490
|
+
"""Every edit this mapping makes to `text`, left to right.
|
|
1491
|
+
|
|
1492
|
+
One `finditer` over one pattern, so the matches are non-overlapping
|
|
1493
|
+
and leftmost, and at each start the longest viable surface form wins.
|
|
1494
|
+
Nothing this pass produces is ever matched again, which is the
|
|
1495
|
+
single-pass guarantee.
|
|
1496
|
+
"""
|
|
1497
|
+
found: List[Replacement] = []
|
|
1498
|
+
for match in compiled.pattern.finditer(text):
|
|
1499
|
+
matched = match.group(0)
|
|
1500
|
+
entry = compiled.entry_for(matched)
|
|
1501
|
+
found.append(Replacement(match.start(), match.end(), entry, matched,
|
|
1502
|
+
compiled.replacement_for(entry, matched)))
|
|
1503
|
+
return found
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def apply_replacements(text: str, replacements: Sequence[Replacement]) -> str:
|
|
1507
|
+
"""The rewritten text."""
|
|
1508
|
+
if not replacements:
|
|
1509
|
+
return text
|
|
1510
|
+
pieces: List[str] = []
|
|
1511
|
+
cursor = 0
|
|
1512
|
+
for item in replacements:
|
|
1513
|
+
pieces.append(text[cursor:item.start])
|
|
1514
|
+
pieces.append(item.text)
|
|
1515
|
+
cursor = item.end
|
|
1516
|
+
pieces.append(text[cursor:])
|
|
1517
|
+
return "".join(pieces)
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
# --------------------------------------------------------------------------
|
|
1521
|
+
# Diagnostics (computed from the same pass)
|
|
1522
|
+
# --------------------------------------------------------------------------
|
|
1523
|
+
|
|
1524
|
+
def pre_existing_pseudonym_occurrences(
|
|
1525
|
+
compiled: Compiled, text: str,
|
|
1526
|
+
max_spans: Optional[int] = None) -> List[Dict[str, Any]]:
|
|
1527
|
+
"""Where each pseudonym ALREADY occurs in the text (D1 3.4).
|
|
1528
|
+
|
|
1529
|
+
A warning, never a refusal (owner ruling Q3): some researchers reuse
|
|
1530
|
+
common given names deliberately. What it costs is told plainly: after
|
|
1531
|
+
the run the rewritten text cannot distinguish a pseudonym this tool
|
|
1532
|
+
wrote from one that was always there, so a reversal driven by text
|
|
1533
|
+
would be unsafe. The run manifest records spans rather than strings
|
|
1534
|
+
for exactly this reason.
|
|
1535
|
+
"""
|
|
1536
|
+
counts: Dict[int, List[Tuple[int, int]]] = {}
|
|
1537
|
+
for match in compiled.pseudonym_pattern.finditer(text):
|
|
1538
|
+
matched = match.group(0)
|
|
1539
|
+
key = _fold(matched) if compiled.flags else matched
|
|
1540
|
+
for index in compiled._pseudonym_entries.get(key, ()):
|
|
1541
|
+
counts.setdefault(index, []).append((match.start(), match.end()))
|
|
1542
|
+
report: List[Dict[str, Any]] = []
|
|
1543
|
+
for index in sorted(counts):
|
|
1544
|
+
spans = counts[index]
|
|
1545
|
+
shown = spans if max_spans is None else spans[:max_spans]
|
|
1546
|
+
item: Dict[str, Any] = {
|
|
1547
|
+
"entry": index,
|
|
1548
|
+
"pseudonym": compiled.mapping.entries[index].pseudonym,
|
|
1549
|
+
"count": len(spans),
|
|
1550
|
+
"spans": [[s, e] for s, e in shown],
|
|
1551
|
+
}
|
|
1552
|
+
if max_spans is not None and len(spans) > len(shown):
|
|
1553
|
+
item["spans_truncated"] = True
|
|
1554
|
+
report.append(item)
|
|
1555
|
+
return report
|
|
1556
|
+
|
|
1557
|
+
|
|
1558
|
+
def overlap_conflicts(compiled: Compiled, text: str,
|
|
1559
|
+
replacements: Sequence[Replacement]
|
|
1560
|
+
) -> Tuple[List[Dict[str, Any]], bool, bool]:
|
|
1561
|
+
"""Surface forms that lost the competition for the same characters.
|
|
1562
|
+
|
|
1563
|
+
"Ann Marie" and "Marie Curie" over "Ann Marie Curie": leftmost and
|
|
1564
|
+
longest chooses "Ann Marie" and leaves "Curie" behind, so the second
|
|
1565
|
+
entry never fires there. The researcher has to see that, because
|
|
1566
|
+
nothing else in the preview would show it.
|
|
1567
|
+
|
|
1568
|
+
Only the neighbourhood of each chosen match is examined: a form can
|
|
1569
|
+
only conflict with a match it overlaps, so the scan is bounded by the
|
|
1570
|
+
number of matches times the longest surface form, never by the text.
|
|
1571
|
+
That can still be a great deal (the fourth re-verification's B4-3),
|
|
1572
|
+
so the forms examined are counted, and past
|
|
1573
|
+
`MAX_OVERLAP_CANDIDATES` the scan stops with what it has found.
|
|
1574
|
+
|
|
1575
|
+
Returns (conflicts, truncated, capped): truncated when
|
|
1576
|
+
`MAX_OVERLAP_CONFLICTS` conflicts were found, capped when the scan
|
|
1577
|
+
stopped at `MAX_OVERLAP_CANDIDATES` forms examined first.
|
|
1578
|
+
"""
|
|
1579
|
+
conflicts: List[Dict[str, Any]] = []
|
|
1580
|
+
truncated = False
|
|
1581
|
+
examined = 0
|
|
1582
|
+
by_first = compiled._by_first
|
|
1583
|
+
for item in replacements:
|
|
1584
|
+
first = max(0, item.start - compiled.max_form_len + 1)
|
|
1585
|
+
for position in range(first, item.end):
|
|
1586
|
+
examined += 1 + len(by_first.get(_fold(text[position]), ()))
|
|
1587
|
+
if examined > MAX_OVERLAP_CANDIDATES:
|
|
1588
|
+
return conflicts, truncated, True
|
|
1589
|
+
for form, index in compiled.forms_at(text, position):
|
|
1590
|
+
end = position + len(form)
|
|
1591
|
+
if end <= item.start:
|
|
1592
|
+
continue # ends before this match
|
|
1593
|
+
if index == item.entry:
|
|
1594
|
+
continue # the same entry, not a clash
|
|
1595
|
+
if position == item.start and end == item.end:
|
|
1596
|
+
continue # the very match that was chosen
|
|
1597
|
+
if len(conflicts) >= MAX_OVERLAP_CONFLICTS:
|
|
1598
|
+
return conflicts, True, False
|
|
1599
|
+
conflicts.append({
|
|
1600
|
+
"entry": index,
|
|
1601
|
+
"form": form,
|
|
1602
|
+
"span": [position, end],
|
|
1603
|
+
"loses_to_entry": item.entry,
|
|
1604
|
+
"chosen_span": [item.start, item.end],
|
|
1605
|
+
})
|
|
1606
|
+
return conflicts, truncated, False
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
def short_forms(mapping: Mapping) -> List[Dict[str, Any]]:
|
|
1610
|
+
"""The surface forms shorter than `SHORT_FORM_CHARS` (a warning).
|
|
1611
|
+
|
|
1612
|
+
Entry index and length only, never the form: this block travels on
|
|
1613
|
+
the sidecar path too, where a surface form is the researcher's
|
|
1614
|
+
reverse key. Length is in code points, the same measure the
|
|
1615
|
+
validator's minimums use.
|
|
1616
|
+
"""
|
|
1617
|
+
found: List[Dict[str, Any]] = []
|
|
1618
|
+
for entry in mapping.entries:
|
|
1619
|
+
for form in entry.forms:
|
|
1620
|
+
if len(form) < SHORT_FORM_CHARS:
|
|
1621
|
+
found.append({"entry": entry.index, "length": len(form)})
|
|
1622
|
+
return found
|
|
1623
|
+
|
|
1624
|
+
|
|
1625
|
+
def case_variants_seen(compiled: Compiled, text: str) -> List[Dict[str, Any]]:
|
|
1626
|
+
"""Occurrences that differ from every form only in case (D1 3.4).
|
|
1627
|
+
|
|
1628
|
+
Reported under `case_mode="exact"` only, where they are precisely the
|
|
1629
|
+
occurrences this run does NOT replace. Counts, never spans: the point
|
|
1630
|
+
is "TOM appears twelve times and is not being touched", and a span
|
|
1631
|
+
list would not add to it.
|
|
1632
|
+
"""
|
|
1633
|
+
if compiled.case_mode != "exact":
|
|
1634
|
+
return []
|
|
1635
|
+
insensitive = re.compile(compiled.pattern.pattern, re.IGNORECASE)
|
|
1636
|
+
pools: Dict[str, Dict[str, int]] = {}
|
|
1637
|
+
for match in insensitive.finditer(text):
|
|
1638
|
+
matched = match.group(0)
|
|
1639
|
+
pool = pools.setdefault(_fold(matched), {})
|
|
1640
|
+
pool[matched] = pool.get(matched, 0) + 1
|
|
1641
|
+
exact_forms = {form for form, _ in compiled.forms}
|
|
1642
|
+
report: List[Dict[str, Any]] = []
|
|
1643
|
+
for form, index in compiled.forms:
|
|
1644
|
+
pool = pools.get(_fold(form), {})
|
|
1645
|
+
other = sum(count for spelling, count in pool.items()
|
|
1646
|
+
if spelling not in exact_forms)
|
|
1647
|
+
if other:
|
|
1648
|
+
report.append({"entry": index, "form": form,
|
|
1649
|
+
"other_case_count": other})
|
|
1650
|
+
return report
|
|
1651
|
+
|
|
1652
|
+
|
|
1653
|
+
# --------------------------------------------------------------------------
|
|
1654
|
+
# Names left in the file text (v0.13, Brief 1 item 4)
|
|
1655
|
+
# --------------------------------------------------------------------------
|
|
1656
|
+
|
|
1657
|
+
# The kinds a wide occurrence in the file text is split into, in the
|
|
1658
|
+
# order the report lists them. `unattributed` is not a kind: it is the
|
|
1659
|
+
# occurrences no entry could be charged to, counted beside them.
|
|
1660
|
+
TEXT_RESIDUE_REASONS = (
|
|
1661
|
+
"inside_a_longer_word", "case_only", "joined_differently",
|
|
1662
|
+
"normalisation_variants", "put_back_by_a_pseudonym",
|
|
1663
|
+
"whole_word_in_a_file_not_rewritten")
|
|
1664
|
+
|
|
1665
|
+
# What `names_left_in_text` returns for one file, JSON-ready:
|
|
1666
|
+
# `occurrences` ({"wide": N, "whole_word": M}), `unattributed`,
|
|
1667
|
+
# `entries` (one row per entry with anything to report), and the two
|
|
1668
|
+
# spelling diagnostics `case_variants_seen` and
|
|
1669
|
+
# `normalisation_variants_seen`.
|
|
1670
|
+
TextResidue = Dict[str, Any]
|
|
1671
|
+
|
|
1672
|
+
_UNPLACED = object()
|
|
1673
|
+
|
|
1674
|
+
|
|
1675
|
+
def _residue_key(value: str) -> Tuple[str, str]:
|
|
1676
|
+
"""The attribution key of `value`: its reader words, joined, casefolded.
|
|
1677
|
+
|
|
1678
|
+
`Mary Ann`, `MaryAnn`, `Mary_Ann` and `MARY ANN` all key to
|
|
1679
|
+
`maryann`, which is exactly the equivalence the detector's
|
|
1680
|
+
alternation encodes (`_detector_parts` splits on separators after
|
|
1681
|
+
NFKC and strips the unseen characters inside each word). The key
|
|
1682
|
+
JOINS the words: one that kept the word split disagreed with the
|
|
1683
|
+
capture-group reference in 792 of 5,479 random runs in the counts
|
|
1684
|
+
study, and this one in none, so the joined key is the right one. A
|
|
1685
|
+
value with no letters or digits keys by its literal reading, in a
|
|
1686
|
+
namespace of its own, so it never collides with a word key.
|
|
1687
|
+
|
|
1688
|
+
Since fix round 2 the words are keyed the way `re.IGNORECASE`
|
|
1689
|
+
compares them (`ignorecase_key`), which is the comparison the direct
|
|
1690
|
+
pattern made, rather than casefolded: the two disagree on a few
|
|
1691
|
+
letters (a capital I with a dot above, a dotless i, a long s), and a
|
|
1692
|
+
match the pattern accepted and the casefolded key could not place was
|
|
1693
|
+
charged to no entry. Where the interpreter lacks the functions that
|
|
1694
|
+
key is built from, casefolding stands in, as before.
|
|
1695
|
+
"""
|
|
1696
|
+
parts = _detector_parts(value)
|
|
1697
|
+
if parts:
|
|
1698
|
+
return ("words", _word_key("".join(parts)))
|
|
1699
|
+
return ("literal", _word_key(_reader_sees(value)))
|
|
1700
|
+
|
|
1701
|
+
|
|
1702
|
+
_NOT_A_WORD_CHARACTER = re.compile(r"[\W_]+")
|
|
1703
|
+
|
|
1704
|
+
|
|
1705
|
+
def _direct_key(matched: str) -> Tuple[str, str]:
|
|
1706
|
+
"""The attribution key of a match of the DIRECT pattern.
|
|
1707
|
+
|
|
1708
|
+
The direct pattern runs over the reader's reading, already normalised
|
|
1709
|
+
and stripped, and each alternative is a form's words escaped and
|
|
1710
|
+
joined by separator runs, so a match is those words, in any case the
|
|
1711
|
+
pattern accepts, with separators between. Keyed as it stands with the
|
|
1712
|
+
separators taken out, never normalised again: `_residue_key` of the
|
|
1713
|
+
match composed a mark the reading had left apart (an invisible
|
|
1714
|
+
character between a letter and a combining accent stops the
|
|
1715
|
+
composition when the text is read; the strip then removes it, and
|
|
1716
|
+
NFKC of the match composes the two), so the key matched no form and
|
|
1717
|
+
the occurrence was charged to no entry (the second re-verification's
|
|
1718
|
+
CORR2-2, "Mary" U+200B U+0301 "Ann" under {"Mary Ann"}; fix round 2
|
|
1719
|
+
fixed the same mechanism in the folded pass). The key of each form,
|
|
1720
|
+
`_residue_key`, is the same words joined, so this is its key.
|
|
1721
|
+
"""
|
|
1722
|
+
words = _NOT_A_WORD_CHARACTER.sub("", matched)
|
|
1723
|
+
if words:
|
|
1724
|
+
return ("words", _word_key(words))
|
|
1725
|
+
return ("literal", _word_key(matched))
|
|
1726
|
+
|
|
1727
|
+
|
|
1728
|
+
def _folded_key(folded: str) -> Tuple[str, str]:
|
|
1729
|
+
"""The attribution key of a match of the FOLDED pattern, and of the
|
|
1730
|
+
folded reading of a form.
|
|
1731
|
+
|
|
1732
|
+
The folded pattern is the forms' words casefolded, escaped and
|
|
1733
|
+
joined by separator runs, compiled without flags, so a match of it
|
|
1734
|
+
is those casefolded words exactly, with separators between. Keyed
|
|
1735
|
+
as it stands with the separators taken out, never normalised again:
|
|
1736
|
+
fix round 1 keyed it through `_residue_key`, whose NFKC composed a
|
|
1737
|
+
combining mark in the text onto the casefolded letter before it
|
|
1738
|
+
("ß" followed by U+0301 folds to "ss" and then reads "sś"), so the
|
|
1739
|
+
key matched no form and the occurrence was charged to no entry (the
|
|
1740
|
+
re-verification's CORR-1; the property that asserted it never
|
|
1741
|
+
happens failed on about one random seed in twenty). A form's own
|
|
1742
|
+
folded words keyed the same way are the key of every match of its
|
|
1743
|
+
alternative, so this pass charges every match to an entry.
|
|
1744
|
+
"""
|
|
1745
|
+
words = _NOT_A_WORD_CHARACTER.sub("", folded)
|
|
1746
|
+
if words:
|
|
1747
|
+
return ("words", words)
|
|
1748
|
+
return ("literal", folded)
|
|
1749
|
+
|
|
1750
|
+
|
|
1751
|
+
def _form_folded_key(form: str) -> Tuple[str, str]:
|
|
1752
|
+
"""`_folded_key` of what a form's folded alternative matches."""
|
|
1753
|
+
parts = _detector_parts(form)
|
|
1754
|
+
if parts:
|
|
1755
|
+
return _folded_key("".join(_fold(part) for part in parts))
|
|
1756
|
+
return ("literal", _fold(_reader_sees(form)))
|
|
1757
|
+
|
|
1758
|
+
|
|
1759
|
+
class _TextLookup:
|
|
1760
|
+
"""What the file-text count needs from one compiled mapping.
|
|
1761
|
+
|
|
1762
|
+
Built once per `Compiled`, on first use, from the forms in the
|
|
1763
|
+
pattern's own order (`_form_sort_key`: longest first). Each key maps
|
|
1764
|
+
to every form that shares it, in that order; `_pick` breaks a tie
|
|
1765
|
+
the way the combined pattern breaks it.
|
|
1766
|
+
"""
|
|
1767
|
+
|
|
1768
|
+
__slots__ = ("direct", "folded", "reader_form", "all_ascii",
|
|
1769
|
+
"folded_form", "entry_forms", "rewriter_form",
|
|
1770
|
+
"seen_alone", "own_patterns")
|
|
1771
|
+
|
|
1772
|
+
def __init__(self, compiled: "Compiled"):
|
|
1773
|
+
self.direct: Dict[Tuple[str, str], List[Tuple[str, int, Any]]] = {}
|
|
1774
|
+
self.folded: Dict[Tuple[str, str], List[Tuple[str, int, Any]]] = {}
|
|
1775
|
+
self.reader_form: Dict[Tuple[int, str], str] = {}
|
|
1776
|
+
self.folded_form: Dict[Tuple[int, str], str] = {}
|
|
1777
|
+
self.entry_forms: Dict[int, List[str]] = {}
|
|
1778
|
+
# Per matched spelling, across every file of one preview (fix
|
|
1779
|
+
# round 2, B-3): the rewriter's form, and whether the detector
|
|
1780
|
+
# sees a whole word read with its trailing marks.
|
|
1781
|
+
self.rewriter_form: Dict[str, Tuple[int, str]] = {}
|
|
1782
|
+
self.seen_alone: Dict[str, bool] = {}
|
|
1783
|
+
# Each form's own two detector alternatives, direct and folded:
|
|
1784
|
+
# the cheap first question about a whole word the rewriter
|
|
1785
|
+
# matched, before the whole detector is asked.
|
|
1786
|
+
self.own_patterns: Dict[Tuple[int, str], Tuple[Any, Any]] = {}
|
|
1787
|
+
for form, index in compiled.forms:
|
|
1788
|
+
self.entry_forms.setdefault(index, []).append(form)
|
|
1789
|
+
self.folded_form.setdefault((index, _fold(form)), form)
|
|
1790
|
+
if not _detector_alternative(form):
|
|
1791
|
+
continue # a form no reader can see matches nothing
|
|
1792
|
+
# Each pattern built from the escaping builder at the site
|
|
1793
|
+
# that compiles it, as every pattern in this module is.
|
|
1794
|
+
direct = re.compile(_detector_alternative(form), re.IGNORECASE)
|
|
1795
|
+
folded = re.compile(_detector_alternative(form, fold=True))
|
|
1796
|
+
self.own_patterns[(index, form)] = (direct, folded)
|
|
1797
|
+
self.direct.setdefault(_residue_key(form), []).append(
|
|
1798
|
+
(form, index, direct))
|
|
1799
|
+
self.folded.setdefault(_form_folded_key(form), []).append(
|
|
1800
|
+
(form, index, folded))
|
|
1801
|
+
self.reader_form[(index, form)] = _reader_sees(form)
|
|
1802
|
+
self.all_ascii = all(form.isascii() for form, _ in compiled.forms)
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
def _pick(candidates: Sequence[Tuple[str, int, Any]], matched: str
|
|
1806
|
+
) -> Tuple[str, int, Any]:
|
|
1807
|
+
"""The form the combined pattern chose for `matched`, among those
|
|
1808
|
+
sharing its key.
|
|
1809
|
+
|
|
1810
|
+
The alternation takes the first branch that matches at a position,
|
|
1811
|
+
so the first form, in the pattern's own order, whose alternative
|
|
1812
|
+
matches the whole of `matched` is the one it chose. One candidate
|
|
1813
|
+
is the common case and needs no regex at all.
|
|
1814
|
+
"""
|
|
1815
|
+
if len(candidates) == 1:
|
|
1816
|
+
return candidates[0]
|
|
1817
|
+
for candidate in candidates:
|
|
1818
|
+
if candidate[2].fullmatch(matched):
|
|
1819
|
+
return candidate
|
|
1820
|
+
return candidates[0]
|
|
1821
|
+
|
|
1822
|
+
|
|
1823
|
+
def _attributed(matches, lookup_table, key=_direct_key):
|
|
1824
|
+
"""Each match of a detector pattern, with the form it is charged to.
|
|
1825
|
+
|
|
1826
|
+
Yields `(match, (form, entry) or None, charge)`, `charge` being what
|
|
1827
|
+
the match budget is charged for it: one match for a spelling placed
|
|
1828
|
+
before, and `RESIDUE_FIRST_SIGHT_MATCHES` plus one for each further
|
|
1829
|
+
candidate sharing its key for the first sight of a spelling, which
|
|
1830
|
+
is what placing it costs (fix round 2, B-3). The entry is recovered
|
|
1831
|
+
from the matched TEXT by key, never from the regex engine: one
|
|
1832
|
+
capture group per alternative made the count cost 82 seconds per
|
|
1833
|
+
1.27 MB at the documented ceiling of 2,000 forms, which is the shape
|
|
1834
|
+
of the uncapped-form defect `MAX_FORM_CHARS` exists for. A match
|
|
1835
|
+
the lookup cannot place is yielded with None: it is still counted,
|
|
1836
|
+
and charged to no entry. Since fix round 3 both passes key a match
|
|
1837
|
+
as it stands (`_direct_key`, `_folded_key`) and none is known to
|
|
1838
|
+
reach None; the two causes found before, `re.IGNORECASE` against
|
|
1839
|
+
`str.casefold` (fix round 2) and a match normalised again (CORR-1 in
|
|
1840
|
+
the folded pass, CORR2-2 in the direct one), are gone. It stays
|
|
1841
|
+
counted if one is ever found.
|
|
1842
|
+
"""
|
|
1843
|
+
placed: Dict[str, Any] = {}
|
|
1844
|
+
for match in matches:
|
|
1845
|
+
matched = match.group(0)
|
|
1846
|
+
candidate = placed.get(matched, _UNPLACED)
|
|
1847
|
+
charge = 1
|
|
1848
|
+
if candidate is _UNPLACED:
|
|
1849
|
+
candidates = lookup_table.get(key(matched))
|
|
1850
|
+
candidate = (_pick(candidates, matched)[:2] if candidates
|
|
1851
|
+
else None)
|
|
1852
|
+
placed[matched] = candidate
|
|
1853
|
+
charge = RESIDUE_FIRST_SIGHT_MATCHES + (
|
|
1854
|
+
len(candidates) - 1 if candidates else 0)
|
|
1855
|
+
yield match, candidate, charge
|
|
1856
|
+
|
|
1857
|
+
|
|
1858
|
+
def direct_attribution(compiled: "Compiled", seen: str
|
|
1859
|
+
) -> Tuple[Dict[Tuple[int, str], int], int]:
|
|
1860
|
+
"""The wide reading's direct pass over `seen`, charged by form.
|
|
1861
|
+
|
|
1862
|
+
One UNGROUPED `finditer` of the detector's direct pattern over the
|
|
1863
|
+
reader's text, the entry recovered by key (`_attributed`). Returns
|
|
1864
|
+
({(entry, form): count}, unattributed). The counterpart the tests
|
|
1865
|
+
compare it with, one capture group per alternative, lives in the
|
|
1866
|
+
test file and never here.
|
|
1867
|
+
"""
|
|
1868
|
+
lookup = compiled.text_lookup()
|
|
1869
|
+
counts: Dict[Tuple[int, str], int] = {}
|
|
1870
|
+
unattributed = 0
|
|
1871
|
+
for _match, candidate, _charge in _attributed(
|
|
1872
|
+
compiled.detector._direct.finditer(seen), lookup.direct):
|
|
1873
|
+
if candidate is None:
|
|
1874
|
+
unattributed += 1
|
|
1875
|
+
continue
|
|
1876
|
+
key = (candidate[1], candidate[0])
|
|
1877
|
+
counts[key] = counts.get(key, 0) + 1
|
|
1878
|
+
return counts, unattributed
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
def _rewriter_form(compiled: "Compiled", lookup: _TextLookup, matched: str
|
|
1882
|
+
) -> Tuple[int, str]:
|
|
1883
|
+
"""(entry, form) for one match of the REWRITER's pattern.
|
|
1884
|
+
|
|
1885
|
+
The entry is `Compiled.entry_for`'s, which is exact by construction;
|
|
1886
|
+
the form is the mapping's own spelling the match stands for, looked
|
|
1887
|
+
up exactly, then case-folded, then by the same full match
|
|
1888
|
+
`entry_for` falls back to, against the patterns `Compiled` holds
|
|
1889
|
+
compiled. Decided once per spelling (fix round 2, B-3).
|
|
1890
|
+
"""
|
|
1891
|
+
found = lookup.rewriter_form.get(matched)
|
|
1892
|
+
if found is not None:
|
|
1893
|
+
return found
|
|
1894
|
+
index = compiled.entry_for(matched)
|
|
1895
|
+
if compiled._exact.get(matched) == index:
|
|
1896
|
+
found = (index, matched)
|
|
1897
|
+
else:
|
|
1898
|
+
form = lookup.folded_form.get((index, _fold(matched)))
|
|
1899
|
+
if form is None:
|
|
1900
|
+
form = next((candidate for candidate in lookup.entry_forms[index]
|
|
1901
|
+
if compiled.form_pattern(candidate).fullmatch(
|
|
1902
|
+
matched)),
|
|
1903
|
+
compiled.mapping.entries[index].original)
|
|
1904
|
+
found = (index, form)
|
|
1905
|
+
lookup.rewriter_form[matched] = found
|
|
1906
|
+
return found
|
|
1907
|
+
|
|
1908
|
+
|
|
1909
|
+
# Every combining mark (categories Mn, Mc and Me of the running
|
|
1910
|
+
# interpreter), as one pattern for a run of them, built on first use.
|
|
1911
|
+
_MARK_RUN_PATTERN: Optional["re.Pattern[str]"] = None
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def _mark_run_pattern() -> "re.Pattern[str]":
|
|
1915
|
+
"""A pattern for a run of zero or more combining marks, built once."""
|
|
1916
|
+
global _MARK_RUN_PATTERN
|
|
1917
|
+
pattern = _MARK_RUN_PATTERN
|
|
1918
|
+
if pattern is None:
|
|
1919
|
+
if _MARK_RANGES is None:
|
|
1920
|
+
_sweep_code_points()
|
|
1921
|
+
ranges = _MARK_RANGES
|
|
1922
|
+
pattern = _MARK_RUN_PATTERN = re.compile(
|
|
1923
|
+
"[" + "".join(
|
|
1924
|
+
re.escape(chr(low)) if low == high
|
|
1925
|
+
else re.escape(chr(low)) + "-" + re.escape(chr(high))
|
|
1926
|
+
for low, high in ranges) + "]*")
|
|
1927
|
+
return pattern
|
|
1928
|
+
|
|
1929
|
+
|
|
1930
|
+
def _with_trailing_marks(text: str, start: int, end: int) -> str:
|
|
1931
|
+
"""A whole-word match with the combining marks that follow it.
|
|
1932
|
+
|
|
1933
|
+
The rewriter's `(?!\\w)` treats a combining mark as a boundary, so
|
|
1934
|
+
"Rene" followed by U+0301 is a whole word to it; a reader, and NFKC,
|
|
1935
|
+
join the mark to the letter and read "René". Reading the match
|
|
1936
|
+
together with its trailing marks is how the detector is asked about
|
|
1937
|
+
the occurrence as a reader sees it.
|
|
1938
|
+
|
|
1939
|
+
The WHOLE run of marks is read (fix round 2, the re-verification's
|
|
1940
|
+
CORR-2): fix round 1 read sixteen, and a seventeenth that composes
|
|
1941
|
+
onto the name's last letter across sixteen that do not (a mark of a
|
|
1942
|
+
lower combining class) brought QA-1's reading back. A run belongs to
|
|
1943
|
+
the one match before it, so reading it whole keeps the cost linear in
|
|
1944
|
+
the text; it is found with one compiled pattern, and no mark comes
|
|
1945
|
+
before U+0300, so an ASCII text never builds it.
|
|
1946
|
+
"""
|
|
1947
|
+
if end < len(text) and text[end] >= "̀":
|
|
1948
|
+
end = _mark_run_pattern().match(text, end).end()
|
|
1949
|
+
return text[start:end]
|
|
1950
|
+
|
|
1951
|
+
|
|
1952
|
+
def _reading_extra_pattern() -> "re.Pattern[str]":
|
|
1953
|
+
"""The code points of `_READING_EXTRA` as one character class, built
|
|
1954
|
+
once, as `_unseen_pattern` builds its own."""
|
|
1955
|
+
global _READING_EXTRA_PATTERN
|
|
1956
|
+
pattern = _READING_EXTRA_PATTERN
|
|
1957
|
+
if pattern is None:
|
|
1958
|
+
if _READING_EXTRA is None:
|
|
1959
|
+
_sweep_code_points()
|
|
1960
|
+
ranges: List[List[int]] = []
|
|
1961
|
+
for code_point in sorted(_READING_EXTRA or {}):
|
|
1962
|
+
_add_to_ranges(ranges, code_point)
|
|
1963
|
+
pattern = _READING_EXTRA_PATTERN = re.compile(
|
|
1964
|
+
"[" + "".join(
|
|
1965
|
+
re.escape(chr(low)) if low == high
|
|
1966
|
+
else re.escape(chr(low)) + "-" + re.escape(chr(high))
|
|
1967
|
+
for low, high in ranges) + "]")
|
|
1968
|
+
return pattern
|
|
1969
|
+
|
|
1970
|
+
|
|
1971
|
+
def reading_length(text: str, stop_past_work: Optional[int] = None) -> int:
|
|
1972
|
+
"""The length the count is priced at: its reading, not its store.
|
|
1973
|
+
|
|
1974
|
+
The count and the cheap question read `_reader_sees(text)` (NFKC) and
|
|
1975
|
+
its casefolded form, and one character's NFKC can be eighteen (U+FDFA,
|
|
1976
|
+
the Arabic ligature of the honorific), so the stored length
|
|
1977
|
+
under-prices such a text (fix round 6, the fourth re-verification's
|
|
1978
|
+
B4-1). Returned: an upper bound on the length of both readings, and
|
|
1979
|
+
never less than the stored length, which the normalisation itself
|
|
1980
|
+
reads. Linear, and never normalising (the normalisation of U+FDFA
|
|
1981
|
+
costs what the count does, which is what the estimate must not):
|
|
1982
|
+
ASCII text reads as it stands; text already in NFKC reads no longer
|
|
1983
|
+
than its casefolded form (the reading only removes invisible
|
|
1984
|
+
characters, and casefolding maps each character on its own), which
|
|
1985
|
+
is measured exactly; any other text is bounded character by
|
|
1986
|
+
character, each adding what `_READING_EXTRA` says (its NFKC,
|
|
1987
|
+
casefolded, less one). The sum bounds the reading because composing
|
|
1988
|
+
only shortens a text, and a composition that folds longer than its
|
|
1989
|
+
parts is charged to the letter it is built on; both are checked on
|
|
1990
|
+
every code point and every composition of the interpreter's tables,
|
|
1991
|
+
and on random texts, in the tests.
|
|
1992
|
+
|
|
1993
|
+
With `stop_past_work`, the sum stops once the text would cost more
|
|
1994
|
+
than that under a mapping of one form, the least any mapping costs:
|
|
1995
|
+
the caller needs no more to know the file is too large for any
|
|
1996
|
+
mapping, and a text of U+FDFA is not read to its end for it.
|
|
1997
|
+
"""
|
|
1998
|
+
if text.isascii():
|
|
1999
|
+
return len(text)
|
|
2000
|
+
if unicodedata.is_normalized("NFKC", text):
|
|
2001
|
+
return len(text.casefold())
|
|
2002
|
+
pattern = _reading_extra_pattern()
|
|
2003
|
+
extra = _READING_EXTRA or {}
|
|
2004
|
+
length = len(text)
|
|
2005
|
+
limit = (None if stop_past_work is None else
|
|
2006
|
+
stop_past_work // (1 + RESIDUE_WORK_PER_CHARACTER_NON_ASCII))
|
|
2007
|
+
for start in range(0, len(text), _READING_CHUNK):
|
|
2008
|
+
length += sum(extra[ord(char)] for char in pattern.findall(
|
|
2009
|
+
text, start, start + _READING_CHUNK))
|
|
2010
|
+
if limit is not None and length > limit:
|
|
2011
|
+
break
|
|
2012
|
+
return length
|
|
2013
|
+
|
|
2014
|
+
|
|
2015
|
+
# The stretch `reading_length` sums at a time between two looks at its
|
|
2016
|
+
# limit: small beside a file, large beside the pattern's own start-up.
|
|
2017
|
+
_READING_CHUNK = 65536
|
|
2018
|
+
|
|
2019
|
+
|
|
2020
|
+
def residue_work(compiled: "Compiled", text: str,
|
|
2021
|
+
length: Optional[int] = None) -> int:
|
|
2022
|
+
"""What counting (or checking) `text` costs, in the budgets' units.
|
|
2023
|
+
|
|
2024
|
+
`reading_length(text) * (the forms' units + the per-character term
|
|
2025
|
+
of the text's class)`: `RESIDUE_WORK_PER_CHARACTER` when the text is
|
|
2026
|
+
ASCII, `RESIDUE_WORK_PER_CHARACTER_NON_ASCII` when it is not (fix
|
|
2027
|
+
round 2, B-1). Since fix round 6 the length is the reading's (B4-1),
|
|
2028
|
+
and `length` passes it when the caller has it already, and the forms
|
|
2029
|
+
are priced by their length, `Compiled.form_units` (B4-2): a unit for
|
|
2030
|
+
each ten characters of a form, so a name is one unit, as before. The same units
|
|
2031
|
+
price the count, the cheap question and the whole detector asked
|
|
2032
|
+
about one word.
|
|
2033
|
+
"""
|
|
2034
|
+
term = (RESIDUE_WORK_PER_CHARACTER if text.isascii()
|
|
2035
|
+
else RESIDUE_WORK_PER_CHARACTER_NON_ASCII)
|
|
2036
|
+
if length is None:
|
|
2037
|
+
length = reading_length(text)
|
|
2038
|
+
return length * (compiled.form_units + term)
|
|
2039
|
+
|
|
2040
|
+
|
|
2041
|
+
def residue_work_at_one_form(text: str,
|
|
2042
|
+
length: Optional[int] = None) -> int:
|
|
2043
|
+
"""`residue_work` of `text` under a mapping of one surface form, the
|
|
2044
|
+
least any mapping costs: a file whose work passes the budget even so
|
|
2045
|
+
cannot be counted in a preview with any mapping, and fewer names
|
|
2046
|
+
cannot help it (the lead's follow-on to fix round 5)."""
|
|
2047
|
+
term = (RESIDUE_WORK_PER_CHARACTER if text.isascii()
|
|
2048
|
+
else RESIDUE_WORK_PER_CHARACTER_NON_ASCII)
|
|
2049
|
+
if length is None:
|
|
2050
|
+
length = reading_length(text)
|
|
2051
|
+
return length * (1 + term)
|
|
2052
|
+
|
|
2053
|
+
|
|
2054
|
+
def in_a_no_separator_script(char: str) -> bool:
|
|
2055
|
+
"""Whether `char` belongs to one of `NO_SEPARATOR_RANGES`."""
|
|
2056
|
+
code_point = ord(char)
|
|
2057
|
+
index = bisect_right(_NO_SEPARATOR_STARTS, code_point) - 1
|
|
2058
|
+
return index >= 0 and code_point <= NO_SEPARATOR_RANGES[index][1]
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
def _longer_word(seen: str, start: int, end: int) -> Optional[str]:
|
|
2062
|
+
"""The run of word characters around a match, as the reader sees it,
|
|
2063
|
+
when it is a word.
|
|
2064
|
+
|
|
2065
|
+
`Thomas_P01` is one word and `Thomas_Smith.txt` gives
|
|
2066
|
+
`Thomas_Smith`. None, and the occurrence is counted and not listed,
|
|
2067
|
+
when the run extends the name by more than
|
|
2068
|
+
`MAX_LONGER_WORD_EXTENSION` characters, or when what extends it
|
|
2069
|
+
carries a character of a script written without separators
|
|
2070
|
+
(`NO_SEPARATOR_RANGES`), where the run is a clause. The walk stops
|
|
2071
|
+
one character past the bound on each side, so a match inside a long
|
|
2072
|
+
run costs a fixed handful of steps, never a walk of the run.
|
|
2073
|
+
"""
|
|
2074
|
+
budget = MAX_LONGER_WORD_EXTENSION
|
|
2075
|
+
left = start
|
|
2076
|
+
while left > 0 and _is_word_char(seen[left - 1]):
|
|
2077
|
+
left -= 1
|
|
2078
|
+
budget -= 1
|
|
2079
|
+
if budget < 0:
|
|
2080
|
+
return None
|
|
2081
|
+
right = end
|
|
2082
|
+
while right < len(seen) and _is_word_char(seen[right]):
|
|
2083
|
+
right += 1
|
|
2084
|
+
budget -= 1
|
|
2085
|
+
if budget < 0:
|
|
2086
|
+
return None
|
|
2087
|
+
extension = seen[left:start] + seen[end:right]
|
|
2088
|
+
if any(in_a_no_separator_script(char) for char in extension):
|
|
2089
|
+
return None
|
|
2090
|
+
return seen[left:right]
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def names_left_in_text(compiled: "Compiled", text: str,
|
|
2094
|
+
list_longer_words: bool,
|
|
2095
|
+
rewritten_by_this_run: bool,
|
|
2096
|
+
max_matches: Optional[int] = None,
|
|
2097
|
+
max_extra_work: Optional[int] = None,
|
|
2098
|
+
stopped: Optional[Dict[str, Any]] = None
|
|
2099
|
+
) -> Optional[TextResidue]:
|
|
2100
|
+
"""Where this mapping's names are left in one file's text.
|
|
2101
|
+
|
|
2102
|
+
`text` is the file as it will be AFTER the run: the rewritten text
|
|
2103
|
+
for the one file the run rewrites, the stored text for every other.
|
|
2104
|
+
Counts, never spans and never surrounding text; with
|
|
2105
|
+
`list_longer_words`, the longer words a name sits inside (the typed
|
|
2106
|
+
path only, decided by the caller).
|
|
2107
|
+
|
|
2108
|
+
Two readings of every count, in one object `{"wide", "whole_word"}`:
|
|
2109
|
+
|
|
2110
|
+
- wide: the detector's reading of the reader's text, and beside it
|
|
2111
|
+
every whole word the rewriter matches that the reader's reading
|
|
2112
|
+
does not see (a name followed by a combining mark, which NFKC
|
|
2113
|
+
composes onto its last letter): an occurrence counts in the wide
|
|
2114
|
+
reading when either matcher finds it. Every wide occurrence is
|
|
2115
|
+
charged to exactly one kind (`TEXT_RESIDUE_REASONS`) or to
|
|
2116
|
+
`unattributed`, so the kinds and `unattributed` add up to the wide
|
|
2117
|
+
count on every row; the split is a heuristic and the total is
|
|
2118
|
+
not.
|
|
2119
|
+
- whole_word: the rewriter's own pattern over the RAW text, reported
|
|
2120
|
+
as it stands. On the file this run rewrote it is normally zero,
|
|
2121
|
+
and non-zero means a pseudonym put a name back.
|
|
2122
|
+
|
|
2123
|
+
The kinds, for each wide match in the reader's text: a word
|
|
2124
|
+
character touches either end (inside_a_longer_word); otherwise the
|
|
2125
|
+
match casefolds differently from the form as the reader sees it
|
|
2126
|
+
(joined_differently); otherwise it differs from that form and the
|
|
2127
|
+
case mode is exact (case_only); otherwise it IS the form, whole
|
|
2128
|
+
word, and the rewriter left it. That last class has two causes,
|
|
2129
|
+
split per form by the whole-word count of the raw text: on the row
|
|
2130
|
+
of the file this run rewrote, `min(whole_word, class)` is charged to
|
|
2131
|
+
put_back_by_a_pseudonym and the rest to normalisation_variants; on
|
|
2132
|
+
every other row nothing was put back by this run, so the same share
|
|
2133
|
+
is charged to whole_word_in_a_file_not_rewritten instead. The `min`
|
|
2134
|
+
and `max` are a declared HEURISTIC about which cause an occurrence
|
|
2135
|
+
is charged to when both are present; their sum is exact.
|
|
2136
|
+
|
|
2137
|
+
`max_matches` bounds the work a hostile text can cost: every match of
|
|
2138
|
+
the three passes counts against it, the first sight of each spelling
|
|
2139
|
+
in a pass as `RESIDUE_FIRST_SIGHT_MATCHES` (placing a new spelling
|
|
2140
|
+
costs several matches; a spelling met before is looked up; fix round
|
|
2141
|
+
2), and past it the count stops and the function returns None, which the
|
|
2142
|
+
caller reads as "not counted in full" (the match budget, fix round
|
|
2143
|
+
1, S-4). The result's `matches` is what the count spent, in those
|
|
2144
|
+
units.
|
|
2145
|
+
|
|
2146
|
+
`max_extra_work` bounds the one cost the caller's work model does not
|
|
2147
|
+
see (fix round 2): asking the whole detector about a whole word the
|
|
2148
|
+
rewriter matched and the word's own alternatives do not find in its
|
|
2149
|
+
reading (a name before a composing mark). Such a question costs what
|
|
2150
|
+
the count costs per character of the word and its marks, and is
|
|
2151
|
+
asked once per distinct spelling in a preview, so a text written to
|
|
2152
|
+
make every spelling distinct could otherwise cost the count's time
|
|
2153
|
+
again. Each such question is charged `residue_work` of the word and
|
|
2154
|
+
its marks, and one match; past the allowance the
|
|
2155
|
+
function returns None, as past the match budget. The result's
|
|
2156
|
+
`extra_work` is what it charged. When the count stops, `stopped` (a
|
|
2157
|
+
dict, if given) is told what the count had spent, so the caller can
|
|
2158
|
+
charge it (fix round 5, the lead's rule 1: everything spent is
|
|
2159
|
+
charged): `cause` ("matches" or "work", the allowance it passed),
|
|
2160
|
+
`matches` and `extra_work` as the result would give them, and
|
|
2161
|
+
`at_least`, how many occurrences the count had found by then (rule
|
|
2162
|
+
3). Every stop comes on a match or on a question about one, so
|
|
2163
|
+
`at_least` is 1 or more and the file certainly shows a name. It is a
|
|
2164
|
+
lower bound on the larger of the two readings: the wide count is
|
|
2165
|
+
exactly the larger of the direct and folded passes' matches plus the
|
|
2166
|
+
whole words only the rewriter sees, so each pass's matches so far
|
|
2167
|
+
bound it, and the whole words so far bound the whole-word count.
|
|
2168
|
+
"""
|
|
2169
|
+
lookup = compiled.text_lookup()
|
|
2170
|
+
spent = 0
|
|
2171
|
+
limit = max_matches if max_matches is not None else -1
|
|
2172
|
+
extra_work = 0
|
|
2173
|
+
extra_limit = max_extra_work if max_extra_work is not None else -1
|
|
2174
|
+
seen = _reader_sees(text)
|
|
2175
|
+
rows: Dict[Tuple[int, str], Dict[str, Any]] = {}
|
|
2176
|
+
direct_total = 0
|
|
2177
|
+
folded_seen = 0
|
|
2178
|
+
whole_word_total = 0
|
|
2179
|
+
only_seen = 0
|
|
2180
|
+
|
|
2181
|
+
def stop(cause: str) -> None:
|
|
2182
|
+
if stopped is not None:
|
|
2183
|
+
stopped.update(
|
|
2184
|
+
cause=cause, matches=spent, extra_work=extra_work,
|
|
2185
|
+
at_least=max(max(direct_total, folded_seen) + only_seen,
|
|
2186
|
+
whole_word_total))
|
|
2187
|
+
|
|
2188
|
+
def row_for(key: Tuple[int, str]) -> Dict[str, Any]:
|
|
2189
|
+
row = rows.get(key)
|
|
2190
|
+
if row is None:
|
|
2191
|
+
row = rows[key] = {"direct": 0, "inside_a_longer_word": 0,
|
|
2192
|
+
"case_only": 0, "joined_differently": 0,
|
|
2193
|
+
"left_whole": 0, "folded_excess": 0,
|
|
2194
|
+
"whole_word": 0, "rewriter_only": 0,
|
|
2195
|
+
"words": {}, "words_withheld": False}
|
|
2196
|
+
return row
|
|
2197
|
+
|
|
2198
|
+
unattributed = 0
|
|
2199
|
+
exact_mode = compiled.case_mode == "exact"
|
|
2200
|
+
for match, candidate, charge in _attributed(
|
|
2201
|
+
compiled.detector._direct.finditer(seen), lookup.direct):
|
|
2202
|
+
direct_total += 1
|
|
2203
|
+
spent += charge
|
|
2204
|
+
if spent > limit >= 0:
|
|
2205
|
+
stop("matches")
|
|
2206
|
+
return None
|
|
2207
|
+
if candidate is None:
|
|
2208
|
+
unattributed += 1
|
|
2209
|
+
continue
|
|
2210
|
+
form, index = candidate
|
|
2211
|
+
row = row_for((index, form))
|
|
2212
|
+
row["direct"] += 1
|
|
2213
|
+
matched = match.group(0)
|
|
2214
|
+
start, end = match.span()
|
|
2215
|
+
if ((start > 0 and _is_word_char(seen[start - 1]))
|
|
2216
|
+
or (end < len(seen) and _is_word_char(seen[end]))):
|
|
2217
|
+
row["inside_a_longer_word"] += 1
|
|
2218
|
+
if list_longer_words:
|
|
2219
|
+
word = _longer_word(seen, start, end)
|
|
2220
|
+
if word is None:
|
|
2221
|
+
row["words_withheld"] = True
|
|
2222
|
+
else:
|
|
2223
|
+
row["words"][word] = row["words"].get(word, 0) + 1
|
|
2224
|
+
continue
|
|
2225
|
+
reader_form = lookup.reader_form[(index, form)]
|
|
2226
|
+
# Compared the way the pattern compared them (fix round 2): a
|
|
2227
|
+
# capital I with a dot above is the same letter to it, and to
|
|
2228
|
+
# casefolding two code points, which read as joined differently.
|
|
2229
|
+
if _word_key(matched) != _word_key(reader_form):
|
|
2230
|
+
row["joined_differently"] += 1
|
|
2231
|
+
elif matched != reader_form and exact_mode:
|
|
2232
|
+
row["case_only"] += 1
|
|
2233
|
+
else:
|
|
2234
|
+
row["left_whole"] += 1
|
|
2235
|
+
|
|
2236
|
+
# The casefolded second reading, which `NameDetector.contains` makes
|
|
2237
|
+
# too: `re.IGNORECASE` and `str.casefold` disagree on some code
|
|
2238
|
+
# points (U+00DF casefolds to "ss" and does not match "SS" under
|
|
2239
|
+
# IGNORECASE), so a spelling only case-folding reaches is missed by
|
|
2240
|
+
# the direct pass above. Where the folded pass finds more for a form
|
|
2241
|
+
# than the direct pass did, the excess is added to that form's wide
|
|
2242
|
+
# count and charged to normalisation_variants. This is a HEURISTIC,
|
|
2243
|
+
# declared as one: a spelling only case-folding reaches is one the
|
|
2244
|
+
# rewriter cannot see under any case mode, so it is charged there,
|
|
2245
|
+
# and it is what keeps `wide` from being zero on a file where
|
|
2246
|
+
# `detector.contains` says a name shows. The excess is capped at the
|
|
2247
|
+
# folded pass's net surplus over the direct one, so an occurrence the
|
|
2248
|
+
# direct pass could not place (dotted capital I, which IGNORECASE
|
|
2249
|
+
# matches and casefolding spells as two code points) and the folded
|
|
2250
|
+
# pass could is not counted twice. On ASCII text with ASCII forms
|
|
2251
|
+
# the two passes cannot disagree, so the second is skipped.
|
|
2252
|
+
if not (seen.isascii() and lookup.all_ascii):
|
|
2253
|
+
folded_counts: Dict[Tuple[int, str], int] = {}
|
|
2254
|
+
folded_unattributed = 0
|
|
2255
|
+
for _match, candidate, charge in _attributed(
|
|
2256
|
+
compiled.detector._folded.finditer(_fold(seen)),
|
|
2257
|
+
lookup.folded, key=_folded_key):
|
|
2258
|
+
folded_seen += 1
|
|
2259
|
+
spent += charge
|
|
2260
|
+
if spent > limit >= 0:
|
|
2261
|
+
stop("matches")
|
|
2262
|
+
return None
|
|
2263
|
+
if candidate is None:
|
|
2264
|
+
folded_unattributed += 1
|
|
2265
|
+
continue
|
|
2266
|
+
key = (candidate[1], candidate[0])
|
|
2267
|
+
folded_counts[key] = folded_counts.get(key, 0) + 1
|
|
2268
|
+
surplus = (sum(folded_counts.values()) + folded_unattributed
|
|
2269
|
+
- direct_total)
|
|
2270
|
+
for form, index in compiled.forms:
|
|
2271
|
+
if surplus <= 0:
|
|
2272
|
+
break
|
|
2273
|
+
key = (index, form)
|
|
2274
|
+
excess = folded_counts.get(key, 0) - (
|
|
2275
|
+
rows[key]["direct"] if key in rows else 0)
|
|
2276
|
+
if excess > 0:
|
|
2277
|
+
excess = min(excess, surplus)
|
|
2278
|
+
row_for(key)["folded_excess"] += excess
|
|
2279
|
+
surplus -= excess
|
|
2280
|
+
if surplus > 0:
|
|
2281
|
+
unattributed += min(surplus, max(
|
|
2282
|
+
0, folded_unattributed - unattributed))
|
|
2283
|
+
|
|
2284
|
+
# The whole-word half, over the RAW text. A whole word the rewriter
|
|
2285
|
+
# matches that the detector does not see, read with the combining
|
|
2286
|
+
# marks after it as a reader reads it, is an occurrence of the wide
|
|
2287
|
+
# reading too (the union; lead's ruling on QA-1), charged below to
|
|
2288
|
+
# the whole-word kind of its row. Asked once per distinct spelling in
|
|
2289
|
+
# the preview: first of the form's own two alternatives, which find
|
|
2290
|
+
# it in all but the composing-mark case, then, only when they do not,
|
|
2291
|
+
# of the whole detector, whose cost is charged (`max_extra_work`).
|
|
2292
|
+
detector_sees = lookup.seen_alone
|
|
2293
|
+
for match in compiled.pattern.finditer(text):
|
|
2294
|
+
whole_word_total += 1
|
|
2295
|
+
matched = match.group(0)
|
|
2296
|
+
spent += (1 if matched in lookup.rewriter_form
|
|
2297
|
+
else RESIDUE_FIRST_SIGHT_MATCHES)
|
|
2298
|
+
if spent > limit >= 0:
|
|
2299
|
+
stop("matches")
|
|
2300
|
+
return None
|
|
2301
|
+
key = _rewriter_form(compiled, lookup, matched)
|
|
2302
|
+
row = row_for(key)
|
|
2303
|
+
row["whole_word"] += 1
|
|
2304
|
+
snippet = _with_trailing_marks(text, match.start(), match.end())
|
|
2305
|
+
seen_alone = detector_sees.get(snippet)
|
|
2306
|
+
if seen_alone is None:
|
|
2307
|
+
reading = _reader_sees(snippet)
|
|
2308
|
+
own = lookup.own_patterns.get(key)
|
|
2309
|
+
seen_alone = bool(own and (own[0].search(reading)
|
|
2310
|
+
or own[1].search(_fold(reading))))
|
|
2311
|
+
if not seen_alone:
|
|
2312
|
+
spent += 1
|
|
2313
|
+
extra_work += residue_work(compiled, snippet)
|
|
2314
|
+
if spent > limit >= 0 or (
|
|
2315
|
+
extra_limit >= 0 and extra_work > extra_limit):
|
|
2316
|
+
stop("matches" if spent > limit >= 0 else "work")
|
|
2317
|
+
return None
|
|
2318
|
+
seen_alone = compiled.detector.contains(snippet)
|
|
2319
|
+
detector_sees[snippet] = seen_alone
|
|
2320
|
+
if not seen_alone:
|
|
2321
|
+
row["rewriter_only"] += 1
|
|
2322
|
+
only_seen += 1
|
|
2323
|
+
|
|
2324
|
+
entries: Dict[int, Dict[str, Any]] = {}
|
|
2325
|
+
case_variants: List[Dict[str, Any]] = []
|
|
2326
|
+
normalisation_variants: List[Dict[str, Any]] = []
|
|
2327
|
+
wide_total = unattributed
|
|
2328
|
+
for form, index in compiled.forms:
|
|
2329
|
+
row = rows.get((index, form))
|
|
2330
|
+
if row is None:
|
|
2331
|
+
continue
|
|
2332
|
+
# The heuristic split of the class the rewriter left whole, per
|
|
2333
|
+
# form (see the docstring); the sum of the two is exact. The
|
|
2334
|
+
# whole words only the rewriter sees are whole words of that
|
|
2335
|
+
# kind outright, and are added to the wide count.
|
|
2336
|
+
only = row["rewriter_only"]
|
|
2337
|
+
seen_whole = row["whole_word"] - only
|
|
2338
|
+
shared = min(seen_whole, row["left_whole"]) + only
|
|
2339
|
+
spelling = (max(0, row["left_whole"] - seen_whole)
|
|
2340
|
+
+ row["folded_excess"])
|
|
2341
|
+
wide = row["direct"] + row["folded_excess"] + only
|
|
2342
|
+
wide_total += wide
|
|
2343
|
+
entry = entries.get(index)
|
|
2344
|
+
if entry is None:
|
|
2345
|
+
entry = entries[index] = {
|
|
2346
|
+
"entry": index,
|
|
2347
|
+
"form": compiled.mapping.entries[index].original,
|
|
2348
|
+
"occurrences": {"wide": 0, "whole_word": 0},
|
|
2349
|
+
"inside_a_longer_word": 0, "case_only": 0,
|
|
2350
|
+
"joined_differently": 0, "normalisation_variants": 0,
|
|
2351
|
+
"put_back_by_a_pseudonym": 0,
|
|
2352
|
+
"whole_word_in_a_file_not_rewritten": 0,
|
|
2353
|
+
"_words": {}, "_withheld": False}
|
|
2354
|
+
entry["occurrences"]["wide"] += wide
|
|
2355
|
+
entry["occurrences"]["whole_word"] += row["whole_word"]
|
|
2356
|
+
entry["inside_a_longer_word"] += row["inside_a_longer_word"]
|
|
2357
|
+
entry["case_only"] += row["case_only"]
|
|
2358
|
+
entry["joined_differently"] += row["joined_differently"]
|
|
2359
|
+
entry["normalisation_variants"] += spelling
|
|
2360
|
+
if rewritten_by_this_run:
|
|
2361
|
+
entry["put_back_by_a_pseudonym"] += shared
|
|
2362
|
+
else:
|
|
2363
|
+
entry["whole_word_in_a_file_not_rewritten"] += shared
|
|
2364
|
+
for word, count in row["words"].items():
|
|
2365
|
+
entry["_words"][word] = entry["_words"].get(word, 0) + count
|
|
2366
|
+
entry["_withheld"] = entry["_withheld"] or row["words_withheld"]
|
|
2367
|
+
if row["case_only"]:
|
|
2368
|
+
case_variants.append({"entry": index, "form": form,
|
|
2369
|
+
"other_case_count": row["case_only"]})
|
|
2370
|
+
if spelling:
|
|
2371
|
+
normalisation_variants.append({"entry": index, "form": form,
|
|
2372
|
+
"count": spelling})
|
|
2373
|
+
|
|
2374
|
+
listed: List[Dict[str, Any]] = []
|
|
2375
|
+
# In wide-count order, the entry index breaking ties, so a cap on the
|
|
2376
|
+
# rows (the caller's) keeps the entries with the most left.
|
|
2377
|
+
for index in sorted(entries, key=lambda i: (
|
|
2378
|
+
-entries[i]["occurrences"]["wide"],
|
|
2379
|
+
-entries[i]["occurrences"]["whole_word"], i)):
|
|
2380
|
+
entry = entries[index]
|
|
2381
|
+
words = entry.pop("_words")
|
|
2382
|
+
withheld = entry.pop("_withheld")
|
|
2383
|
+
if not (entry["occurrences"]["wide"]
|
|
2384
|
+
or entry["occurrences"]["whole_word"]):
|
|
2385
|
+
continue
|
|
2386
|
+
if list_longer_words:
|
|
2387
|
+
ranked = sorted(words.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
2388
|
+
entry["longer_words"] = [
|
|
2389
|
+
{"word": word, "count": count}
|
|
2390
|
+
for word, count in ranked[:MAX_LONGER_WORDS_PER_ENTRY]]
|
|
2391
|
+
if withheld or len(ranked) > MAX_LONGER_WORDS_PER_ENTRY:
|
|
2392
|
+
entry["longer_words_truncated"] = True
|
|
2393
|
+
listed.append(entry)
|
|
2394
|
+
return {"occurrences": {"wide": wide_total,
|
|
2395
|
+
"whole_word": whole_word_total},
|
|
2396
|
+
"unattributed": unattributed,
|
|
2397
|
+
"entries": listed,
|
|
2398
|
+
"case_variants_seen": case_variants,
|
|
2399
|
+
"normalisation_variants_seen": normalisation_variants,
|
|
2400
|
+
"matches": spent,
|
|
2401
|
+
"extra_work": extra_work}
|
|
2402
|
+
|
|
2403
|
+
|
|
2404
|
+
# The owner's ruling of 2026-09-23 on Brief 1's finding F-1: a pseudonym
|
|
2405
|
+
# that contains a name from the mapping is withheld, by the rule that
|
|
2406
|
+
# withholds a file name carrying one, from the run record and the journal
|
|
2407
|
+
# entry on both mapping paths and from the preview on the
|
|
2408
|
+
# `use_project_pseudonyms` path. The one sentence every such place says.
|
|
2409
|
+
PSEUDONYMS_WITHHELD_NOTE = (
|
|
2410
|
+
"A pseudonym that contains a name from this mapping is withheld here, "
|
|
2411
|
+
"by the rule that withholds a file name that carries one; its entry "
|
|
2412
|
+
"number identifies it. The run still writes it into the text.")
|
|
2413
|
+
|
|
2414
|
+
|
|
2415
|
+
def pseudonyms_withheld(compiled: "Compiled") -> Tuple[int, ...]:
|
|
2416
|
+
"""The entries whose pseudonym carries a name from the mapping.
|
|
2417
|
+
|
|
2418
|
+
`Compiled.carries_a_name` over each pseudonym: the union of the two
|
|
2419
|
+
matchers, so "Alex Smith" (a whole word of `Smith`), "Thomas Jr" (its
|
|
2420
|
+
own original) and "Thomasina" (inside a longer word, which the static
|
|
2421
|
+
check of `pseudonyms_containing_a_name` cannot see) are all withheld.
|
|
2422
|
+
The price is the wide reading's: a short name withholds a pseudonym
|
|
2423
|
+
that merely contains its letters ("Ed" in "Fred"), and the entry
|
|
2424
|
+
number still identifies it.
|
|
2425
|
+
"""
|
|
2426
|
+
return tuple(entry.index for entry in compiled.mapping.entries
|
|
2427
|
+
if compiled.carries_a_name(entry.pseudonym))
|
|
2428
|
+
|
|
2429
|
+
|
|
2430
|
+
def pseudonyms_containing_a_name(compiled: "Compiled"
|
|
2431
|
+
) -> List[Dict[str, Any]]:
|
|
2432
|
+
"""Pseudonyms the rewriter's own rule finds a name in (v0.13, item 5).
|
|
2433
|
+
|
|
2434
|
+
A static check, before any text is read: the rewriter's compiled
|
|
2435
|
+
pattern run over each pseudonym. `{"Smith": "Jones", "Thomas Smith":
|
|
2436
|
+
"Alex Smith"}` puts the real surname back wherever the second entry
|
|
2437
|
+
fires, and a pseudonym that contains its OWN original ("Thomas" to
|
|
2438
|
+
"Thomas Jr") does the same. What it cannot see, and says so where
|
|
2439
|
+
it is reported: a pseudonym that puts a name back inside a longer
|
|
2440
|
+
word (`xThomasx`), which no whole-word rule matches, and a
|
|
2441
|
+
pseudonym that forms a name with the words around it.
|
|
2442
|
+
|
|
2443
|
+
Each finding once per (entry, contained form): the entry whose
|
|
2444
|
+
pseudonym it is, the pseudonym, the entry it contains and that
|
|
2445
|
+
entry's form as the mapping spells it.
|
|
2446
|
+
"""
|
|
2447
|
+
lookup = compiled.text_lookup()
|
|
2448
|
+
found: List[Dict[str, Any]] = []
|
|
2449
|
+
for entry in compiled.mapping.entries:
|
|
2450
|
+
reported = set()
|
|
2451
|
+
for match in compiled.pattern.finditer(entry.pseudonym):
|
|
2452
|
+
index, form = _rewriter_form(compiled, lookup, match.group(0))
|
|
2453
|
+
if (index, form) in reported:
|
|
2454
|
+
continue
|
|
2455
|
+
reported.add((index, form))
|
|
2456
|
+
found.append({"entry": entry.index,
|
|
2457
|
+
"pseudonym": entry.pseudonym,
|
|
2458
|
+
"contains_entry": index, "form": form})
|
|
2459
|
+
return found
|
|
2460
|
+
|
|
2461
|
+
|
|
2462
|
+
# --------------------------------------------------------------------------
|
|
2463
|
+
# The remap
|
|
2464
|
+
# --------------------------------------------------------------------------
|
|
2465
|
+
|
|
2466
|
+
class RemappedSpan:
|
|
2467
|
+
"""What the run would do to one stored span."""
|
|
2468
|
+
|
|
2469
|
+
__slots__ = ("pos0", "pos1", "change", "clamped", "touched")
|
|
2470
|
+
|
|
2471
|
+
def __init__(self, pos0: Optional[int], pos1: Optional[int],
|
|
2472
|
+
change: str, clamped: bool, touched: bool):
|
|
2473
|
+
self.pos0 = pos0
|
|
2474
|
+
self.pos1 = pos1
|
|
2475
|
+
self.change = change
|
|
2476
|
+
self.clamped = clamped
|
|
2477
|
+
# Whether the old span met an edit at all. A row whose positions
|
|
2478
|
+
# do not move can still need its `seltext` refreshed, because a
|
|
2479
|
+
# pseudonym of the same length changes the text inside it.
|
|
2480
|
+
self.touched = touched
|
|
2481
|
+
|
|
2482
|
+
def __repr__(self) -> str: # pragma: no cover - debugging
|
|
2483
|
+
return (f"RemappedSpan({self.pos0}, {self.pos1}, {self.change}"
|
|
2484
|
+
f"{', clamped' if self.clamped else ''})")
|
|
2485
|
+
|
|
2486
|
+
|
|
2487
|
+
class SpanMapper:
|
|
2488
|
+
"""Maps old offsets to new ones for one text and one edit list.
|
|
2489
|
+
|
|
2490
|
+
`snap_to_pseudonym` is arithmetic on a prefix sum and needs no walk;
|
|
2491
|
+
`qualcoder_edit_parity` walks the edits per row, which is the same
|
|
2492
|
+
semantics as upstream's walk over all rows per edit but the opposite
|
|
2493
|
+
iteration order, so the two are a real cross-check of each other.
|
|
2494
|
+
"""
|
|
2495
|
+
|
|
2496
|
+
def __init__(self, replacements: Sequence[Replacement], old_len: int):
|
|
2497
|
+
self.old_len = old_len
|
|
2498
|
+
self.starts = [item.start for item in replacements]
|
|
2499
|
+
self.ends = [item.end for item in replacements]
|
|
2500
|
+
self.new_lengths = [len(item.text) for item in replacements]
|
|
2501
|
+
self.replacements = list(replacements)
|
|
2502
|
+
cumulative = [0]
|
|
2503
|
+
for item in replacements:
|
|
2504
|
+
cumulative.append(cumulative[-1] + item.delta)
|
|
2505
|
+
self.cumulative = cumulative
|
|
2506
|
+
self.new_len = old_len + cumulative[-1]
|
|
2507
|
+
|
|
2508
|
+
# -- snap_to_pseudonym ------------------------------------------------
|
|
2509
|
+
|
|
2510
|
+
def _edit_containing(self, position: int) -> Optional[int]:
|
|
2511
|
+
"""The edit `position` falls strictly inside, or None."""
|
|
2512
|
+
index = bisect_right(self.ends, position)
|
|
2513
|
+
if index < len(self.starts) and self.starts[index] < position:
|
|
2514
|
+
return index
|
|
2515
|
+
return None
|
|
2516
|
+
|
|
2517
|
+
def _delta_before(self, position: int) -> int:
|
|
2518
|
+
return self.cumulative[bisect_right(self.ends, position)]
|
|
2519
|
+
|
|
2520
|
+
def map_start(self, position: int) -> int:
|
|
2521
|
+
"""Where a span's START goes under `snap_to_pseudonym`.
|
|
2522
|
+
|
|
2523
|
+
Strictly inside a replaced name, the start snaps OUTWARD to the
|
|
2524
|
+
pseudonym's first character, so a coding that began in the middle
|
|
2525
|
+
of the name now begins at the pseudonym. Anywhere else it moves
|
|
2526
|
+
by the cumulative length change of the edits that finished before
|
|
2527
|
+
it.
|
|
2528
|
+
"""
|
|
2529
|
+
inside = self._edit_containing(position)
|
|
2530
|
+
if inside is not None:
|
|
2531
|
+
return self.starts[inside] + self.cumulative[inside]
|
|
2532
|
+
return position + self._delta_before(position)
|
|
2533
|
+
|
|
2534
|
+
def map_end(self, position: int) -> int:
|
|
2535
|
+
"""Where a span's END goes under `snap_to_pseudonym`.
|
|
2536
|
+
|
|
2537
|
+
The mirror of `map_start`: strictly inside a replaced name the
|
|
2538
|
+
end snaps outward to just past the pseudonym, so a coding that
|
|
2539
|
+
stopped in the middle of the name now contains the whole
|
|
2540
|
+
pseudonym. An end exactly at the name's end lands exactly at the
|
|
2541
|
+
pseudonym's end.
|
|
2542
|
+
"""
|
|
2543
|
+
inside = self._edit_containing(position)
|
|
2544
|
+
if inside is not None:
|
|
2545
|
+
return (self.starts[inside] + self.new_lengths[inside]
|
|
2546
|
+
+ self.cumulative[inside])
|
|
2547
|
+
return position + self._delta_before(position)
|
|
2548
|
+
|
|
2549
|
+
def _intersects_edit(self, pos0: int, pos1: int) -> bool:
|
|
2550
|
+
index = bisect_right(self.ends, pos0)
|
|
2551
|
+
return index < len(self.starts) and self.starts[index] < pos1
|
|
2552
|
+
|
|
2553
|
+
def _snap(self, pos0: int, pos1: int) -> RemappedSpan:
|
|
2554
|
+
cut = (self._edit_containing(pos0) is not None
|
|
2555
|
+
or self._edit_containing(pos1) is not None)
|
|
2556
|
+
new0 = self.map_start(pos0)
|
|
2557
|
+
new1 = self.map_end(pos1)
|
|
2558
|
+
return self._classify(pos0, pos1, new0, new1, cut)
|
|
2559
|
+
|
|
2560
|
+
# -- qualcoder_edit_parity --------------------------------------------
|
|
2561
|
+
|
|
2562
|
+
def _parity(self, pos0: int, pos1: int,
|
|
2563
|
+
keep_start_anchor: bool) -> RemappedSpan:
|
|
2564
|
+
"""Master's `apply_delete` then `apply_insert`, per edit.
|
|
2565
|
+
|
|
2566
|
+
Transcribed from `code_text.py:5906-5927` and `:5893-5904` at pin
|
|
2567
|
+
9bddf17 and driven by the running offset of `:5933-5950`: each
|
|
2568
|
+
replacement is the deletion of the name followed by the insertion
|
|
2569
|
+
of the pseudonym at the same evolving offset. The consequences
|
|
2570
|
+
are upstream's own and are the reason this is the OPTION and not
|
|
2571
|
+
the default: a span that sits exactly on a name is deleted, and a
|
|
2572
|
+
span that merely touches one is trimmed to exclude the pseudonym
|
|
2573
|
+
entirely.
|
|
2574
|
+
"""
|
|
2575
|
+
original_pos0 = pos0
|
|
2576
|
+
new0: Optional[int] = pos0
|
|
2577
|
+
new1 = pos1
|
|
2578
|
+
shift = 0
|
|
2579
|
+
head_cut = False
|
|
2580
|
+
tail_cut = False
|
|
2581
|
+
for index, item in enumerate(self.replacements):
|
|
2582
|
+
at = item.start + shift
|
|
2583
|
+
length = item.end - item.start
|
|
2584
|
+
inserted = self.new_lengths[index]
|
|
2585
|
+
if new0 is not None:
|
|
2586
|
+
# apply_delete (code_text.py:5906-5927)
|
|
2587
|
+
if new0 >= at + length:
|
|
2588
|
+
new0 -= length
|
|
2589
|
+
new1 -= length
|
|
2590
|
+
elif new0 >= at:
|
|
2591
|
+
if new1 <= at + length:
|
|
2592
|
+
new0 = None # span fully inside the name
|
|
2593
|
+
else:
|
|
2594
|
+
# The span's head held the name, whether it began
|
|
2595
|
+
# inside it or exactly on it; either way the
|
|
2596
|
+
# pseudonym ends up outside the span.
|
|
2597
|
+
new0 = at
|
|
2598
|
+
new1 -= length
|
|
2599
|
+
head_cut = True
|
|
2600
|
+
elif new1 > at:
|
|
2601
|
+
# This branch also fires for a span that STRICTLY
|
|
2602
|
+
# CONTAINS the name, where nothing is cut at all: the
|
|
2603
|
+
# span shrinks by the name and grows by the pseudonym
|
|
2604
|
+
# through the insert below. Only an end at or inside
|
|
2605
|
+
# the name's end really loses the pseudonym, so only
|
|
2606
|
+
# that is recorded as a cut.
|
|
2607
|
+
if new1 <= at + length:
|
|
2608
|
+
tail_cut = True
|
|
2609
|
+
new1 -= min(new1, at + length) - at
|
|
2610
|
+
if new1 <= new0:
|
|
2611
|
+
new0 = None
|
|
2612
|
+
if new0 is not None:
|
|
2613
|
+
# apply_insert (code_text.py:5893-5904)
|
|
2614
|
+
if new0 >= at:
|
|
2615
|
+
new0 += inserted
|
|
2616
|
+
new1 += inserted
|
|
2617
|
+
if keep_start_anchor and original_pos0 == 0:
|
|
2618
|
+
new0 = 0
|
|
2619
|
+
elif new0 < at < new1:
|
|
2620
|
+
new1 += inserted
|
|
2621
|
+
shift += inserted - length
|
|
2622
|
+
if new0 is None:
|
|
2623
|
+
return RemappedSpan(None, None, DELETED, False, True)
|
|
2624
|
+
# A span anchored at the file start never moves its start, so a
|
|
2625
|
+
# head cut on it was undone in the same step and is not a cut.
|
|
2626
|
+
anchored = keep_start_anchor and original_pos0 == 0
|
|
2627
|
+
cut = tail_cut or (head_cut and not anchored)
|
|
2628
|
+
return self._classify(pos0, pos1, new0, new1, cut)
|
|
2629
|
+
|
|
2630
|
+
# -- shared ------------------------------------------------------------
|
|
2631
|
+
|
|
2632
|
+
def _substitutes(self, pos0: int, pos1: int, new0: int, new1: int
|
|
2633
|
+
) -> bool:
|
|
2634
|
+
"""Whether the old span WAS one replaced name and the new span
|
|
2635
|
+
IS that name's pseudonym, both boundaries on the edit's own.
|
|
2636
|
+
|
|
2637
|
+
Under `snap_to_pseudonym` the second half follows from the first
|
|
2638
|
+
by the arithmetic of `map_start` and `map_end`; under
|
|
2639
|
+
`qualcoder_edit_parity` such a span is deleted before it reaches
|
|
2640
|
+
`_classify`. Both halves are checked all the same, because the
|
|
2641
|
+
class is defined by both (owner ruling 7.3(3)) and a third
|
|
2642
|
+
policy would inherit the definition rather than the coincidence.
|
|
2643
|
+
"""
|
|
2644
|
+
index = bisect_left(self.starts, pos0)
|
|
2645
|
+
if index >= len(self.starts) or self.starts[index] != pos0 \
|
|
2646
|
+
or self.ends[index] != pos1:
|
|
2647
|
+
return False
|
|
2648
|
+
start = self.starts[index] + self.cumulative[index]
|
|
2649
|
+
return (new0, new1) == (start, start + self.new_lengths[index])
|
|
2650
|
+
|
|
2651
|
+
def _classify(self, pos0: int, pos1: int, new0: int, new1: int,
|
|
2652
|
+
cut: bool) -> RemappedSpan:
|
|
2653
|
+
"""One vocabulary for both policies.
|
|
2654
|
+
|
|
2655
|
+
`snapped` means the run had to MOVE a boundary because it met a
|
|
2656
|
+
replaced name: under `snap_to_pseudonym` a boundary that lay
|
|
2657
|
+
strictly inside a name snapped outward to contain the whole
|
|
2658
|
+
pseudonym; under `qualcoder_edit_parity` a head or tail cut
|
|
2659
|
+
excluded the pseudonym from the span. `substituted` is a span
|
|
2660
|
+
that sat exactly on one replaced name and now sits exactly on
|
|
2661
|
+
that name's pseudonym, whatever the two lengths: the coder
|
|
2662
|
+
marked the name and still marks it under its new spelling, so
|
|
2663
|
+
no coding decision changed (owner ruling 7.3(3) of 2026-09-15,
|
|
2664
|
+
refining X1; before it this span was classified by length, a
|
|
2665
|
+
shift when the lengths matched and a resize when they did not).
|
|
2666
|
+
`resized` is any other length change, which is what a span that
|
|
2667
|
+
contains a name without being the name gets when the pseudonym
|
|
2668
|
+
is a different length: "Mr Thomas said" grows or shrinks with the
|
|
2669
|
+
pseudonym and records the same decision about the same words, so
|
|
2670
|
+
ruling 7.4 of 2026-09-16 exempts it from the hidden-coder
|
|
2671
|
+
override as well. `shifted` is a span that kept its length and
|
|
2672
|
+
moved, which includes one containing a name whose pseudonym is
|
|
2673
|
+
exactly as long: the coder still marks the same passage.
|
|
2674
|
+
|
|
2675
|
+
The order matters. A cut is a snap before anything else, so no
|
|
2676
|
+
span that had to grow can read as a substitution. `clamped` is
|
|
2677
|
+
decided in `map_row`, which alone holds the flag, and it comes
|
|
2678
|
+
before every test here but the two whose classes are gated
|
|
2679
|
+
already.
|
|
2680
|
+
"""
|
|
2681
|
+
touched = self._intersects_edit(pos0, pos1) or cut
|
|
2682
|
+
if cut:
|
|
2683
|
+
change = SNAPPED
|
|
2684
|
+
elif self._substitutes(pos0, pos1, new0, new1):
|
|
2685
|
+
change = SUBSTITUTED
|
|
2686
|
+
elif (new1 - new0) != (pos1 - pos0):
|
|
2687
|
+
change = RESIZED
|
|
2688
|
+
elif (new0, new1) != (pos0, pos1):
|
|
2689
|
+
change = SHIFTED
|
|
2690
|
+
else:
|
|
2691
|
+
change = UNCHANGED
|
|
2692
|
+
return RemappedSpan(new0, new1, change, False, touched)
|
|
2693
|
+
|
|
2694
|
+
def map_row(self, pos0: Any, pos1: Any, policy: str,
|
|
2695
|
+
keep_start_anchor: bool) -> Optional[RemappedSpan]:
|
|
2696
|
+
"""Map one stored row, or None when the row is not mappable.
|
|
2697
|
+
|
|
2698
|
+
Returning None is the "leave it exactly as it is" answer for a
|
|
2699
|
+
row whose stored positions are not a span: NULL, not an integer,
|
|
2700
|
+
NEGATIVE, or `pos0 >= pos1`. None refuses the run; all are
|
|
2701
|
+
listed. A negative position is a damaged row QualCoder never
|
|
2702
|
+
writes, and mapping one would write it back negative and could
|
|
2703
|
+
collide with the parked values the write uses; left alone and
|
|
2704
|
+
reported, it is a row the researcher can go and look at (QA F-9).
|
|
2705
|
+
|
|
2706
|
+
A `pos1` past the end of the text is CLAMPED on the old side
|
|
2707
|
+
before mapping and reported, which is master's own rule applied a
|
|
2708
|
+
step earlier (`code_text.py:6232-6233`, "clamp, never delete a
|
|
2709
|
+
valid code"). Clamping before rather than after is what makes
|
|
2710
|
+
the guarantee that no row is written with an end past the new
|
|
2711
|
+
text hold under both policies rather than only under one.
|
|
2712
|
+
|
|
2713
|
+
A clamped row has a class of its own, `clamped`, which nothing
|
|
2714
|
+
else produces. The classes describe what happens to the STORED
|
|
2715
|
+
span, and a clamp truncates it: the annotation stored at
|
|
2716
|
+
(76, 200) on an 81-character text is written as (66, 71), which
|
|
2717
|
+
is a 119-character truncation however the rest of the run moved
|
|
2718
|
+
it. Classifying that as `shifted` put it under ruling X1's
|
|
2719
|
+
exemption and let a hidden coder's row be cut back with no
|
|
2720
|
+
override asked for (Security S5); classifying it as `resized`
|
|
2721
|
+
kept it gated until ruling 7.4 of 2026-09-16 exempted the
|
|
2722
|
+
ordinary resize, which is why the clamp now has a class instead
|
|
2723
|
+
of borrowing one. The same holds when the clamp lands the
|
|
2724
|
+
truncated span exactly on a name that ends the text: the coder
|
|
2725
|
+
never marked that name alone, so the row is `clamped` and not a
|
|
2726
|
+
substitution. Only a snap and a deletion keep their own class
|
|
2727
|
+
through a clamp, both being on the override side already; the
|
|
2728
|
+
test is written that way round so that a class added later
|
|
2729
|
+
stays gated rather than exempt by omission.
|
|
2730
|
+
"""
|
|
2731
|
+
if isinstance(pos0, bool) or isinstance(pos1, bool):
|
|
2732
|
+
return None
|
|
2733
|
+
if not isinstance(pos0, int) or not isinstance(pos1, int):
|
|
2734
|
+
return None
|
|
2735
|
+
if pos0 < 0 or pos1 < 0:
|
|
2736
|
+
return None
|
|
2737
|
+
clamped = False
|
|
2738
|
+
if pos1 > self.old_len:
|
|
2739
|
+
pos1 = self.old_len
|
|
2740
|
+
clamped = True
|
|
2741
|
+
if pos0 >= pos1:
|
|
2742
|
+
return None
|
|
2743
|
+
if policy == "qualcoder_edit_parity":
|
|
2744
|
+
mapped = self._parity(pos0, pos1, keep_start_anchor)
|
|
2745
|
+
elif policy == "snap_to_pseudonym":
|
|
2746
|
+
mapped = self._snap(pos0, pos1)
|
|
2747
|
+
else:
|
|
2748
|
+
raise ValueError(
|
|
2749
|
+
f"overlap_policy must be one of "
|
|
2750
|
+
f"{', '.join(OVERLAP_POLICIES)}.")
|
|
2751
|
+
mapped.clamped = clamped
|
|
2752
|
+
if clamped and mapped.change not in (SNAPPED, DELETED):
|
|
2753
|
+
mapped.change = CLAMPED
|
|
2754
|
+
return mapped
|
|
2755
|
+
|
|
2756
|
+
|
|
2757
|
+
def unique_constraint_collisions(keys: Sequence[Tuple[Any, ...]],
|
|
2758
|
+
row_ids: Sequence[Any],
|
|
2759
|
+
shown=None) -> List[Dict[str, Any]]:
|
|
2760
|
+
"""Rows that would land on the same unique key after the remap.
|
|
2761
|
+
|
|
2762
|
+
`code_text` is unique on (cid, fid, pos0, pos1, owner) and
|
|
2763
|
+
`annotation` on (fid, pos0, pos1, owner) in QualCoder's own schema
|
|
2764
|
+
(`__main__.py:1819-1821` and `:1800-1801` at master, in that order,
|
|
2765
|
+
identical at the 3.8.2 tag). Two rows can collapse onto one span when both were cut by
|
|
2766
|
+
the same name: one marking "Thomas" and one marking "Thom" both snap
|
|
2767
|
+
to the pseudonym. Rare enough that the right answer is a human
|
|
2768
|
+
decision, so the preview lists it and the execute refuses.
|
|
2769
|
+
|
|
2770
|
+
Rows are GROUPED on the whole key, because that is the constraint,
|
|
2771
|
+
and REPORTED under `shown(key)` when a caller gives one. The
|
|
2772
|
+
database layer uses it to leave the owner column out of what the
|
|
2773
|
+
preview carries: the key's owner can be a coder the project hides,
|
|
2774
|
+
and two rows of one hidden coder cut by one name is an ordinary
|
|
2775
|
+
shape (one marking "Thomas", one marking "Thom"), so the owner in
|
|
2776
|
+
the reported key named a hidden coder in a preview whose every
|
|
2777
|
+
other field withholds the name (X1). `row_ids` identify the rows
|
|
2778
|
+
on their own.
|
|
2779
|
+
"""
|
|
2780
|
+
groups: Dict[Tuple[Any, ...], List[Any]] = {}
|
|
2781
|
+
for key, row_id in zip(keys, row_ids):
|
|
2782
|
+
groups.setdefault(key, []).append(row_id)
|
|
2783
|
+
return [{"key": list(key if shown is None else shown(key)),
|
|
2784
|
+
"row_ids": sorted(ids)}
|
|
2785
|
+
for key, ids in sorted(groups.items(), key=lambda kv: str(kv[0]))
|
|
2786
|
+
if len(ids) > 1]
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def context_for(text: str, start: int, end: int, chars: int) -> str:
|
|
2790
|
+
"""The text around one match, for `include_context`.
|
|
2791
|
+
|
|
2792
|
+
This returns FILE CONTENT, which is why the argument is off by
|
|
2793
|
+
default and why the tool description says so in as many words.
|
|
2794
|
+
"""
|
|
2795
|
+
chars = max(0, min(int(chars), MAX_CONTEXT_CHARS))
|
|
2796
|
+
return text[max(0, start - chars):min(len(text), end + chars)]
|