backrefs 5.9__tar.gz → 6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {backrefs-5.9 → backrefs-6.0}/PKG-INFO +1 -1
- {backrefs-5.9 → backrefs-6.0}/backrefs/__meta__.py +1 -1
- {backrefs-5.9 → backrefs-6.0}/backrefs/_bre_parse.py +54 -27
- {backrefs-5.9 → backrefs-6.0}/backrefs/_bregex_parse.py +39 -24
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/about/changelog.md +13 -2
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/refs.md +82 -56
- {backrefs-5.9 → backrefs-6.0}/tests/test_bre.py +25 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_bregex.py +6 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unipropgen.py +11 -14
- {backrefs-5.9 → backrefs-6.0}/.coveragerc +0 -0
- {backrefs-5.9 → backrefs-6.0}/.gitignore +0 -0
- {backrefs-5.9 → backrefs-6.0}/.pyspelling.yml +0 -0
- {backrefs-5.9 → backrefs-6.0}/LICENSE.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/README.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/__init__.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/_bregex_typing.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/bre.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/bregex.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/py.typed +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/uniprops/__init__.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/backrefs/util.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/.snippets/links.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/about/contributing.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/about/license.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/about/security.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/index.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/src/markdown/usage.md +0 -0
- {backrefs-5.9 → backrefs-6.0}/docs/theme/announce.html +0 -0
- {backrefs-5.9 → backrefs-6.0}/hatch_build.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/mkdocs.yml +0 -0
- {backrefs-5.9 → backrefs-6.0}/pyproject.toml +0 -0
- {backrefs-5.9 → backrefs-6.0}/requirements/docs.txt +0 -0
- {backrefs-5.9 → backrefs-6.0}/requirements/lint.txt +0 -0
- {backrefs-5.9 → backrefs-6.0}/requirements/test.txt +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/__init__.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_age.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_bidiclass.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_bidipairedbrackettype.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_binary.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_block.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_canonicalcombiningclass.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_decompositiontype.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_eastasianwidth.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_generalcategory.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_graphemeclusterbreak.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_hangulsyllabletype.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_indicpositionalcategory.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_indicsylabiccategory.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_joininggroup.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_joiningtype.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_linebreak.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_numerictype.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_numericvalue.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_quickcheck.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_script.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_scriptextensions.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_sentencebreak.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_uniprops.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_versions.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_verticalorientation.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tests/test_wordbreak.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/__init__.py +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unicodedata/13.0.0.zip +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unicodedata/14.0.0.zip +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unicodedata/15.0.0.zip +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unicodedata/15.1.0.zip +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unicodedata/16.0.0.zip +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unicodedata/LICENSE +0 -0
- {backrefs-5.9 → backrefs-6.0}/tools/unidatadownload.py +0 -0
|
@@ -193,5 +193,5 @@ def parse_version(ver: str) -> Version:
|
|
|
193
193
|
return Version(major, minor, micro, release, pre, post, dev)
|
|
194
194
|
|
|
195
195
|
|
|
196
|
-
__version_info__ = Version(
|
|
196
|
+
__version_info__ = Version(6, 0, 0, "final")
|
|
197
197
|
__version__ = __version_info__._get_canonical()
|
|
@@ -37,11 +37,19 @@ _STANDARD_ESCAPES = frozenset(('a', 'b', 'f', 'n', 'r', 't', 'v'))
|
|
|
37
37
|
_CURLY_BRACKETS = frozenset(('{', '}'))
|
|
38
38
|
_PROPERTY_STRIP = frozenset((' ', '-', '_'))
|
|
39
39
|
_PROPERTY = _WORD | _DIGIT | _PROPERTY_STRIP
|
|
40
|
-
_GLOBAL_FLAGS = frozenset(('a', '
|
|
41
|
-
_SCOPED_FLAGS = frozenset(('i', 'm', 's', '
|
|
40
|
+
_GLOBAL_FLAGS = frozenset(('a', 'i', 'L', 'm', 's', 'u', 'x'))
|
|
41
|
+
_SCOPED_FLAGS = frozenset(('i', 'm', 's', 'x'))
|
|
42
|
+
_SCOPED_END = frozenset((':', ')'))
|
|
42
43
|
|
|
43
44
|
_CURLY_BRACKETS_ORD = frozenset((0x7b, 0x7d))
|
|
44
45
|
|
|
46
|
+
_COMPATIBILITY_PROPERTIES = frozenset(
|
|
47
|
+
(
|
|
48
|
+
'alpha', 'lower', 'upper', 'punct', 'digit', 'xdigit', 'alnum',
|
|
49
|
+
'space', 'blank', 'cntrl', 'graph', 'print', 'word'
|
|
50
|
+
)
|
|
51
|
+
)
|
|
52
|
+
|
|
45
53
|
# Case upper or lower
|
|
46
54
|
_UPPER = 1
|
|
47
55
|
_LOWER = 2
|
|
@@ -173,20 +181,24 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
173
181
|
def flags(self, text: str, scoped: bool = False) -> None:
|
|
174
182
|
"""Analyze flags."""
|
|
175
183
|
|
|
184
|
+
flags = text.split('-')
|
|
185
|
+
enable = flags[0]
|
|
186
|
+
disable = flags[1] if len(flags) > 1 else ''
|
|
187
|
+
|
|
176
188
|
global_retry = False
|
|
177
|
-
if ('a' in
|
|
189
|
+
if ('a' in enable or 'L' in enable) and self.unicode:
|
|
178
190
|
self.unicode = False
|
|
179
191
|
if not scoped:
|
|
180
192
|
self.temp_global_flag_swap["unicode"] = True
|
|
181
193
|
global_retry = True
|
|
182
|
-
elif 'u' in
|
|
194
|
+
elif 'u' in enable and not self.unicode and not self.is_bytes:
|
|
183
195
|
self.unicode = True
|
|
184
196
|
if not scoped:
|
|
185
197
|
self.temp_global_flag_swap["unicode"] = True
|
|
186
198
|
global_retry = True
|
|
187
|
-
if '
|
|
199
|
+
if 'x' in disable and self.verbose:
|
|
188
200
|
self.verbose = False
|
|
189
|
-
elif 'x' in
|
|
201
|
+
elif 'x' in enable and not self.verbose:
|
|
190
202
|
self.verbose = True
|
|
191
203
|
if not scoped:
|
|
192
204
|
self.temp_global_flag_swap["verbose"] = True
|
|
@@ -253,7 +265,15 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
253
265
|
else:
|
|
254
266
|
raise SyntaxError(f"Missing or unmatched '{{' at {index}!") from e
|
|
255
267
|
|
|
256
|
-
|
|
268
|
+
p = ''.join(prop).lower()
|
|
269
|
+
v = ''.join(value).lower()
|
|
270
|
+
|
|
271
|
+
# Ensure when using POSIX form, that any property considered a compatibility property uses the POSIX form.
|
|
272
|
+
# POSIX form is not guaranteed to be different from standard form and sometimes is just an alias for standard.
|
|
273
|
+
if brackets and p in _COMPATIBILITY_PROPERTIES:
|
|
274
|
+
p = 'posix' + p
|
|
275
|
+
|
|
276
|
+
return p, v
|
|
257
277
|
|
|
258
278
|
def get_named_unicode(self, i: _util.StringIter) -> str:
|
|
259
279
|
"""Get Unicode name."""
|
|
@@ -345,51 +365,56 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
345
365
|
|
|
346
366
|
return ''.join(value)
|
|
347
367
|
|
|
348
|
-
def get_flags(self, i: _util.StringIter
|
|
349
|
-
"""
|
|
368
|
+
def get_flags(self, i: _util.StringIter) -> tuple[str | None, bool]:
|
|
369
|
+
"""
|
|
370
|
+
Get flags.
|
|
371
|
+
|
|
372
|
+
In Re, flags are quite predictable when global or scoped.
|
|
373
|
+
Global can never be disabled with minus, and never have a `:` after them.
|
|
374
|
+
The global flag set is also very specific, but can be used as enablers in scoped.
|
|
375
|
+
|
|
376
|
+
The returned scoped status will indicate whether flags are generally considered
|
|
377
|
+
scoped flags or global flags.
|
|
378
|
+
"""
|
|
350
379
|
|
|
351
380
|
index = i.index
|
|
352
381
|
value = ['(']
|
|
353
382
|
toggle = False
|
|
354
|
-
|
|
383
|
+
smells_scoped = False
|
|
355
384
|
try:
|
|
356
385
|
c = next(i)
|
|
357
386
|
if c != '?':
|
|
358
387
|
i.rewind(1)
|
|
359
|
-
return None
|
|
388
|
+
return None, False
|
|
360
389
|
value.append(c)
|
|
361
390
|
c = next(i)
|
|
362
|
-
while c
|
|
391
|
+
while c not in _SCOPED_END:
|
|
363
392
|
if toggle:
|
|
364
393
|
if c not in _SCOPED_FLAGS:
|
|
365
394
|
raise ValueError('Bad scope')
|
|
366
|
-
|
|
367
|
-
|
|
395
|
+
elif c == '-':
|
|
396
|
+
smells_scoped = True
|
|
368
397
|
toggle = True
|
|
369
|
-
elif
|
|
370
|
-
raise ValueError("Bad flag")
|
|
371
|
-
elif c not in _GLOBAL_FLAGS and c not in _SCOPED_FLAGS:
|
|
398
|
+
elif c not in _GLOBAL_FLAGS:
|
|
372
399
|
raise ValueError("Bad flag")
|
|
373
400
|
value.append(c)
|
|
374
401
|
c = next(i)
|
|
402
|
+
if smells_scoped and c != ':':
|
|
403
|
+
raise ValueError("Bad flag")
|
|
404
|
+
elif c == ':':
|
|
405
|
+
smells_scoped = True
|
|
375
406
|
value.append(c)
|
|
376
407
|
except Exception:
|
|
377
408
|
i.rewind(i.index - index)
|
|
378
409
|
value = []
|
|
379
410
|
|
|
380
|
-
return ''.join(value) if value else None
|
|
411
|
+
return ''.join(value) if value else None, smells_scoped
|
|
381
412
|
|
|
382
413
|
def subgroup(self, t: str, i: _util.StringIter) -> list[str]:
|
|
383
414
|
"""Handle parenthesis."""
|
|
384
415
|
|
|
385
416
|
current = [] # type: list[str]
|
|
386
417
|
|
|
387
|
-
# (?flags)
|
|
388
|
-
flags = self.get_flags(i)
|
|
389
|
-
if flags:
|
|
390
|
-
self.flags(flags[2:-1])
|
|
391
|
-
return [flags]
|
|
392
|
-
|
|
393
418
|
# (?#comment)
|
|
394
419
|
comments = self.get_comments(i)
|
|
395
420
|
if comments:
|
|
@@ -398,11 +423,13 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
398
423
|
verbose = self.verbose
|
|
399
424
|
unicode_flag = self.unicode
|
|
400
425
|
|
|
401
|
-
# (?flags:pattern)
|
|
402
|
-
flags = self.get_flags(i
|
|
426
|
+
# (?flags:pattern) or (?flags)
|
|
427
|
+
flags, scoped = self.get_flags(i)
|
|
403
428
|
if flags: # pragma: no cover
|
|
404
429
|
t = flags
|
|
405
|
-
self.flags(flags[2:-1], scoped=
|
|
430
|
+
self.flags(flags[2:-1], scoped=scoped)
|
|
431
|
+
if not scoped:
|
|
432
|
+
return [flags]
|
|
406
433
|
|
|
407
434
|
current = []
|
|
408
435
|
try:
|
|
@@ -30,9 +30,10 @@ _STANDARD_ESCAPES = frozenset(('a', 'b', 'f', 'n', 'r', 't', 'v'))
|
|
|
30
30
|
_CURLY_BRACKETS = frozenset(('{', '}'))
|
|
31
31
|
_PROPERTY_STRIP = frozenset((' ', '-', '_'))
|
|
32
32
|
_PROPERTY = _WORD | _DIGIT | _PROPERTY_STRIP
|
|
33
|
-
_GLOBAL_FLAGS = frozenset(('
|
|
34
|
-
_SCOPED_FLAGS = frozenset(('i', 'm', 's', '
|
|
33
|
+
_GLOBAL_FLAGS = frozenset(('b', 'e', 'p', 'r', 'u'))
|
|
34
|
+
_SCOPED_FLAGS = frozenset(('a', 'f', 'i', 'L', 'm', 's', 'u', 'w', 'x'))
|
|
35
35
|
_VERSIONS = frozenset(('0', '1'))
|
|
36
|
+
_SCOPED_END = frozenset((':', ')'))
|
|
36
37
|
|
|
37
38
|
_CURLY_BRACKETS_ORD = frozenset((0x7b, 0x7d))
|
|
38
39
|
|
|
@@ -163,21 +164,25 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
163
164
|
def flags(self, text: str, scoped: bool = False) -> None:
|
|
164
165
|
"""Analyze flags."""
|
|
165
166
|
|
|
167
|
+
flags = text.split('-')
|
|
168
|
+
enable = flags[0]
|
|
169
|
+
disable = flags[1] if len(flags) > 1 else ''
|
|
170
|
+
|
|
166
171
|
global_retry = False
|
|
167
|
-
if (self.version == _regex.V1 or scoped) and '
|
|
172
|
+
if (self.version == _regex.V1 or scoped) and 'x' in disable and self.verbose:
|
|
168
173
|
self.verbose = False
|
|
169
|
-
elif 'x' in
|
|
174
|
+
elif 'x' in enable and not self.verbose:
|
|
170
175
|
self.verbose = True
|
|
171
176
|
if not scoped and self.version == _regex.V0:
|
|
172
177
|
self.temp_global_flag_swap['verbose'] = True
|
|
173
178
|
global_retry = True
|
|
174
|
-
if 'V0' in
|
|
179
|
+
if 'V0' in enable and self.version == _regex.V1: # pragma: no cover
|
|
175
180
|
# Default is V0 if none is selected,
|
|
176
181
|
# so it is unlikely that this will be selected.
|
|
177
182
|
self.temp_global_flag_swap['version'] = True
|
|
178
183
|
self.version = _regex.V0
|
|
179
184
|
global_retry = True
|
|
180
|
-
elif "V1" in
|
|
185
|
+
elif "V1" in enable and self.version == _regex.V0:
|
|
181
186
|
self.temp_global_flag_swap['version'] = True
|
|
182
187
|
self.version = _regex.V1
|
|
183
188
|
global_retry = True
|
|
@@ -260,54 +265,61 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
260
265
|
|
|
261
266
|
return ''.join(value) if value else None
|
|
262
267
|
|
|
263
|
-
def get_flags(self, i: _util.StringIter
|
|
264
|
-
"""
|
|
268
|
+
def get_flags(self, i: _util.StringIter) -> tuple[str | None, bool]:
|
|
269
|
+
"""
|
|
270
|
+
Get flags.
|
|
271
|
+
|
|
272
|
+
Regex is more difficult to determine when flags are used in a global and scoped context.
|
|
273
|
+
There is a specific list of global flags, but they can be used in scoped notation and will still
|
|
274
|
+
be considered global, but that does not mean other flags are global. Additionally, flags that can be
|
|
275
|
+
scoped can also used be used in global syntax, but can be disabled with a minus unlike in RE.
|
|
276
|
+
|
|
277
|
+
Bregex only cares about capturing the "verbose" flag and the version flags. Version flags are always
|
|
278
|
+
global and verbose flags will be scoped based on whether they are in a scoped group. The returned
|
|
279
|
+
"scoped" parameter only refers to "verbose".
|
|
280
|
+
"""
|
|
265
281
|
|
|
266
282
|
index = i.index
|
|
267
283
|
value = ['(']
|
|
268
284
|
version = False
|
|
269
285
|
toggle = False
|
|
270
|
-
|
|
286
|
+
smells_scoped = False
|
|
271
287
|
try:
|
|
272
288
|
c = next(i)
|
|
273
289
|
if c != '?':
|
|
274
290
|
i.rewind(1)
|
|
275
|
-
return None
|
|
291
|
+
return None, False
|
|
276
292
|
value.append(c)
|
|
277
293
|
c = next(i)
|
|
278
|
-
while c
|
|
294
|
+
while c not in _SCOPED_END:
|
|
279
295
|
if toggle:
|
|
280
296
|
if c not in _SCOPED_FLAGS:
|
|
281
297
|
raise ValueError('Bad scope')
|
|
282
|
-
toggle = False
|
|
283
|
-
elif (not version0 or scoped) and c == '-':
|
|
284
|
-
toggle = True
|
|
285
298
|
elif version:
|
|
286
299
|
if c not in _VERSIONS:
|
|
287
300
|
raise ValueError('Bad version')
|
|
288
301
|
version = False
|
|
302
|
+
elif c == '-':
|
|
303
|
+
toggle = True
|
|
289
304
|
elif c == 'V':
|
|
290
305
|
version = True
|
|
291
306
|
elif c not in _GLOBAL_FLAGS and c not in _SCOPED_FLAGS:
|
|
292
307
|
raise ValueError("Bad flag")
|
|
293
308
|
value.append(c)
|
|
294
309
|
c = next(i)
|
|
310
|
+
if c == ':':
|
|
311
|
+
smells_scoped = True
|
|
312
|
+
|
|
295
313
|
value.append(c)
|
|
296
314
|
except Exception:
|
|
297
315
|
i.rewind(i.index - index)
|
|
298
316
|
value = []
|
|
299
317
|
|
|
300
|
-
return ''.join(value) if value else None
|
|
318
|
+
return ''.join(value) if value else None, smells_scoped
|
|
301
319
|
|
|
302
320
|
def subgroup(self, t: str, i: _util.StringIter) -> list[str]:
|
|
303
321
|
"""Handle parenthesis."""
|
|
304
322
|
|
|
305
|
-
# (?flags)
|
|
306
|
-
flags = self.get_flags(i, self.version == _regex.V0)
|
|
307
|
-
if flags:
|
|
308
|
-
self.flags(flags[2:-1])
|
|
309
|
-
return [flags]
|
|
310
|
-
|
|
311
323
|
# (?#comment)
|
|
312
324
|
comments = self.get_comments(i)
|
|
313
325
|
if comments:
|
|
@@ -315,11 +327,14 @@ class _SearchParser(Generic[AnyStr]):
|
|
|
315
327
|
|
|
316
328
|
verbose = self.verbose
|
|
317
329
|
|
|
318
|
-
# (?flags:pattern)
|
|
319
|
-
|
|
330
|
+
# (?flags:pattern) or (?flags)
|
|
331
|
+
# "scoped" only refers to verbose
|
|
332
|
+
flags, scoped = self.get_flags(i)
|
|
320
333
|
if flags:
|
|
321
334
|
t = flags
|
|
322
|
-
self.flags(flags[2:-1], scoped=
|
|
335
|
+
self.flags(flags[2:-1], scoped=scoped)
|
|
336
|
+
if not scoped:
|
|
337
|
+
return [flags]
|
|
323
338
|
|
|
324
339
|
current = [] # type: list[str]
|
|
325
340
|
try:
|
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 6.0
|
|
4
|
+
|
|
5
|
+
- **NEW**: POSIX character classes will now always use POSIX compatibility rules instead of Unicode standard rules,
|
|
6
|
+
if any are specified in the Unicode specification. The affected character classes are: `[[:alnum:]]`, `[[:digit:]]`,
|
|
7
|
+
`[[:xdigit:]]`, and `[[:punct:]]`. To explicitly use standard Unicode rules for these compatibility properties, use
|
|
8
|
+
the Unicode property form instead: `[\p{Alnum}]`, `[\p{Digit}]`, `[\p{Punct}]`, or `[\p{XDigit}]`. This has changed
|
|
9
|
+
to ensure no confusion for users expecting compatible POSIX style character class properties.
|
|
10
|
+
- **FIX**: Scoped ASCII/Unicode flags (`(?a:pattern)`/`(?u:pattern)`) should be respected for Unicode properties in
|
|
11
|
+
`bre` and will ensure ASCII or Unicode range if used.
|
|
12
|
+
- **FIX**: Fix issues related to detecting disabled scoped flags.
|
|
13
|
+
|
|
3
14
|
## 5.9
|
|
4
15
|
|
|
5
16
|
- **NEW**: Add support for Python 3.14.
|
|
@@ -28,8 +39,8 @@
|
|
|
28
39
|
|
|
29
40
|
## 5.5
|
|
30
41
|
|
|
31
|
-
- **NEW**: `\e` and `\h` have both been deprecated
|
|
32
|
-
|
|
42
|
+
- **NEW**: `\e` and `\h` have both been deprecated. Please migrate to using `\x1b` and `\p{Horiz_Space}` in their
|
|
43
|
+
respective place.
|
|
33
44
|
- **FIX**: Fix flag issue with `sub` functions.
|
|
34
45
|
|
|
35
46
|
## 5.4
|
|
@@ -204,7 +204,7 @@ When it comes to short names, each new Unicode version, there is a risk that new
|
|
|
204
204
|
with existing names and/or aliases. Currently, most of the conflicts involve the Block properties. To reduce
|
|
205
205
|
friction, they are evaluated last.
|
|
206
206
|
|
|
207
|
-
Generally, it is discouraged to use short names for Block properties. But the option is still supported, but Block
|
|
207
|
+
Generally, it is discouraged to use short names for Block properties. But the option is still supported, but Block
|
|
208
208
|
properties will be evaluated last. There are currently no known conflicts with `In*` properties, but in future
|
|
209
209
|
Unicode versions there could.
|
|
210
210
|
|
|
@@ -259,67 +259,18 @@ Verbose\ Property\ Form | Terse\ Property\ Form
|
|
|
259
259
|
`Line_Separator` | `Zl`
|
|
260
260
|
`Paragraph_Separator` | `Z`
|
|
261
261
|
|
|
262
|
-
### POSIX Style Properties
|
|
263
|
-
|
|
264
|
-
/// new | New in 5.0
|
|
265
|
-
5.0 brings significant improvements and bug fixes to Unicode property handling. Properties are sensitive to the
|
|
266
|
-
`ASCII` flag along with more extensive testing and bug fixes. Additionally, POSIX style properties are now just
|
|
267
|
-
an extension of normal Unicode properties. All the POSIX names are available and now conform to
|
|
268
|
-
the [Unicode specification for POSIX compatibility](https://unicode.org/reports/tr18/#Compatibility_Properties).
|
|
269
|
-
Read on to learn more.
|
|
270
|
-
///
|
|
271
|
-
|
|
272
|
-
Backrefs allows for POSIX style properties in the form `[:name:]`. These properties can only be used inside character
|
|
273
|
-
classes and are another form for expressing Unicode properties. Any Unicode property that can be expressed via the
|
|
274
|
-
`\p{name}` form can also be expressed in the `[:name:]` form. To illustrate, the following are all the same:
|
|
275
|
-
|
|
276
|
-
- `[[:upper:]]` == `[\p{upper}]`
|
|
277
|
-
- `[[:^upper:]]` == `[\p{^upper}]`
|
|
278
|
-
- `[[:alpha=yes:]]` == `[\p{alpha=yes}]`
|
|
279
|
-
|
|
280
|
-
A number of POSIX property names are available via compatibility properties as outlined in the
|
|
281
|
-
[Unicode specification for POSIX compatibility](https://unicode.org/reports/tr18/#Compatibility_Properties). These
|
|
282
|
-
properties will operate in the Unicode range and the ASCII range depending on the regular expression mode. These
|
|
283
|
-
patterns, like all Unicode properties, are sensitive to the `ASCII` flag (or `LOCALE` which will treat them as `ASCII`).
|
|
284
|
-
|
|
285
|
-
It is important to note that whether used in the form `[[:name:]]` or `\p{name}`, each POSIX name is available both with
|
|
286
|
-
and without the `posix` prefix, but it is recommended to use the `posix` prefix to get the POSIX definition of the
|
|
287
|
-
pattern as number of patterns have both a POSIX and Unicode definition that differ. The
|
|
288
|
-
[Unicode specification for POSIX compatibility](https://unicode.org/reports/tr18/#Compatibility_Properties) outlines all
|
|
289
|
-
the POSIX compatible properties and the ones which have dual definitions: `punct`, `alnum`, `digit`, and `xdigit` all
|
|
290
|
-
have a Unicode standard and a POSIX compatibility variant and must be accessed with the `posix` prefix to get the POSIX
|
|
291
|
-
form.
|
|
292
|
-
|
|
293
|
-
In the table below, patterns with `--` mean `[[in this] -- [but not this]]`.
|
|
294
|
-
|
|
295
|
-
\[:posix:] | \\p\{Posix} | ASCII | Unicode
|
|
296
|
-
---------- | ------------- | ------------------------------------------------- | -------
|
|
297
|
-
`alpha` | `Alpha` | `[a-zA-Z]` | `\p{Alphabetic}`
|
|
298
|
-
`alnum` | `PosixAlnum` | `[[:alpha:][:digit:]]` | `[[:alpha:][:digit:]]`
|
|
299
|
-
`blank` | `Blank` | `[ \t]` | `[\p{Zs}\t]`
|
|
300
|
-
`cntrl` | `Cntrl` | `[\x00-\x1F\x7F]` | `\p{Cc}`
|
|
301
|
-
`digit` | `PosixDigit` | `[0-9]` | `[0-9]`
|
|
302
|
-
`graph` | `Graph` | `[^ [:cntrl:]]` | `[^[:space:][:cntrl:]\p{Cn}\p{Cs}]`
|
|
303
|
-
`lower` | `Lower` | `[a-z]` | `[\p{Lowercase}]`
|
|
304
|
-
`print` | `Print` | `[[:graph:] ]` | `[[\p{P}\p{S}]--[\p{alpha}]]`
|
|
305
|
-
`punct` | `PosixPunct` | ``[!\"\#$%&'()*+,\-./:;<=>?@\[\\\]^_`{}~]`` | `[[[:graph:][:blank:]]--[[:cntrl:]]]`
|
|
306
|
-
`space` | `Space` | `[ \t\r\n\v\f]` | `[\p{Whitespace}]`
|
|
307
|
-
`upper` | `Upper` | `[A-Z]` | `[\p{Uppercase}]`
|
|
308
|
-
`xdigit` | `PosixXDigit` | `[A-Fa-f0-9]` | `[A-Fa-f0-9]`
|
|
309
|
-
|
|
310
262
|
### Compatibility Properties
|
|
311
263
|
|
|
312
264
|
/// new | New in 5.0
|
|
313
265
|
While many of the properties were available before 5.0, `word` is newly available. And all the properties now
|
|
314
|
-
conform to the [Unicode specification for
|
|
266
|
+
conform to the [Unicode specification for compatibility properties](https://unicode.org/reports/tr18/#Compatibility_Properties).
|
|
315
267
|
///
|
|
316
268
|
|
|
317
|
-
[Unicode specification for
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
In the table below, patterns with `--` mean `[[in this] -- [but not this]]`.
|
|
269
|
+
[Unicode specification for compatibility properties][unicode-posix] defines a number of properties for use in regular
|
|
270
|
+
expressions. The table below shows the names of these properties along with their standard rules as implemented in
|
|
271
|
+
Backrefs. Patterns with `--` mean `[[in this] -- [but not this]]`.
|
|
321
272
|
|
|
322
|
-
\\p\{
|
|
273
|
+
\\p\{name} | Unicode
|
|
323
274
|
------------- | -------
|
|
324
275
|
`Alpha` | `\p{Alphabetic}`
|
|
325
276
|
`Alnum` | `[\p{Alpha}\p{Digit}]`
|
|
@@ -328,9 +279,84 @@ In the table below, patterns with `--` mean `[[in this] -- [but not this]]`.
|
|
|
328
279
|
`Digit` | `\p{Nd}`
|
|
329
280
|
`Graph` | `[^\p{Space}\p{Cntrl}\p{Cn}\p{Cs}]`
|
|
330
281
|
`Lower` | `\p{Lowercase}`
|
|
331
|
-
`Print` | `[
|
|
282
|
+
`Print` | `[\p{graph}\p{blank}--\p{cntrl}]`
|
|
332
283
|
`Punct` | `\p{P}`
|
|
333
284
|
`Space` | `\p{Whitespace}`
|
|
334
285
|
`Upper` | `\p{Uppercase}`
|
|
335
286
|
`Word` | `[\p{Alnum}\p{M}\p{Pc}\p{JoinControl}]`
|
|
336
287
|
`XDigit` | `[\p{Nd}\p{HexDigit}]`
|
|
288
|
+
|
|
289
|
+
The [Unicode specification for compatibility properties](https://unicode.org/reports/tr18/#Compatibility_Properties)
|
|
290
|
+
also defines rules for POSIX compatibility which provides modifications to the standard recommendation required to meet
|
|
291
|
+
the formal requirements of POSIX, and also to maintain (as much as possible) compatibility with the POSIX usage in
|
|
292
|
+
practice.
|
|
293
|
+
|
|
294
|
+
To accommodate these POSIX compatible rules, Backrefs defines POSIX property names for all the compatibility properties.
|
|
295
|
+
For any that do not have special accommodations, the POSIX property name will act as an alias for the standard Unicode
|
|
296
|
+
name. In all, four properties are specifically affected with POSIX specific accommodations: `Alnum`, `Digit`, `Punct`,
|
|
297
|
+
and `XDigit`.
|
|
298
|
+
|
|
299
|
+
\\p\{name} | Unicode
|
|
300
|
+
------------- | -------
|
|
301
|
+
`PosixAlpha` | `\p{Alpha}`
|
|
302
|
+
`PosixAlnum` | `[\p{Alpha}\p{PosixDigit}]`
|
|
303
|
+
`PosixBlank` | `\p{Blank}`
|
|
304
|
+
`PosixCntrl` | `\p{Cntrl}`
|
|
305
|
+
`PosixDigit` | `[0-9]`
|
|
306
|
+
`PosixGraph` | `\p{Graph}`
|
|
307
|
+
`PosixLower` | `\p{Lower}`
|
|
308
|
+
`PosixPrint` | `\p{Print}`
|
|
309
|
+
`PosixPunct` | `[[\p{P}\p{S}]--[\p{alpha}]]`
|
|
310
|
+
`PosixSpace` | `\p{Space}`
|
|
311
|
+
`PosixUpper` | `\p{Uppper}`
|
|
312
|
+
`PosixWord` | `\p{Word}`
|
|
313
|
+
`PosixXDigit` | `[A-Fa-f0-9]`
|
|
314
|
+
|
|
315
|
+
### POSIX Character Classes
|
|
316
|
+
|
|
317
|
+
/// new | New in 6.0
|
|
318
|
+
POSIX character classes in the form of `[:name:]` are now forced to always use POSIX compatible rules if available.
|
|
319
|
+
These rules are defined in the [Unicode specification for compatibility properties]
|
|
320
|
+
(https://unicode.org/reports/tr18/#Compatibility_Properties).
|
|
321
|
+
///
|
|
322
|
+
|
|
323
|
+
Backrefs allows for POSIX style character classes in the form of `[:name:]`. These properties can only be used inside
|
|
324
|
+
character classes, e.g. `[[:name:]]`.
|
|
325
|
+
|
|
326
|
+
When using POSIX character classes, all character classes will use the matching rules as stated in the
|
|
327
|
+
[Unicode specification for compatibility properties](https://unicode.org/reports/tr18/#Compatibility_Properties). If
|
|
328
|
+
there are specific rules for POSIX compatibility, those will be used in the bracket expression instead of the Unicode
|
|
329
|
+
standard form. For all others, the Unicode standard form will be used.
|
|
330
|
+
|
|
331
|
+
The table below outlines the supported POSIX names that can be used in the form `[:name:]` and details the differences
|
|
332
|
+
in rules if using ASCII mode or Unicode mode.
|
|
333
|
+
|
|
334
|
+
\[:posix:] | ASCII | Unicode
|
|
335
|
+
---------- | ------------------------------------------------------------- | -------
|
|
336
|
+
`alpha` | `[a-zA-Z]` | `\p{Alphabetic}`
|
|
337
|
+
`alnum` | `[[:alpha:][:digit:]]` | `[[:alpha:][:digit:]]`
|
|
338
|
+
`blank` | `[ \t]` | `[\p{Zs}\t]`
|
|
339
|
+
`cntrl` | `[\x00-\x1F\x7F]` | `\p{Cc}`
|
|
340
|
+
`digit` | `[0-9]` | `[0-9]`
|
|
341
|
+
`graph` | `[^ [:cntrl:]]` | `[^[:space:][:cntrl:]\p{Cn}\p{Cs}]`
|
|
342
|
+
`lower` | `[a-z]` | `[\p{Lowercase}]`
|
|
343
|
+
`print` | `[[:graph:] ]` | `[[[:graph:][:blank:]]--[[:cntrl:]]]`
|
|
344
|
+
`punct` | ``[!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~]`` | `[[\p{P}\p{S}]--[\p{alpha}]]`
|
|
345
|
+
`space` | `[ \t\r\n\v\f]` | `[\p{Whitespace}]`
|
|
346
|
+
`upper` | `[A-Z]` | `[\p{Uppercase}]`
|
|
347
|
+
`word` | `[A-Za-z0-9_]` | `[\p{Alnum}\p{M}\p{Pc}\p{JoinControl}]`
|
|
348
|
+
`xdigit` | `[A-Fa-f0-9]` | `[A-Fa-f0-9]`
|
|
349
|
+
|
|
350
|
+
/// Tip
|
|
351
|
+
Using POSIX character classes is equivalent to using Unicode compatibility properties for POSIX, e.g. `[[:punct:]]`
|
|
352
|
+
== `\p{Posix Punct}`.
|
|
353
|
+
///
|
|
354
|
+
|
|
355
|
+
POSIX style character classes have also been extended to allow for specifying normal Unicode properties as well. Any
|
|
356
|
+
Unicode property, outside the compatibility properties, will be accepted in the bracket extension form and behave just
|
|
357
|
+
like normal Unicode properties. To illustrate, the following are equivalent:
|
|
358
|
+
|
|
359
|
+
- `[[:math:]]` == `[\p{math}]`
|
|
360
|
+
- `[[:scx=latin:]]` == `[\p{scx: latin}]`
|
|
361
|
+
- `[[:^scx=latin:]]` == `[\P{scx: latin}]`
|
|
362
|
+
|
|
@@ -11,6 +11,7 @@ import copy
|
|
|
11
11
|
|
|
12
12
|
PY39_PLUS = (3, 9) <= sys.version_info
|
|
13
13
|
PY311_PLUS = (3, 11) <= sys.version_info
|
|
14
|
+
PY313_PLUS = (3, 13) <= sys.version_info
|
|
14
15
|
|
|
15
16
|
if PY311_PLUS:
|
|
16
17
|
import re._constants as _constants
|
|
@@ -21,6 +22,20 @@ else:
|
|
|
21
22
|
class TestSearchTemplate(unittest.TestCase):
|
|
22
23
|
"""Search template tests."""
|
|
23
24
|
|
|
25
|
+
def test_multi_disable_flag(self):
|
|
26
|
+
"""Test when multiple flags are disabled."""
|
|
27
|
+
|
|
28
|
+
self.assertTrue(bre.match(r'(?-ix:Test Test)', 'Test Test', flags=bre.IGNORECASE | bre.VERBOSE))
|
|
29
|
+
self.assertTrue(bre.match(r'(?s-ix:Test Test)', 'Test Test', flags=bre.IGNORECASE | bre.VERBOSE))
|
|
30
|
+
|
|
31
|
+
def test_inline_unicode(self):
|
|
32
|
+
"""Test inline Unicode/ASCII cases."""
|
|
33
|
+
|
|
34
|
+
self.assertTrue(bre.match(r'\p{N}', '\uff19', flags=bre.ASCII) is None)
|
|
35
|
+
self.assertTrue(bre.match(r'(?u:\p{N})', '\uff19', flags=bre.ASCII) is not None)
|
|
36
|
+
self.assertTrue(bre.match(r'\p{N}', '\uff19', flags=bre.UNICODE) is not None)
|
|
37
|
+
self.assertTrue(bre.match(r'(?a:\p{N})', '\uff19', flags=bre.UNICODE) is None)
|
|
38
|
+
|
|
24
39
|
def test_custom_binary_properties(self):
|
|
25
40
|
"""Test new custom binary properties."""
|
|
26
41
|
|
|
@@ -1986,6 +2001,16 @@ class TestReplaceTemplate(unittest.TestCase):
|
|
|
1986
2001
|
class TestExceptions(unittest.TestCase):
|
|
1987
2002
|
"""Test Exceptions."""
|
|
1988
2003
|
|
|
2004
|
+
def test_bad_flag(self):
|
|
2005
|
+
"""Test bad flag."""
|
|
2006
|
+
|
|
2007
|
+
if PY313_PLUS:
|
|
2008
|
+
with self.assertRaises(re.PatternError):
|
|
2009
|
+
bre.compile(r'(?-i)')
|
|
2010
|
+
else:
|
|
2011
|
+
with self.assertRaises(re.error):
|
|
2012
|
+
bre.compile(r'(?-i)')
|
|
2013
|
+
|
|
1989
2014
|
def test_format_existing_group_no_match_with_index(self):
|
|
1990
2015
|
"""Test format group with no match and attempt at indexing."""
|
|
1991
2016
|
|
|
@@ -20,6 +20,12 @@ PY39_PLUS = (3, 9) <= sys.version_info
|
|
|
20
20
|
class TestSearchTemplate(unittest.TestCase):
|
|
21
21
|
"""Search template tests."""
|
|
22
22
|
|
|
23
|
+
def test_multi_disable_flag(self):
|
|
24
|
+
"""Test when multiple flags are disabled."""
|
|
25
|
+
|
|
26
|
+
self.assertTrue(bregex.match(r'(?-ix:Test Test)', 'Test Test', flags=bregex.IGNORECASE | bregex.VERBOSE))
|
|
27
|
+
self.assertTrue(bregex.match(r'(?s-ix:Test Test)', 'Test Test', flags=bregex.IGNORECASE | bregex.VERBOSE))
|
|
28
|
+
|
|
23
29
|
def test_named_lists_attribute(self):
|
|
24
30
|
"""Test named list attribute."""
|
|
25
31
|
|
|
@@ -740,16 +740,6 @@ def gen_binary(table, output, ascii_props=False, append=False, prefix="", aliase
|
|
|
740
740
|
|
|
741
741
|
gen_uposix(table, binary, ascii_props)
|
|
742
742
|
|
|
743
|
-
# One off Unicode property found under https://unicode.org/reports/tr18/#Compatibility_Properties
|
|
744
|
-
# `Word: [\p{alnum}\p{M}\p{Pc}\p{JoinControl}]`
|
|
745
|
-
s = set(binary["alnum"])
|
|
746
|
-
for k, v in table['m'].items():
|
|
747
|
-
if not k.startswith('^'):
|
|
748
|
-
s |= set(v)
|
|
749
|
-
s |= set(table["p"]["c"])
|
|
750
|
-
s |= set(binary["joincontrol"])
|
|
751
|
-
binary["word"] = list(s)
|
|
752
|
-
|
|
753
743
|
if aliases:
|
|
754
744
|
for v in aliases.get('binary', {}).values():
|
|
755
745
|
if v not in binary and '^' + v not in binary:
|
|
@@ -881,9 +871,14 @@ def gen_uposix(table, posix_table, ascii_props):
|
|
|
881
871
|
s -= set(posix_table["posixcntrl"])
|
|
882
872
|
posix_table["posixprint"] = list(s)
|
|
883
873
|
|
|
884
|
-
# `
|
|
885
|
-
s = set(
|
|
886
|
-
|
|
874
|
+
# `Word: [\p{alnum}\p{M}\p{Pc}\p{JoinControl}]`
|
|
875
|
+
s = set(posix_table["alnum"])
|
|
876
|
+
for k, v in table['m'].items():
|
|
877
|
+
if not k.startswith('^'):
|
|
878
|
+
s |= set(v)
|
|
879
|
+
s |= set(table["p"]["c"])
|
|
880
|
+
s |= set(posix_table["joincontrol"])
|
|
881
|
+
posix_table['posixword'] = list(s)
|
|
887
882
|
|
|
888
883
|
|
|
889
884
|
def gen_alias(nonbinary, binary, output):
|
|
@@ -904,9 +899,11 @@ def gen_alias(nonbinary, binary, output):
|
|
|
904
899
|
'posixlower': 'lowercase',
|
|
905
900
|
'posixupper': 'uppercase',
|
|
906
901
|
'posixspace': 'whitespace',
|
|
902
|
+
'posixascii': 'basiclatin',
|
|
907
903
|
'blank': 'posixblank',
|
|
908
904
|
'graph': 'posixgraph',
|
|
909
|
-
'print': 'posixprint'
|
|
905
|
+
'print': 'posixprint',
|
|
906
|
+
'word': 'posixword'
|
|
910
907
|
}
|
|
911
908
|
toplevel = (
|
|
912
909
|
'catalog', 'enumerated', 'numeric', 'miscellaneous'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|