backrefs 8.2.1__tar.gz → 8.3__tar.gz

{backrefs-8.2.1 → backrefs-8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: backrefs
-Version: 8.2.1
+Version: 8.3
 Summary: A wrapper around re and regex that adds additional back references.
 Project-URL: Homepage, https://github.com/facelessuser/backrefs
 Author-email: Isaac Muse <Isaac.Muse@gmail.com>

{backrefs-8.2.1 → backrefs-8.3}/backrefs/__meta__.py

@@ -193,5 +193,5 @@ def parse_version(ver: str) -> Version:
     return Version(major, minor, micro, release, pre, post, dev)
 
 
-__version_info__ = Version(8, 2, 1, "final")
+__version_info__ = Version(8, 3, 0, "final")
 __version__ = __version_info__._get_canonical()

{backrefs-8.2.1 → backrefs-8.3}/backrefs/_bre_parse.py

@@ -77,6 +77,7 @@ class _SearchParser(Generic[AnyStr]):
     verbose: bool
     unicode: bool
     named_unicode: tuple[str, int]
+    negated_members: list[str]
     ascii: bool  # noqa: A003
     is_bytes: bool
     search: AnyStr
@@ -437,6 +438,8 @@ class _SearchParser(Generic[AnyStr]):
         start = ('', 0, None)  # type: tuple[str, int, int | None]
         self.found_property = False
         self.found_named_unicode = False
+        self.negated_members = []
+        negated_set = False
 
         def add_member(raw: str, index: int, value: int | None) -> None:
             """Track a member that was just added, which may start or end a range."""
@@ -522,6 +525,7 @@ class _SearchParser(Generic[AnyStr]):
                     # Only a `^` directly after the opening `[` negates the set.
                     first = pos
                     current.append(t)
+                    negated_set = True
                 elif t == "]" and found and (pos != first + 1):
                     found = False
                     current.append(t)
@@ -552,7 +556,9 @@ class _SearchParser(Generic[AnyStr]):
         # This will occur when a property's values exceed
         # either the Unicode char limit on a narrow system,
         # or the ASCII limit in a byte string pattern.
-        if found_property or self.found_named_unicode:
+        if self.negated_members:
+            current = [self.negated_set_text(''.join(current), negated_set)]
+        elif found_property or self.found_named_unicode:
             temp = "".join(current)
             if temp == '[]':
                 # We specified some properties, but they are all
@@ -565,6 +571,32 @@ class _SearchParser(Generic[AnyStr]):
 
         return current
 
+    def negated_set_text(self, text: str, negated: bool) -> str:
+        r"""
+        Get a character set that contains negated properties, such as `[a\P{Lu}]`.
+
+        Under case-insensitive matching, a negated property must be a negated set of the property's characters
+        (`[^...]`), as a set of the characters outside the property would also match the other case of the
+        characters in it. A set can't contain a negated set, so each part of the set is matched separately:
+        `[R\P{A}\P{B}]` becomes `(?:[R]|[^A]|[^B])`, and `[^R\P{A}\P{B}]` becomes `(?![R])(?=[A])[B]`.
+        """
+
+        members = text[2 if negated else 1:-1]
+        if members.startswith('^'):
+            # A literal `^` after the negating `^` would negate the members' own set.
+            members = '\\' + members
+
+        if not negated:
+            alternatives = [f'[^{m}]' for m in self.negated_members]
+            if members:
+                alternatives.insert(0, f'[{members}]')
+            return alternatives[0] if len(alternatives) == 1 else f"(?:{'|'.join(alternatives)})"
+
+        conditions = [f'(?=[{m}])' for m in self.negated_members[:-1]]
+        if members:
+            conditions.insert(0, f'(?![{members}])')
+        return ''.join(conditions) + f'[{self.negated_members[-1]}]'
+
     def normal(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle normal chars."""
 
@@ -680,14 +712,23 @@ class _SearchParser(Generic[AnyStr]):
         else:
             mode = _uniprops.MODE_UNICODE
 
-        v = _uniprops.get_unicode_property(props, prop_value, mode)
-        if not in_group:
-            if not v:
-                v = f'^{_uniprops.ASCII_RANGE if self.is_bytes else _uniprops.UNICODE_RANGE}'
-            v = f"[{v}]"
-        properties = [v]
-
-        return properties
+        # Negated properties are expressed as a negated set of the property's characters, so that they work with
+        # case-insensitive matching (see `negated_set_text`).
+        v, negated = _uniprops.get_unicode_property_parts(props, prop_value, mode)
+        everything = _uniprops.ASCII_RANGE if self.is_bytes else _uniprops.UNICODE_RANGE
+        if in_group:
+            if negated:
+                if not v:
+                    # The inverse of a property that matches nothing is everything, which case-insensitive
+                    # matching can't extend, so it can be a member of the set.
+                    return [everything]
+                # Added when the set is complete.
+                self.negated_members.append(v)
+                return ['']
+            return [v]
+        if negated:
+            return [f'[^{v}]' if v else f'[{everything}]']
+        return [f'[{v}]' if v else f'[^{everything}]']
 
     def main_group(self, i: _util.StringIter) -> list[str]:
         """The main group: group 0."""

{backrefs-8.2.1 → backrefs-8.3}/backrefs/uniprops/__init__.py

@@ -1,5 +1,6 @@
 """Unicode Properties."""
 from __future__ import annotations
+from typing import Callable
 from .unidata import alias
 
 UNICODE_RANGE = '\u0000-\U0010ffff'
@@ -487,7 +488,7 @@ def get_vertical_orientation_property(value: str, mode: int = MODE_UNICODE) -> s
 
 
 def get_is_property(value: str, mode: int = MODE_UNICODE) -> str:
-    """Get shortcut for `SC` or `Binary` property."""
+    """Get shortcut for `GC`, `SC`, or `Binary` property."""
 
     from .unidata import scriptextensions as scx
     from .unidata import binary
@@ -504,6 +505,10 @@ def get_is_property(value: str, mode: int = MODE_UNICODE) -> str:
     if prefix != 'is':
         raise ValueError("Does not start with 'is'!")
 
+    category = _general_category_name(temp)
+    if category is not None:
+        return get_gc_property(negate + category, mode)
+
     script_obj = scx.ascii_script_extensions if mode != MODE_UNICODE else scx.unicode_script_extensions
     bin_obj = binary.ascii_binary if mode != MODE_UNICODE else binary.unicode_binary
 
@@ -549,86 +554,100 @@ def _is_binary(name: str) -> bool:
     return name in prop_table.unicode_binary or name in alias.unicode_alias['binary']
 
 
+def _general_category_name(value: str) -> str | None:
+    """Get the short name of a General Category value, or `None` if it isn't one."""
+
+    from .unidata import generalcategory as prop_table
+
+    name = alias.unicode_alias['generalcategory'].get(value, value)
+    table = prop_table.unicode_properties
+    if len(name) == 1:
+        valid = name in table
+    else:
+        valid = len(name) == 2 and name[0] in table and name[1] in table[name[0]]
+    return name if valid else None
+
+
+def _strip_is(name: str, is_valid: Callable[[str], bool]) -> str:
+    """
+    Remove an initial `is` from a name that isn't valid, if it is valid without it.
+
+    Unicode allows an initial `is` on property names and values (UAX #44, `UAX44-LM3`). The name as written is
+    always preferred, and `is` is only removed once. A negated name (`^name`) keeps its negation.
+    """
+
+    negate = '^' if name.startswith('^') else ''
+    base = name[len(negate):]
+    if not is_valid(base) and base.startswith('is') and is_valid(base[2:]):
+        base = base[2:]
+    return negate + base
+
+
+# Properties that have values, by their normalized name.
+_ENUMERATED_PROPERTIES: dict[str, Callable[[str, int], str]] = {
+    'generalcategory': get_gc_property,
+    'script': get_script_property,
+    'scriptextensions': get_script_extension_property,
+    'block': get_block_property,
+    'bidiclass': get_bidi_property,
+    'bidipairedbrackettype': get_bidi_paired_bracket_type_property,
+    'age': get_age_property,
+    'eastasianwidth': get_east_asian_width_property,
+    'indicpositionalcategory': get_indic_positional_category_property,
+    'indicsyllabiccategory': get_indic_syllabic_category_property,
+    'hangulsyllabletype': get_hangul_syllable_type_property,
+    'decompositiontype': get_decomposition_type_property,
+    'canonicalcombiningclass': get_canonical_combining_class_property,
+    'numerictype': get_numeric_type_property,
+    'numericvalue': get_numeric_value_property,
+    'joiningtype': get_joining_type_property,
+    'joininggroup': get_joining_group_property,
+    'graphemeclusterbreak': get_grapheme_cluster_break_property,
+    'linebreak': get_line_break_property,
+    'sentencebreak': get_sentence_break_property,
+    'wordbreak': get_word_break_property,
+    'nfcquickcheck': get_nfc_quick_check_property,
+    'nfdquickcheck': get_nfd_quick_check_property,
+    'nfkcquickcheck': get_nfkc_quick_check_property,
+    'nfkdquickcheck': get_nfkd_quick_check_property,
+    'verticalorientation': get_vertical_orientation_property,
+}
+
+
+def _is_property_name(name: str) -> bool:
+    """Check if a name is a property that can be given a value (`name=value`)."""
+
+    return _is_binary(name) or alias.unicode_alias['_'].get(name, name) in _ENUMERATED_PROPERTIES
+
+
 def get_unicode_property(prop: str, value: str | None = None, mode: int = MODE_UNICODE) -> str:
     """Retrieve the Unicode category from the table."""
 
     if value is not None:
 
         negate = prop.startswith('^')
+        name = _strip_is(prop[1:] if negate else prop, _is_property_name)
 
         # Normalize binary true/false input so we can handle it properly
-        if _is_binary(prop):
-            name = prop[1:] if negate else prop
-
+        if _is_binary(name):
             if value in ('n', 'no', 'f', 'false'):
                 negate = not negate
             elif value not in ('y', 'yes', 't', 'true'):
                 raise ValueError(f"'{value}' is not a valid value for the binary property '{prop}'")
 
             return get_binary_property('^' + name if negate else name, mode)
-        else:
-            if negate:
-                value = '^' + value
-                name = prop[1:]
-            else:
-                name = prop
+
+        if negate:
+            value = '^' + value
 
         name = alias.unicode_alias['_'].get(name, name)
         try:
             if name == 'generalcategory':
-                return get_gc_property(value, mode)
-            elif name == 'script':
-                return get_script_property(value, mode)
-            elif name == 'scriptextensions':
-                return get_script_extension_property(value, mode)
-            elif name == 'block':
-                return get_block_property(value, mode)
-            elif name == 'bidiclass':
-                return get_bidi_property(value, mode)
-            elif name == 'bidipairedbrackettype':
-                return get_bidi_paired_bracket_type_property(value, mode)
-            elif name == 'age':
-                return get_age_property(value, mode)
-            elif name == 'eastasianwidth':
-                return get_east_asian_width_property(value, mode)
-            elif name == 'indicpositionalcategory':
-                return get_indic_positional_category_property(value, mode)
-            elif name == 'indicsyllabiccategory':
-                return get_indic_syllabic_category_property(value, mode)
-            elif name == 'hangulsyllabletype':
-                return get_hangul_syllable_type_property(value, mode)
-            elif name == 'decompositiontype':
-                return get_decomposition_type_property(value, mode)
-            elif name == 'canonicalcombiningclass':
-                return get_canonical_combining_class_property(value, mode)
-            elif name == 'numerictype':
-                return get_numeric_type_property(value, mode)
-            elif name == 'numericvalue':
-                return get_numeric_value_property(value, mode)
-            elif name == 'joiningtype':
-                return get_joining_type_property(value, mode)
-            elif name == 'joininggroup':
-                return get_joining_group_property(value, mode)
-            elif name == 'graphemeclusterbreak':
-                return get_grapheme_cluster_break_property(value, mode)
-            elif name == 'linebreak':
-                return get_line_break_property(value, mode)
-            elif name == 'sentencebreak':
-                return get_sentence_break_property(value, mode)
-            elif name == 'wordbreak':
-                return get_word_break_property(value, mode)
-            elif name == 'nfcquickcheck':
-                return get_nfc_quick_check_property(value, mode)
-            elif name == 'nfdquickcheck':
-                return get_nfd_quick_check_property(value, mode)
-            elif name == 'nfkcquickcheck':
-                return get_nfkc_quick_check_property(value, mode)
-            elif name == 'nfkdquickcheck':
-                return get_nfkd_quick_check_property(value, mode)
-            elif name == 'verticalorientation':
-                return get_vertical_orientation_property(value, mode)
-            else:
+                value = _strip_is(value, lambda v: _general_category_name(v) is not None)
+            get_property = _ENUMERATED_PROPERTIES.get(name)
+            if get_property is None:
                 raise ValueError(f"'{prop}={value}' does not have a valid property name")
+            return get_property(value, mode)
         except Exception as e:
             raise ValueError(f"'{prop}={value}' does not appear to be a valid property") from e
 
@@ -663,3 +682,21 @@ def get_unicode_property(prop: str, value: str | None = None, mode: int = MODE_U
         pass
 
     raise ValueError(f"'{prop}' does not appear to be a valid property")
+
+
+def get_unicode_property_parts(prop: str, value: str | None = None, mode: int = MODE_UNICODE) -> tuple[str, bool]:
+    """
+    Retrieve the characters of a Unicode property without its negation, and whether it is negated.
+
+    A property is negated when it is an inverse property (`^name`). With the characters of the property itself,
+    a negated property can be expressed as a negated character set (`[^...]`), which case-insensitive matching
+    handles correctly, unlike a set of the characters outside the property.
+
+    A binary property with a false value (`name=no`) is not a negation: it is the set of characters with that
+    value, as it is in Regex.
+    """
+
+    negate = prop.startswith('^')
+    if negate:
+        prop = prop[1:]
+    return get_unicode_property(prop, value, mode), negate

{backrefs-8.2.1 → backrefs-8.3}/docs/src/dictionary/en-custom.txt

@@ -27,6 +27,8 @@ Sep
 Tidelift
 Tox
 Twemoji
+UAX
+UTS
 Zensical
 accessor
 backend
@@ -39,6 +41,7 @@ formatter
 grapheme
 hashable
 incrementing
+lookbehinds
 namespaced
 natively
 parsers

{backrefs-8.2.1 → backrefs-8.3}/docs/src/markdown/about/changelog.md

@@ -3,6 +3,19 @@ icon: lucide/scroll-text
 ---
 # Changelog
 
+## 8.3
+
+-   **NEW**: `bre`: Add the binary properties `Any`, `Assigned`, and `ASCII`, which UTS \#18 requires but the Unicode
+    Character Database does not define. `ASCII` was previously only available as an alias of the `Basic_Latin` block.
+-   **NEW**: `bre`: Allow an initial `Is` on General Category values (`\p{IsLu}` or `\p{gc=IsLu}`) and on property names
+    (`\p{IsGC=Lu}` or `\p{IsAlphabetic=Yes}`), as Unicode allows, per UAX \#44.
+-   **FIX**: `bre`: Fix the inverse of the `No_Block` block property (`\P{No_Block}`) not including the
+    Supplementary Private Use Area blocks (`U+F0000` to `U+10FFFF`).
+-   **FIX**: `bre`: Fix negated Unicode properties and POSIX classes (`\P{Lu}`, `\p{^Lu}`, `[[:^upper:]]`) matching
+    both cases of the characters in the property when using `IGNORECASE`. Character sets that contain negated
+    properties are now expressed as alternatives (`[a\P{Lu}]` becomes `(?:[a]|[^...])`).
+-   **FIX**: `bre`: Fix an error when negating a binary property alias that has a value (`\P{Alpha=No}`).
+
 ## 8.2.1
 
 -   **FIX**: Reject compiled replace templates from another search pattern in compiled search objects,

{backrefs-8.2.1 → backrefs-8.3}/docs/src/markdown/refs.md

@@ -174,6 +174,15 @@ example, to specify characters that are "alphabetic", we can use `\p{Alphabetic:
 > [!new] New 5.4 Custom Binary properties
 > In 5.4, the new custom binary properties `Vert_space` and `Horiz_Space` were added.
 
+The binary properties `Any`, `Assigned`, and `ASCII` are also available. These are not in the Unicode Character
+Database, but are required of regular expression engines by [UTS #18](https://www.unicode.org/reports/tr18/#RL1.2):
+
+Property   | Matches
+---------- | -------
+`Any`      | Every character, even with the `ASCII` flag or in byte strings, as in Regex.
+`Assigned` | Every character that is assigned (everything not in General Category `Cn`).
+`ASCII`    | Every ASCII character (`U+0000` to `U+007F`). This is the same as the `ASCII` alias of the `Basic_Latin` block.
+
 ### Property Short Names
 
 General Category, Script Extensions, Blocks, and Binary all can be specified in a short form using just their name or
@@ -185,9 +194,13 @@ that is used in Script may be used in Blocks etc.
 4.  Binary
 3.  Blocks
 
-Script Extensions and Binary properties can also be defined in the format `IsValue`.  For instance, if we wanted to
-match characters in the `Latin` script, we could use the syntax `\p{IsLatin}`, which would be the same as `\p{Latin}` or
-`\p{scx: Latin}`.  For Binary properties, `\p{IsAlphabetic}` is the same as `\p{Alphabetic: Y}` or `\p{Alphabetic}`.
+General Category, Script Extensions, and Binary properties can also be defined in the format `IsValue`.  For instance,
+if we wanted to match characters in the `Latin` script, we could use the syntax `\p{IsLatin}`, which would be the same as
+`\p{Latin}` or `\p{scx: Latin}`.  For Binary properties, `\p{IsAlphabetic}` is the same as `\p{Alphabetic: Y}` or
+`\p{Alphabetic}`, and for General Category, `\p{IsLu}` is the same as `\p{Lu}`.
+
+As Unicode allows, an initial `Is` is also accepted on property names (`\p{IsGeneral_Category: Lu}` or
+`\p{IsAlphabetic: Yes}`) and on General Category values (`\p{gc: IsLu}`).
 
 Block properties have a similar short form as Script and Binary properties.  For Blocks you can use `InValue` to specify
 a block. If we wanted to match characters in the `Basic_Latin` block, we could use the syntax `\p{InBasic_Latin}`. This

{backrefs-8.2.1 → backrefs-8.3}/tests/test_binary.py

@@ -155,3 +155,44 @@ class TestBinary(unittest.TestCase):
             result1 = uniprops.get_unicode_property(is_k)
             result2 = uniprops.get_unicode_property(is_v)
             self.assertEqual(result1, result2)
+
+    def test_uts18_properties(self):
+        """Test the UTS #18 properties that are not in the Unicode Character Database: `Any`, `Assigned`, `ASCII`."""
+
+        import re._parser as parser
+        from backrefs.uniprops.unidata import generalcategory
+
+        def code_points(text):
+            if not text:
+                return set()
+            node = parser.parse(f'[{text}]')[0]
+            points = set()
+            for op, av in [node] if node[0] is parser.LITERAL else node[1]:
+                points.update([av] if op is parser.LITERAL else range(av[0], av[1] + 1))
+            return points
+
+        everything = set(range(0x110000))
+        ascii_range = set(range(0x80))
+        assigned = everything - code_points(generalcategory.unicode_properties['c']['n'])
+
+        for table, expected in (
+            (binary.unicode_binary, {'any': everything, 'assigned': assigned, 'ascii': ascii_range}),
+            # `Any` is everything even in ASCII mode, as it is in Regex.
+            (binary.ascii_binary, {'any': everything, 'assigned': assigned & ascii_range, 'ascii': ascii_range}),
+        ):
+            for name, points in expected.items():
+                self.assertEqual(code_points(table[name]), points, name)
+                self.assertEqual(code_points(table['^' + name]), everything - points, name)
+
+    def test_uts18_property_forms(self):
+        """Test that the UTS #18 properties can be used like other binary properties."""
+
+        for name in ('any', 'assigned', 'ascii'):
+            positive = uniprops.get_unicode_property(name)
+            negative = uniprops.get_unicode_property('^' + name)
+            for prop, value in (
+                (name, 'yes'), (name, 't'), ('^' + name, 'no'), ('is' + name, None), ('is' + name, 'y')
+            ):
+                self.assertEqual(uniprops.get_unicode_property(prop, value), positive, (prop, value))
+            for prop, value in ((name, 'no'), ('^' + name, None), ('^' + name, 'yes'), ('^is' + name, None)):
+                self.assertEqual(uniprops.get_unicode_property(prop, value), negative, (prop, value))

{backrefs-8.2.1 → backrefs-8.3}/tests/test_block.py

@@ -31,6 +31,59 @@ class TestBlock(unittest.TestCase):
             if not key.startswith('^'):
                 self.assertTrue('^' + key in keys1)
 
+    def test_inverse_is_complement(self):
+        """Test that every inverse block is exactly the code points not in the block."""
+
+        import re._parser as parser
+
+        def ranges(text):
+            """Get the sorted, merged ranges of a property's code points."""
+
+            if not text:
+                return []
+            node = parser.parse(f'[{text}]')[0]
+            items = []
+            for op, av in [node] if node[0] is parser.LITERAL else node[1]:
+                items.append((av, av) if op is parser.LITERAL else av)
+            merged = []  # type: list[tuple[int, int]]
+            for lo, hi in sorted(items):
+                if merged and lo <= merged[-1][1] + 1:
+                    merged[-1] = (merged[-1][0], max(hi, merged[-1][1]))
+                else:
+                    merged.append((lo, hi))
+            return merged
+
+        def complement(values):
+            """Get the ranges of all code points not in the given ranges."""
+
+            result = []
+            start = 0
+            for lo, hi in values:
+                if lo > start:
+                    result.append((start, lo - 1))
+                start = hi + 1
+            if start <= 0x10FFFF:
+                result.append((start, 0x10FFFF))
+            return result
+
+        for table in (block.unicode_blocks, block.ascii_blocks):
+            for key, value in table.items():
+                if not key.startswith('^'):
+                    self.assertEqual(ranges(table['^' + key]), complement(ranges(value)), key)
+
+    def test_no_block_inverse(self):
+        """Test that the inverse of `No_Block` includes the last blocks (Supplementary Private Use Areas)."""
+
+        from backrefs import bre
+
+        not_in_block = bre.compile(r'\p{No_Block}')
+        in_block = bre.compile(r'\P{No_Block}')
+        for c in (0xF0000, 0xFFFFD, 0x100000, 0x10FFFF):
+            self.assertTrue(in_block.fullmatch(chr(c)) is not None, f'U+{c:04X}')
+            self.assertTrue(not_in_block.fullmatch(chr(c)) is None, f'U+{c:04X}')
+        self.assertTrue(not_in_block.fullmatch('\U000effff') is not None)
+        self.assertTrue(in_block.fullmatch('\U000effff') is None)
+
     def test_block(self):
         """Test `Block` properties."""
 

{backrefs-8.2.1 → backrefs-8.3}/tests/test_bre.py

@@ -411,6 +411,117 @@ class TestSearchTemplate(unittest.TestCase):
             with pytest.raises(PatternError, match='unterminated character set'):
                 bre.compile(pattern)
 
+    def test_uts18_properties(self):
+        """Test `Any`, `Assigned`, and `ASCII`, which UTS #18 requires."""
+
+        unassigned = chr(0x0378)
+        for pattern, expected in (
+            (r'\p{Any}', ['A', 'é', unassigned]),
+            (r'\P{Any}', []),
+            (r'\p{Assigned}', ['A', 'é']),
+            (r'\P{Assigned}', [unassigned]),
+            (r'\p{ASCII}', ['A']),
+            (r'\p{ASCII=No}', ['é', unassigned]),
+            (r'[[:^ascii:]x]', ['é', unassigned]),
+        ):
+            p = bre.compile(pattern)
+            self.assertEqual([c for c in ('A', 'é', unassigned) if p.fullmatch(c)], expected, pattern)
+
+        # `ASCII` is also an alias of the `Basic_Latin` block, with the same characters.
+        expected = [chr(c) for c in range(0x100) if c < 0x80]
+        for pattern in (r'\p{ASCII}', r'\p{InASCII}', r'\p{Block=ASCII}', r'\p{IsASCII}'):
+            p = bre.compile(pattern)
+            self.assertEqual([chr(c) for c in range(0x100) if p.fullmatch(chr(c))], expected, pattern)
+
+        # `Any` is every character, even in ASCII mode, as it is in Regex.
+        self.assertTrue(bre.fullmatch(r'\p{Any}', 'é', re.ASCII) is not None)
+        self.assertTrue(bre.fullmatch(r'\P{Any}', 'é', re.ASCII) is None)
+        self.assertTrue(bre.fullmatch(r'\p{Assigned}', 'é', re.ASCII) is None)
+        self.assertEqual(len([c for c in range(256) if bre.fullmatch(rb'\p{Any}', bytes([c]))]), 256)
+        self.assertEqual(len([c for c in range(256) if bre.fullmatch(rb'\p{ASCII}', bytes([c]))]), 128)
+
+    def test_is_prefix(self):
+        """An initial `is` is allowed on property names and General Category values, as Unicode allows."""
+
+        for pattern, expected in (
+            (r'\p{IsLu}', ['A']),
+            (r'\p{Is_Uppercase_Letter}', ['A']),
+            (r'\P{IsLu}', ['a', '1']),
+            (r'\p{^IsLu}', ['a', '1']),
+            (r'[[:IsLu:]]', ['A']),
+            (r'[x\p{IsLu}]', ['A']),
+            (r'\p{gc=IsLu}', ['A']),
+            (r'\p{IsGC=Lu}', ['A']),
+            (r'\p{Is_General_Category=IsLu}', ['A']),
+            (r'\p{IsAlpha=Yes}', ['A', 'a']),
+            (r'\P{IsAlpha=No}', ['A', 'a']),
+            (r'\p{IsScript=Latin}', ['A', 'a']),
+            # Already supported
+            (r'\p{IsLatin}', ['A', 'a']),
+            (r'\p{IsAlpha}', ['A', 'a']),
+        ):
+            p = bre.compile(pattern)
+            self.assertEqual([c for c in 'Aa1' if p.fullmatch(c)], expected, pattern)
+
+        # Only removed once
+        for pattern in (r'\p{IsIsLu}', r'\p{gc=IsIsLu}', r'\p{IsIsGC=Lu}'):
+            with pytest.raises(ValueError):
+                bre.compile(pattern)
+
+        # Values that begin with `is` are unchanged.
+        self.assertTrue(bre.fullmatch(r'\p{lb=IS}', ',') is not None)
+        self.assertTrue(bre.fullmatch(r'\p{ccc=IS}', chr(0x0345)) is not None)
+        self.assertTrue(bre.fullmatch(r'\p{dt=Iso}', chr(0xFE80)) is not None)
+
+    def test_negated_property_ignorecase(self):
+        """Negated properties should not match the other case of characters in the property."""
+
+        def matches(pattern, chars='Aa1x', flags=re.IGNORECASE):
+            p = bre.compile(pattern, flags)
+            return [c for c in chars if p.fullmatch(c)]
+
+        # On their own
+        for pattern in (r'\P{Lu}', r'\p{^Lu}', r'\PL', r'[\P{Lu}]', r'[[:^upper:]]', r'[^\p{Lu}]'):
+            self.assertEqual(matches(pattern, 'Aa1'), ['1'], pattern)
+        self.assertEqual(matches(r'\P{Lu}', 'Aa1', 0), ['a', '1'])
+
+        # With other members of a set
+        self.assertEqual(matches(r'[\P{Lu}x]'), ['1', 'x'])
+        self.assertEqual(matches(r'[x\P{Lu}\d]'), ['1', 'x'])
+        self.assertEqual(matches(r'[\P{Lu}\P{Ll}]'), ['1'])
+        self.assertEqual(matches(r'[^\P{Lu}x]'), ['A', 'a'])
+        self.assertEqual(matches(r'[^\P{Lu}\P{Ll}]'), ['A', 'a', 'x'])
+        self.assertEqual(matches(r'[^\P{Lu}\d]'), ['A', 'a', 'x'])
+        self.assertEqual(matches(r'[\P{Lu}x]', flags=0), ['a', '1', 'x'])
+        self.assertEqual(matches(r'[^\P{Lu}x]', flags=0), ['A'])
+
+        # A literal `^` after the negating `^`
+        self.assertEqual(matches(r'[^^\P{Nd}]', '^1a'), ['1'])
+
+        # Quantifiers and lookbehinds
+        self.assertTrue(bre.fullmatch(r'[\P{Lu}x]{2}', 'X1', re.IGNORECASE) is not None)
+        self.assertTrue(bre.fullmatch(r'[\P{Lu}x]{2}', 'A1', re.IGNORECASE) is None)
+        self.assertTrue(bre.search(r'(?<=[^\P{Lu}x])b', 'ab', re.IGNORECASE) is not None)
+        self.assertTrue(bre.search(r'(?<=[^\P{Lu}x])b', 'xb', re.IGNORECASE) is None)
+        self.assertTrue(bre.search(r'(?<![\P{Lu}x])b', 'Ab', re.IGNORECASE) is not None)
+
+        # Byte strings
+        self.assertEqual([c for c in b'Aa1' if bre.fullmatch(rb'[\P{Lu}x]', bytes([c]), re.IGNORECASE)], [ord('1')])
+
+    def test_binary_property_false_value(self):
+        """A binary property with a false value is the set of characters with that value, not a negation."""
+
+        # So case-insensitive matching applies to that set, as it does in Regex.
+        p = bre.compile(r'[\p{Lowercase=No}x]', re.IGNORECASE)
+        self.assertEqual([c for c in 'Aa1x' if p.fullmatch(c)], ['A', 'a', '1', 'x'])
+
+        # Negations of a property alias with a value
+        for pattern, expected in (
+            (r'\P{Alpha=No}', ['A', 'a']), (r'\p{^Alpha=No}', ['A', 'a']), (r'\P{Alpha=Yes}', ['1', ' ']),
+            (r'\P{Upper=F}', ['A']), (r'\P{WSpace=No}', [' '])
+        ):
+            self.assertEqual([c for c in 'Aa1 ' if bre.fullmatch(pattern, c)], expected, pattern)
+
     def test_caret_after_negation(self):
         """A `^` after the negating `^` is a literal."""
 

{backrefs-8.2.1 → backrefs-8.3}/tests/test_generalcategory.py

@@ -170,3 +170,48 @@ class TestGeneralCategory(unittest.TestCase):
             result1 = uniprops.get_unicode_property(k)
             result2 = uniprops.get_unicode_property(v)
             self.assertEqual(result1, result2)
+
+    def test_is_prefix(self):
+        """Test that an initial `is` is allowed on General Category values and on the property name (`UAX44-LM3`)."""
+
+        table = generalcategory.unicode_properties
+        names = set(table) | {a + b for a in table for b in table[a] if not b.startswith('^')}
+        names |= set(alias.unicode_alias['generalcategory'])
+        for name in names:
+            expected = uniprops.get_unicode_property(name)
+            negated = uniprops.get_unicode_property('^' + name)
+            self.assertEqual(uniprops.get_unicode_property('is' + name), expected, name)
+            self.assertEqual(uniprops.get_unicode_property('^is' + name), negated, name)
+            for prop in ('gc', 'generalcategory', 'isgc', 'isgeneralcategory'):
+                self.assertEqual(uniprops.get_unicode_property(prop, 'is' + name), expected, (prop, name))
+                self.assertEqual(uniprops.get_unicode_property(prop, name), expected, (prop, name))
+                self.assertEqual(uniprops.get_unicode_property('^' + prop, 'is' + name), negated, (prop, name))
+
+        # An initial `is` is only removed once.
+        for args in (('isislu',), ('gc', 'isislu'), ('isisgc', 'lu'), ('is',), ('gc', 'is')):
+            with self.assertRaises(ValueError):
+                uniprops.get_unicode_property(*args)
+
+    def test_is_prefix_unambiguous(self):
+        """
+        Test that no name begins with `is`.
+
+        Removing an initial `is` relies on names being unique even when an initial `is` is ignored, which Unicode
+        guarantees. As no name begins with `is`, that holds for the names Backrefs supports. If a name ever begins
+        with `is`, check that it can't be confused with another name before allowing it here.
+        """
+
+        from backrefs.uniprops.unidata import binary, block, scriptextensions
+
+        table = generalcategory.unicode_properties
+        namespaces = {
+            'General Category': (
+                set(table) | {a + b for a in table for b in table[a]} | set(alias.unicode_alias['generalcategory'])
+            ),
+            'Script': set(scriptextensions.unicode_script_extensions) | set(alias.unicode_alias['script']),
+            'Binary': set(binary.unicode_binary) | set(alias.unicode_alias['binary']),
+            'Block': set(block.unicode_blocks) | set(alias.unicode_alias['block']),
+            'Property names': set(alias.unicode_alias['_']) | set(alias.unicode_alias['_'].values()),
+        }
+        for namespace, names in namespaces.items():
+            self.assertEqual([n for n in names if n.lstrip('^').startswith('is')], [], namespace)

{backrefs-8.2.1 → backrefs-8.3}/tools/unipropgen.py

@@ -316,6 +316,9 @@ def gen_blocks(output, ascii_props=False, append=False, prefix="", aliases=None)
                     if piece[0] > last + 1:
                         no_block_inverse.append((last + 1, piece[0] - 1))
                     last = piece[1]
+                # Include everything after the last range that is not in a block.
+                if last < max_range:
+                    no_block_inverse.append((last + 1, max_range))
             for block, name in ((no_block, 'noblock'), (no_block_inverse, '^noblock')):
                 f.write('\n    "%s": "' % name)
                 for piece in block:
@@ -688,6 +691,14 @@ def gen_binary(table, output, ascii_props=False, append=False, prefix="", aliase
     binary['horizspace'] = [x for x in binary['horizspace'] if x <= max_range]
     binary['vertspace'] = [x for x in binary['vertspace'] if x <= max_range]
 
+    # Properties required by UTS #18 (`RL1.2`) that are not defined in the Unicode Character Database.
+    # `Any` is every character, even in ASCII mode, as it is in Regex. `Assigned` is everything that is not
+    # unassigned (General Category `Cn`).
+    unassigned = set(table['c']['n'])
+    binary['any'] = list(range(0, MAXUNICODE + 1))
+    binary['assigned'] = [x for x in range(0, max_range + 1) if x not in unassigned]
+    binary['ascii'] = list(range(0, min(MAXVALIDASCII, max_range) + 1))
+
     for filename, include in binary_props:
         with open(os.path.join(HOME, 'unicodedata', UNIVERSION, filename), 'r', encoding='utf-8') as uf:
             for line in uf: