backrefs 8.2__tar.gz → 8.3__tar.gz

{backrefs-8.2 → backrefs-8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: backrefs
-Version: 8.2
+Version: 8.3
 Summary: A wrapper around re and regex that adds additional back references.
 Project-URL: Homepage, https://github.com/facelessuser/backrefs
 Author-email: Isaac Muse <Isaac.Muse@gmail.com>

{backrefs-8.2 → backrefs-8.3}/backrefs/__meta__.py

@@ -193,5 +193,5 @@ def parse_version(ver: str) -> Version:
     return Version(major, minor, micro, release, pre, post, dev)
 
 
-__version_info__ = Version(8, 2, 0, "final")
+__version_info__ = Version(8, 3, 0, "final")
 __version__ = __version_info__._get_canonical()

{backrefs-8.2 → backrefs-8.3}/backrefs/_bre_parse.py

@@ -6,18 +6,13 @@ Copyright (c) 2011 - 2020 Isaac Muse <isaacmuse@gmail.com>
 """
 from __future__ import annotations
 import re as _re
-import sys
+import re._parser as _parser  # type: ignore[import]
 import copyreg as _copyreg
 from . import util as _util
 from .util import PatternError
 from . import uniprops as _uniprops
 from typing import Generic, AnyStr, Match, Any, Pattern, cast
 
-if sys.version_info >= (3, 11):
-    import re._parser as _parser  # type: ignore[import]
-else:
-    import sre_parse as _parser
-
 __all__ = ("ReplaceTemplate", "PatternError")
 
 _ASCII_LETTERS = frozenset(
@@ -76,11 +71,13 @@ class _SearchParser(Generic[AnyStr]):
     _re_start_wb = r"\b(?=\w)"
     _re_end_wb = r"\b(?<=\w)"
     _line_break = r'(?:\r\n|(?!\r\n)[\n\v\f\r\x85\u2028\u2029])'
-    _bytes_line_break = r'(?>\r\n|[\n\v\f\r\x85])' if _util.PY311 else r'(?:\r\n|(?!\r\n)[\n\v\f\r\x85])'
+    _bytes_line_break = r'(?>\r\n|[\n\v\f\r\x85])'
     _grapheme_cluster = r'(?:{}{}*(?!{}))'
 
     verbose: bool
     unicode: bool
+    named_unicode: tuple[str, int]
+    negated_members: list[str]
     ascii: bool  # noqa: A003
     is_bytes: bool
     search: AnyStr
@@ -283,6 +280,7 @@ class _SearchParser(Generic[AnyStr]):
             current.extend(self.unicode_name(text, in_group))
             if in_group:
                 self.found_named_unicode = True
+                self.named_unicode = (f'\\N{{{text}}}', ord(_util.unicode_char(text)))
         else:
             current.extend(["\\", t])
         return current
@@ -418,27 +416,57 @@ class _SearchParser(Generic[AnyStr]):
         self.verbose = verbose
         self.unicode = unicode_flag
 
-        if t == ")":
-            current.append(t)
+        current.append(t)
         return current
 
     def char_groups(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle character groups."""
 
-        current = []
+        current = []  # type: list[str]
         pos = i.index - 1
         found = False
         escaped = False
+        escape_index = 0
         first = 0
         found_property = False
+        # Index of an unescaped `-` in `current` that is acting as a range operator.
+        range_dash = -1
+        # Whether the last member could start a range if followed by `-`.
+        range_start = False
+        # The last member's original text, position, and code point if it is a named character
+        # beyond the byte range of a byte string.
+        start = ('', 0, None)  # type: tuple[str, int, int | None]
         self.found_property = False
         self.found_named_unicode = False
+        self.negated_members = []
+        negated_set = False
+
+        def add_member(raw: str, index: int, value: int | None) -> None:
+            """Track a member that was just added, which may start or end a range."""
+
+            nonlocal range_dash, range_start, start
+
+            if range_dash != -1:
+                # This member ends a range.
+                start_raw, start_index, start_value = start
+                if start_value is not None or value is not None:
+                    end = self.get_named_byte_range(start_raw, raw, start_value, value, start_index)
+                    if end is None:
+                        current[range_dash] = ''
+                    else:
+                        current[-1] = end
+                range_dash = -1
+                range_start = False
+            else:
+                start = (raw, index, value)
+                range_start = True
 
         try:
             while True:
                 # Prevent POSIX/Unicode class from being part of a range.
                 if self.found_property and t == '-':
                     current.append(_re.escape(t))
+                    add_member(t, i.index - 1, None)
                     pos += 1
                     t = next(i)
                     self.found_property = False
@@ -448,15 +476,26 @@ class _SearchParser(Generic[AnyStr]):
 
                 if not escaped and t == "\\":
                     escaped = True
+                    escape_index = i.index - 1
                 elif escaped:
                     escaped = False
                     idx = len(current) - 1
                     current.extend(self.reference(t, i, True))
                     if self.found_property:
                         # Prevent Unicode class from being part of a range.
-                        if idx >= 0 and current[idx] == '-':
+                        if idx == range_dash:
                             current[idx] = _re.escape('-')
                         found_property = True
+                        range_dash = -1
+                        range_start = False
+                    elif t == 'N':
+                        raw, code = self.named_unicode
+                        add_member(raw, escape_index, code if self.is_bytes and code > 0xFF else None)
+                    else:
+                        digits = self.get_escape_digits(i, t)
+                        if digits:
+                            current.append(digits)
+                        add_member(f'\\{t}{digits}', escape_index, None)
                 elif t == "[" and not found:
                     found = True
                     first = pos
@@ -470,22 +509,44 @@ class _SearchParser(Generic[AnyStr]):
                         i.rewind(i.index - index)
                     if prop is not None:
                         value = self.unicode_props(prop[0], prop[1], in_group=True)
-                        if current[-1] == '-':
+                        # Prevent POSIX class from being part of a range.
+                        if len(current) - 1 == range_dash:
                             current[-1] = _re.escape('-')
                         current.extend(value)
                         found_property = True
+                        self.found_property = True
                         pos = i.index - 2
+                        range_dash = -1
+                        range_start = False
                     else:
                         current.append(t)
-                elif t == "^" and found and (pos == first + 1):
+                        add_member(t, index - 1, None)
+                elif t == "^" and found and (pos == first + 1) and len(current) == 1:
+                    # Only a `^` directly after the opening `[` negates the set.
                     first = pos
                     current.append(t)
+                    negated_set = True
                 elif t == "]" and found and (pos != first + 1):
                     found = False
                     current.append(t)
                     break
-                else:
+                elif t == '-' and range_start:
+                    # A `-` after a member starts a range, unless it is followed by `]`.
                     current.append(t)
+                    range_dash = len(current) - 1
+                    range_start = False
+                else:
+                    if (
+                        t == '^' and
+                        (found_property or self.found_named_unicode) and
+                        ''.join(current) in ('[', '[^')
+                    ):
+                        # Everything before this was a property that resolved to nothing,
+                        # so escape `^` to prevent it from being treated as a negation.
+                        current.append(_re.escape(t))
+                    else:
+                        current.append(t)
+                    add_member(t, i.index - 1, None)
                 pos += 1
                 t = next(i)
         except StopIteration as e:
@@ -495,7 +556,9 @@ class _SearchParser(Generic[AnyStr]):
         # This will occur when a property's values exceed
         # either the Unicode char limit on a narrow system,
         # or the ASCII limit in a byte string pattern.
-        if found_property or self.found_named_unicode:
+        if self.negated_members:
+            current = [self.negated_set_text(''.join(current), negated_set)]
+        elif found_property or self.found_named_unicode:
             temp = "".join(current)
             if temp == '[]':
                 # We specified some properties, but they are all
@@ -508,6 +571,32 @@ class _SearchParser(Generic[AnyStr]):
 
         return current
 
+    def negated_set_text(self, text: str, negated: bool) -> str:
+        r"""
+        Get a character set that contains negated properties, such as `[a\P{Lu}]`.
+
+        Under case-insensitive matching, a negated property must be a negated set of the property's characters
+        (`[^...]`), as a set of the characters outside the property would also match the other case of the
+        characters in it. A set can't contain a negated set, so each part of the set is matched separately:
+        `[R\P{A}\P{B}]` becomes `(?:[R]|[^A]|[^B])`, and `[^R\P{A}\P{B}]` becomes `(?![R])(?=[A])[B]`.
+        """
+
+        members = text[2 if negated else 1:-1]
+        if members.startswith('^'):
+            # A literal `^` after the negating `^` would negate the members' own set.
+            members = '\\' + members
+
+        if not negated:
+            alternatives = [f'[^{m}]' for m in self.negated_members]
+            if members:
+                alternatives.insert(0, f'[{members}]')
+            return alternatives[0] if len(alternatives) == 1 else f"(?:{'|'.join(alternatives)})"
+
+        conditions = [f'(?=[{m}])' for m in self.negated_members[:-1]]
+        if members:
+            conditions.insert(0, f'(?![{members}])')
+        return ''.join(conditions) + f'[{self.negated_members[-1]}]'
+
     def normal(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle normal chars."""
 
@@ -540,6 +629,60 @@ class _SearchParser(Generic[AnyStr]):
                 return ['']
         return [f'\\{value:03o}' if value <= 0xFF else chr(value)]
 
+    def get_escape_digits(self, i: _util.StringIter, e: str) -> str:
+        r"""
+        Get the digits that are part of an escape in a character set (`\x`, `\u`, `\U`, or octal).
+
+        This allows an escape to be treated as a single member of the character set.
+        """
+
+        if e == 'x':
+            count, valid = 2, _HEX
+        elif e == 'u' and not self.is_bytes:
+            count, valid = 4, _HEX
+        elif e == 'U' and not self.is_bytes:
+            count, valid = 8, _HEX
+        elif e in _OCTAL:
+            count, valid = 2, _OCTAL
+        else:
+            return ''
+
+        digits = []
+        while count:
+            try:
+                c = next(i)
+            except StopIteration:
+                break
+            if c.lower() not in valid:
+                i.rewind(1)
+                break
+            digits.append(c)
+            count -= 1
+        return ''.join(digits)
+
+    @staticmethod
+    def get_named_byte_range(
+        start: str,
+        end: str,
+        start_value: int | None,
+        end_value: int | None,
+        index: int
+    ) -> str | None:
+        """
+        Get the end of a range in a byte string where either end is a named character beyond the byte range.
+
+        `start_value` and `end_value` are the code points of the ends that are named characters beyond the byte
+        range, and `None` for ends within the byte range. `start` and `end` are the original text of each end,
+        and `index` is where the range starts, both used for errors.
+
+        A range that ends beyond the byte range includes everything up to the end of the byte range, so return
+        the end to use in its place. A range that starts beyond the byte range matches nothing, so return `None`.
+        """
+
+        if start_value is not None and (end_value is None or end_value < start_value):
+            raise PatternError(f'bad character range {start}-{end} at position {index}')
+        return None if start_value is not None else '\\377'
+
     def unicode_props(
         self,
         props: str,
@@ -569,14 +712,23 @@ class _SearchParser(Generic[AnyStr]):
         else:
             mode = _uniprops.MODE_UNICODE
 
-        v = _uniprops.get_unicode_property(props, prop_value, mode)
-        if not in_group:
-            if not v:
-                v = f'^{_uniprops.ASCII_RANGE if self.is_bytes else _uniprops.UNICODE_RANGE}'
-            v = f"[{v}]"
-        properties = [v]
-
-        return properties
+        # Negated properties are expressed as a negated set of the property's characters, so that they work with
+        # case-insensitive matching (see `negated_set_text`).
+        v, negated = _uniprops.get_unicode_property_parts(props, prop_value, mode)
+        everything = _uniprops.ASCII_RANGE if self.is_bytes else _uniprops.UNICODE_RANGE
+        if in_group:
+            if negated:
+                if not v:
+                    # The inverse of a property that matches nothing is everything, which case-insensitive
+                    # matching can't extend, so it can be a member of the set.
+                    return [everything]
+                # Added when the set is complete.
+                self.negated_members.append(v)
+                return ['']
+            return [v]
+        if negated:
+            return [f'[^{v}]' if v else f'[{everything}]']
+        return [f'[{v}]' if v else f'[^{everything}]']
 
     def main_group(self, i: _util.StringIter) -> list[str]:
         """The main group: group 0."""
@@ -1183,7 +1335,7 @@ class _ReplaceParser(Generic[AnyStr]):
         count = len(self.span_stack)
         self.end_found = False
         try:
-            while not self.end_found:
+            while True:
                 t = next(i)
                 if self.use_format and t in _CURLY_BRACKETS:
                     self.handle_format(t, i)
@@ -1236,10 +1388,9 @@ class _ReplaceParser(Generic[AnyStr]):
                 except StopIteration:
                     self.result.append(t)
                     raise
-            elif self.single_stack:
-                this_case = self.get_single_stack()
-                if this_case is not None:
-                    self.result.append(self.convert_case(t, this_case))
+            else:
+                # The stack always has the case just added.
+                self.result.append(self.convert_case(t, cast(int, self.get_single_stack())))
         except StopIteration:
             pass
 

{backrefs-8.2 → backrefs-8.3}/backrefs/_bregex_parse.py

@@ -33,8 +33,16 @@ _LETTERS_UNDERSCORE = _ASCII_LETTERS | frozenset(('_',))
 _WORD = _LETTERS_UNDERSCORE | _DIGIT
 _STANDARD_ESCAPES = frozenset(('a', 'b', 'f', 'n', 'r', 't', 'v'))
 _CURLY_BRACKETS = frozenset(('{', '}'))
-_PROPERTY_STRIP = frozenset((' ', '-', '_'))
-_PROPERTY = _WORD | _DIGIT | _PROPERTY_STRIP
+# Character set grammar, as defined by Regex.
+_ALNUM = _ASCII_LETTERS | _DIGIT
+_PROPERTY_NAME = _ALNUM | frozenset((' ', '&', '_', '-', '.'))
+_PROPERTY_VALUE = _ALNUM | frozenset((' ', '&', '_', '-', '.', '/'))
+_NAMED_CHAR = _ALNUM | frozenset((' ', '-'))
+_HEX_DIGITS = _HEX | frozenset(c.upper() for c in _HEX)
+_HEX_ESCAPES = {'x': 2, 'u': 4, 'U': 8}
+_SHORT_PROPERTIES = frozenset(('C', 'L', 'M', 'N', 'P', 'S', 'Z'))
+_CLASS_ESCAPES = frozenset(('d', 'D', 'h', 's', 'S', 'w', 'W'))
+_SET_OPS = frozenset(('||', '~~', '&&', '--'))
 _GLOBAL_FLAGS = frozenset(('b', 'e', 'p', 'r', 'V'))
 _SCOPED_FLAGS = frozenset(('a', 'f', 'i', 'L', 'm', 's', 'u', 'w', 'x'))
 _VERSIONS = frozenset(('0', '1'))
@@ -72,6 +80,7 @@ class _SearchParser(Generic[AnyStr]):
     version: int
     is_bytes: bool
     search: AnyStr
+    text: str
 
     def __init__(self, search: AnyStr, re_verbose: bool = False, re_version: int = 0) -> None:
         """Initialize."""
@@ -167,37 +176,6 @@ class _SearchParser(Generic[AnyStr]):
             current.extend(["\\", t])
         return current
 
-    def get_posix(self, i: _util.StringIter) -> str | None:
-        """Get POSIX."""
-
-        index = i.index
-        value = ['[']
-        try:
-            c = next(i)
-            if c != ':':
-                raise PatternError('Not a valid property!')
-            else:
-                value.append(c)
-                c = next(i)
-                if c == '^':
-                    value.append(c)
-                    c = next(i)
-                while c != ':':
-                    if c not in _PROPERTY:
-                        raise PatternError('Not a valid property!')
-                    if c not in _PROPERTY_STRIP:
-                        value.append(c)
-                    c = next(i)
-                value.append(c)
-                c = next(i)
-                if c != ']' or not value:
-                    raise PatternError('Unmatched ]')
-                value.append(c)
-        except Exception:
-            i.rewind(i.index - index)
-            value = []
-        return ''.join(value) if value else None
-
     def get_comments(self, i: _util.StringIter) -> str | None:
         """Get comments."""
 
@@ -342,69 +320,175 @@ class _SearchParser(Generic[AnyStr]):
             current.append(t)
         return current
 
+    @staticmethod
+    def peek(i: _util.StringIter, count: int = 1) -> str:
+        """Get up to the next `count` characters without consuming them."""
+
+        chars = []
+        for _ in range(count):
+            try:
+                chars.append(next(i))
+            except StopIteration:
+                break
+        i.rewind(len(chars))
+        return ''.join(chars)
+
+    @staticmethod
+    def match(i: _util.StringIter, text: str) -> bool:
+        """Consume `text` if it is next."""
+
+        index = i.index
+        for c in text:
+            try:
+                if next(i) == c:
+                    continue
+            except StopIteration:
+                pass
+            i.rewind(i.index - index)
+            return False
+        return True
+
+    @staticmethod
+    def consume(i: _util.StringIter, chars: frozenset[str], limit: int = -1) -> str:
+        """Consume characters while they are in `chars`, up to `limit` characters if given."""
+
+        value = []
+        while limit:
+            try:
+                c = next(i)
+            except StopIteration:
+                break
+            if c not in chars:
+                i.rewind(1)
+                break
+            value.append(c)
+            limit -= 1
+        return ''.join(value)
+
     def char_groups(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle character groups."""
 
-        current = []
-        pos = i.index - 1
-        found = 0
-        sub_first = 0
-        escaped = False
-        first = 0
-
+        start = i.index - 1
         try:
-            while True:
-                if not escaped and t == "\\":
-                    escaped = True
-                elif escaped:
-                    escaped = False
-                    current.extend(self.reference(t, i, True))
-                elif t == "[" and not found:
-                    found += 1
-                    first = pos
-                    current.append(t)
-                elif t == "[" and found and self.version == _regex.V1:
-                    # Start of sub char set found
-                    posix = None if self.is_bytes else self.get_posix(i)
-                    if posix:
-                        current.append(posix)
-                        pos = i.index - 2
-                    else:
-                        found += 1
-                        sub_first = pos
-                        current.append(t)
-                elif t == "[":
-                    posix = None if self.is_bytes else self.get_posix(i)
-                    if posix:
-                        current.append(posix)
-                        pos = i.index - 2
-                    else:
-                        current.append(t)
-                elif t == "^" and found == 1 and (pos == first + 1):
-                    # Found ^ at start of first char set; adjust 1st char position
-                    current.append(t)
-                    first = pos
-                elif self.version == _regex.V1 and t == "^" and found > 1 and (pos == sub_first + 1):
-                    # Found ^ at start of sub char set; adjust 1st char sub position
-                    current.append(t)
-                    sub_first = pos
-                elif t == "]" and found == 1 and (pos != first + 1):
-                    # First char set closed; log range
-                    current.append(t)
-                    found = 0
-                    break
-                elif self.version == _regex.V1 and t == "]" and found > 1 and (pos != sub_first + 1):
-                    # Sub char set closed; decrement depth counter
-                    found -= 1
-                    current.append(t)
-                else:
-                    current.append(t)
-                pos += 1
-                t = next(i)
+            self.parse_set(i)
         except StopIteration as e:
-            raise PatternError(f"Missing ']', unterminated character set at position {pos}") from e
+            raise PatternError(f"Missing ']', unterminated character set at position {start}") from e
+
+        return [self.text[start:i.index]]
+
+    def parse_set(self, i: _util.StringIter) -> None:
+        """Parse a character set, starting after its opening `[`, through its closing `]`."""
+
+        self.match(i, '^')
+        while True:
+            self.parse_set_operand(i)
+            if self.version == _regex.V1 and self.peek(i, 2) in _SET_OPS:
+                next(i)
+                next(i)
+                continue
+            # The operand only ends at an operator or `]`.
+            next(i)
+            return
+
+    def parse_set_operand(self, i: _util.StringIter) -> None:
+        """Parse the members of a set (or of an operand in a set operation) up to an operator or `]`."""
+
+        # The first member is always a member, even if it is `]`.
+        self.parse_set_member(i)
+        while True:
+            c = self.peek(i, 2)
+            if not c:
+                raise StopIteration
+            if c[0] == ']' or (self.version == _regex.V1 and c in _SET_OPS):
+                return
+            self.parse_set_member(i)
+
+    def parse_set_member(self, i: _util.StringIter) -> None:
+        """Parse a member of a character set, which may be a range."""
+
+        if not self.parse_set_item(i) or self.peek(i) != '-':
+            return
+
+        after = self.peek(i, 2)
+        if self.version == _regex.V1 and after == '--':
+            # A set difference, not a range.
+            return
+        next(i)
+        if after == '-]':
+            # A literal `-` at the end of the set.
+            return
+        self.parse_set_item(i)
+
+    def parse_set_item(self, i: _util.StringIter) -> bool:
+        """Parse a single item of a character set, and return whether it is a single character."""
 
-        return current
+        c = next(i)
+        if c == '\\':
+            return self.parse_set_escape(i)
+        if c == '[':
+            if self.parse_posix(i):
+                return False
+            if self.version == _regex.V1:
+                # A nested set.
+                self.parse_set(i)
+                return False
+        return True
+
+    def parse_set_escape(self, i: _util.StringIter) -> bool:
+        """Parse an escape in a character set, and return whether it is a single character."""
+
+        c = next(i)
+        if c in _HEX_ESCAPES:
+            self.consume(i, _HEX_DIGITS, _HEX_ESCAPES[c])
+        elif c == 'N':
+            # A named character, or otherwise a literal `N`.
+            index = i.index
+            if self.match(i, '{'):
+                self.consume(i, _NAMED_CHAR)
+                if self.match(i, '}'):
+                    return True
+            i.rewind(i.index - index)
+        elif c in ('p', 'P'):
+            index = i.index
+            if self.match(i, '{'):
+                self.match(i, '^')
+                self.parse_property_name(i)
+                if self.match(i, '}'):
+                    return False
+                # Not a property, so a literal `p` or `P`.
+                i.rewind(i.index - index)
+            elif self.peek(i) in _SHORT_PROPERTIES:
+                next(i)
+                return False
+        elif c in _CLASS_ESCAPES:
+            return False
+        elif c in _DIGIT:
+            self.consume(i, _OCTAL, 2)
+        return True
+
+    def parse_posix(self, i: _util.StringIter) -> bool:
+        """Parse a POSIX class after an opening `[`, or consume nothing if it isn't one."""
+
+        index = i.index
+        if self.match(i, ':'):
+            self.match(i, '^')
+            self.parse_property_name(i)
+            if self.match(i, ':]'):
+                return True
+        i.rewind(i.index - index)
+        return False
+
+    def parse_property_name(self, i: _util.StringIter) -> None:
+        """Parse a property name, which may be qualified with a value: `name`, `name=value`, or `name:value`."""
+
+        self.consume(i, _PROPERTY_NAME)
+        index = i.index
+        if self.peek(i) in (':', '='):
+            next(i)
+            if self.consume(i, _PROPERTY_VALUE).strip():
+                return
+        # No value, so the name is unqualified.
+        i.rewind(i.index - index)
 
     def normal(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle normal chars."""
@@ -445,7 +529,8 @@ class _SearchParser(Generic[AnyStr]):
         self.verbose = bool(self.re_verbose)
         self.version = self.re_version if self.re_version else _regex.DEFAULT_VERSION
         new_pattern = []
-        i = _util.StringIter(self.process_quotes(search))
+        self.text = self.process_quotes(search)
+        i = _util.StringIter(self.text)
         new_pattern = self.main_group(i)
         return "".join(new_pattern)
 

{backrefs-8.2 → backrefs-8.3}/backrefs/bre.py

@@ -267,8 +267,9 @@ class Bre(_util.Immutable, Generic[AnyStr]):
         if isinstance(template, ReplaceTemplate):
             if use_format != template.use_format:
                 raise ValueError("Compiled replace cannot be a format object!")
-        elif isinstance(template, ReplaceTemplate) or (isinstance(template, (str, bytes)) and self.auto_compile):
-            return self.compile(template, (FORMAT if use_format and not isinstance(template, ReplaceTemplate) else 0))
+            return self.compile(template)
+        elif isinstance(template, (str, bytes)) and self.auto_compile:
+            return self.compile(template, (FORMAT if use_format else 0))
         elif isinstance(template, (str, bytes)) and use_format:
             # Reject an attempt to run format replace when auto-compiling
             # of template strings has been disabled and we are using a