backrefs 8.2__tar.gz → 8.2.1__tar.gz

{backrefs-8.2 → backrefs-8.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: backrefs
-Version: 8.2
+Version: 8.2.1
 Summary: A wrapper around re and regex that adds additional back references.
 Project-URL: Homepage, https://github.com/facelessuser/backrefs
 Author-email: Isaac Muse <Isaac.Muse@gmail.com>

{backrefs-8.2 → backrefs-8.2.1}/backrefs/__meta__.py

@@ -193,5 +193,5 @@ def parse_version(ver: str) -> Version:
     return Version(major, minor, micro, release, pre, post, dev)
 
 
-__version_info__ = Version(8, 2, 0, "final")
+__version_info__ = Version(8, 2, 1, "final")
 __version__ = __version_info__._get_canonical()

{backrefs-8.2 → backrefs-8.2.1}/backrefs/_bre_parse.py

@@ -6,18 +6,13 @@ Copyright (c) 2011 - 2020 Isaac Muse <isaacmuse@gmail.com>
 """
 from __future__ import annotations
 import re as _re
-import sys
+import re._parser as _parser  # type: ignore[import]
 import copyreg as _copyreg
 from . import util as _util
 from .util import PatternError
 from . import uniprops as _uniprops
 from typing import Generic, AnyStr, Match, Any, Pattern, cast
 
-if sys.version_info >= (3, 11):
-    import re._parser as _parser  # type: ignore[import]
-else:
-    import sre_parse as _parser
-
 __all__ = ("ReplaceTemplate", "PatternError")
 
 _ASCII_LETTERS = frozenset(
@@ -76,11 +71,12 @@ class _SearchParser(Generic[AnyStr]):
     _re_start_wb = r"\b(?=\w)"
     _re_end_wb = r"\b(?<=\w)"
     _line_break = r'(?:\r\n|(?!\r\n)[\n\v\f\r\x85\u2028\u2029])'
-    _bytes_line_break = r'(?>\r\n|[\n\v\f\r\x85])' if _util.PY311 else r'(?:\r\n|(?!\r\n)[\n\v\f\r\x85])'
+    _bytes_line_break = r'(?>\r\n|[\n\v\f\r\x85])'
     _grapheme_cluster = r'(?:{}{}*(?!{}))'
 
     verbose: bool
     unicode: bool
+    named_unicode: tuple[str, int]
     ascii: bool  # noqa: A003
     is_bytes: bool
     search: AnyStr
@@ -283,6 +279,7 @@ class _SearchParser(Generic[AnyStr]):
             current.extend(self.unicode_name(text, in_group))
             if in_group:
                 self.found_named_unicode = True
+                self.named_unicode = (f'\\N{{{text}}}', ord(_util.unicode_char(text)))
         else:
             current.extend(["\\", t])
         return current
@@ -418,27 +415,55 @@ class _SearchParser(Generic[AnyStr]):
         self.verbose = verbose
         self.unicode = unicode_flag
 
-        if t == ")":
-            current.append(t)
+        current.append(t)
         return current
 
     def char_groups(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle character groups."""
 
-        current = []
+        current = []  # type: list[str]
         pos = i.index - 1
         found = False
         escaped = False
+        escape_index = 0
         first = 0
         found_property = False
+        # Index of an unescaped `-` in `current` that is acting as a range operator.
+        range_dash = -1
+        # Whether the last member could start a range if followed by `-`.
+        range_start = False
+        # The last member's original text, position, and code point if it is a named character
+        # beyond the byte range of a byte string.
+        start = ('', 0, None)  # type: tuple[str, int, int | None]
         self.found_property = False
         self.found_named_unicode = False
 
+        def add_member(raw: str, index: int, value: int | None) -> None:
+            """Track a member that was just added, which may start or end a range."""
+
+            nonlocal range_dash, range_start, start
+
+            if range_dash != -1:
+                # This member ends a range.
+                start_raw, start_index, start_value = start
+                if start_value is not None or value is not None:
+                    end = self.get_named_byte_range(start_raw, raw, start_value, value, start_index)
+                    if end is None:
+                        current[range_dash] = ''
+                    else:
+                        current[-1] = end
+                range_dash = -1
+                range_start = False
+            else:
+                start = (raw, index, value)
+                range_start = True
+
         try:
             while True:
                 # Prevent POSIX/Unicode class from being part of a range.
                 if self.found_property and t == '-':
                     current.append(_re.escape(t))
+                    add_member(t, i.index - 1, None)
                     pos += 1
                     t = next(i)
                     self.found_property = False
@@ -448,15 +473,26 @@ class _SearchParser(Generic[AnyStr]):
 
                 if not escaped and t == "\\":
                     escaped = True
+                    escape_index = i.index - 1
                 elif escaped:
                     escaped = False
                     idx = len(current) - 1
                     current.extend(self.reference(t, i, True))
                     if self.found_property:
                         # Prevent Unicode class from being part of a range.
-                        if idx >= 0 and current[idx] == '-':
+                        if idx == range_dash:
                             current[idx] = _re.escape('-')
                         found_property = True
+                        range_dash = -1
+                        range_start = False
+                    elif t == 'N':
+                        raw, code = self.named_unicode
+                        add_member(raw, escape_index, code if self.is_bytes and code > 0xFF else None)
+                    else:
+                        digits = self.get_escape_digits(i, t)
+                        if digits:
+                            current.append(digits)
+                        add_member(f'\\{t}{digits}', escape_index, None)
                 elif t == "[" and not found:
                     found = True
                     first = pos
@@ -470,22 +506,43 @@ class _SearchParser(Generic[AnyStr]):
                         i.rewind(i.index - index)
                     if prop is not None:
                         value = self.unicode_props(prop[0], prop[1], in_group=True)
-                        if current[-1] == '-':
+                        # Prevent POSIX class from being part of a range.
+                        if len(current) - 1 == range_dash:
                             current[-1] = _re.escape('-')
                         current.extend(value)
                         found_property = True
+                        self.found_property = True
                         pos = i.index - 2
+                        range_dash = -1
+                        range_start = False
                     else:
                         current.append(t)
-                elif t == "^" and found and (pos == first + 1):
+                        add_member(t, index - 1, None)
+                elif t == "^" and found and (pos == first + 1) and len(current) == 1:
+                    # Only a `^` directly after the opening `[` negates the set.
                     first = pos
                     current.append(t)
                 elif t == "]" and found and (pos != first + 1):
                     found = False
                     current.append(t)
                     break
-                else:
+                elif t == '-' and range_start:
+                    # A `-` after a member starts a range, unless it is followed by `]`.
                     current.append(t)
+                    range_dash = len(current) - 1
+                    range_start = False
+                else:
+                    if (
+                        t == '^' and
+                        (found_property or self.found_named_unicode) and
+                        ''.join(current) in ('[', '[^')
+                    ):
+                        # Everything before this was a property that resolved to nothing,
+                        # so escape `^` to prevent it from being treated as a negation.
+                        current.append(_re.escape(t))
+                    else:
+                        current.append(t)
+                    add_member(t, i.index - 1, None)
                 pos += 1
                 t = next(i)
         except StopIteration as e:
@@ -540,6 +597,60 @@ class _SearchParser(Generic[AnyStr]):
                 return ['']
         return [f'\\{value:03o}' if value <= 0xFF else chr(value)]
 
+    def get_escape_digits(self, i: _util.StringIter, e: str) -> str:
+        r"""
+        Get the digits that are part of an escape in a character set (`\x`, `\u`, `\U`, or octal).
+
+        This allows an escape to be treated as a single member of the character set.
+        """
+
+        if e == 'x':
+            count, valid = 2, _HEX
+        elif e == 'u' and not self.is_bytes:
+            count, valid = 4, _HEX
+        elif e == 'U' and not self.is_bytes:
+            count, valid = 8, _HEX
+        elif e in _OCTAL:
+            count, valid = 2, _OCTAL
+        else:
+            return ''
+
+        digits = []
+        while count:
+            try:
+                c = next(i)
+            except StopIteration:
+                break
+            if c.lower() not in valid:
+                i.rewind(1)
+                break
+            digits.append(c)
+            count -= 1
+        return ''.join(digits)
+
+    @staticmethod
+    def get_named_byte_range(
+        start: str,
+        end: str,
+        start_value: int | None,
+        end_value: int | None,
+        index: int
+    ) -> str | None:
+        """
+        Get the end of a range in a byte string where either end is a named character beyond the byte range.
+
+        `start_value` and `end_value` are the code points of the ends that are named characters beyond the byte
+        range, and `None` for ends within the byte range. `start` and `end` are the original text of each end,
+        and `index` is where the range starts, both used for errors.
+
+        A range that ends beyond the byte range includes everything up to the end of the byte range, so return
+        the end to use in its place. A range that starts beyond the byte range matches nothing, so return `None`.
+        """
+
+        if start_value is not None and (end_value is None or end_value < start_value):
+            raise PatternError(f'bad character range {start}-{end} at position {index}')
+        return None if start_value is not None else '\\377'
+
     def unicode_props(
         self,
         props: str,
@@ -1183,7 +1294,7 @@ class _ReplaceParser(Generic[AnyStr]):
         count = len(self.span_stack)
         self.end_found = False
         try:
-            while not self.end_found:
+            while True:
                 t = next(i)
                 if self.use_format and t in _CURLY_BRACKETS:
                     self.handle_format(t, i)
@@ -1236,10 +1347,9 @@ class _ReplaceParser(Generic[AnyStr]):
                 except StopIteration:
                     self.result.append(t)
                     raise
-            elif self.single_stack:
-                this_case = self.get_single_stack()
-                if this_case is not None:
-                    self.result.append(self.convert_case(t, this_case))
+            else:
+                # The stack always has the case just added.
+                self.result.append(self.convert_case(t, cast(int, self.get_single_stack())))
         except StopIteration:
             pass
 

{backrefs-8.2 → backrefs-8.2.1}/backrefs/_bregex_parse.py

@@ -33,8 +33,16 @@ _LETTERS_UNDERSCORE = _ASCII_LETTERS | frozenset(('_',))
 _WORD = _LETTERS_UNDERSCORE | _DIGIT
 _STANDARD_ESCAPES = frozenset(('a', 'b', 'f', 'n', 'r', 't', 'v'))
 _CURLY_BRACKETS = frozenset(('{', '}'))
-_PROPERTY_STRIP = frozenset((' ', '-', '_'))
-_PROPERTY = _WORD | _DIGIT | _PROPERTY_STRIP
+# Character set grammar, as defined by Regex.
+_ALNUM = _ASCII_LETTERS | _DIGIT
+_PROPERTY_NAME = _ALNUM | frozenset((' ', '&', '_', '-', '.'))
+_PROPERTY_VALUE = _ALNUM | frozenset((' ', '&', '_', '-', '.', '/'))
+_NAMED_CHAR = _ALNUM | frozenset((' ', '-'))
+_HEX_DIGITS = _HEX | frozenset(c.upper() for c in _HEX)
+_HEX_ESCAPES = {'x': 2, 'u': 4, 'U': 8}
+_SHORT_PROPERTIES = frozenset(('C', 'L', 'M', 'N', 'P', 'S', 'Z'))
+_CLASS_ESCAPES = frozenset(('d', 'D', 'h', 's', 'S', 'w', 'W'))
+_SET_OPS = frozenset(('||', '~~', '&&', '--'))
 _GLOBAL_FLAGS = frozenset(('b', 'e', 'p', 'r', 'V'))
 _SCOPED_FLAGS = frozenset(('a', 'f', 'i', 'L', 'm', 's', 'u', 'w', 'x'))
 _VERSIONS = frozenset(('0', '1'))
@@ -72,6 +80,7 @@ class _SearchParser(Generic[AnyStr]):
     version: int
     is_bytes: bool
     search: AnyStr
+    text: str
 
     def __init__(self, search: AnyStr, re_verbose: bool = False, re_version: int = 0) -> None:
         """Initialize."""
@@ -167,37 +176,6 @@ class _SearchParser(Generic[AnyStr]):
             current.extend(["\\", t])
         return current
 
-    def get_posix(self, i: _util.StringIter) -> str | None:
-        """Get POSIX."""
-
-        index = i.index
-        value = ['[']
-        try:
-            c = next(i)
-            if c != ':':
-                raise PatternError('Not a valid property!')
-            else:
-                value.append(c)
-                c = next(i)
-                if c == '^':
-                    value.append(c)
-                    c = next(i)
-                while c != ':':
-                    if c not in _PROPERTY:
-                        raise PatternError('Not a valid property!')
-                    if c not in _PROPERTY_STRIP:
-                        value.append(c)
-                    c = next(i)
-                value.append(c)
-                c = next(i)
-                if c != ']' or not value:
-                    raise PatternError('Unmatched ]')
-                value.append(c)
-        except Exception:
-            i.rewind(i.index - index)
-            value = []
-        return ''.join(value) if value else None
-
     def get_comments(self, i: _util.StringIter) -> str | None:
         """Get comments."""
 
@@ -342,69 +320,175 @@ class _SearchParser(Generic[AnyStr]):
             current.append(t)
         return current
 
+    @staticmethod
+    def peek(i: _util.StringIter, count: int = 1) -> str:
+        """Get up to the next `count` characters without consuming them."""
+
+        chars = []
+        for _ in range(count):
+            try:
+                chars.append(next(i))
+            except StopIteration:
+                break
+        i.rewind(len(chars))
+        return ''.join(chars)
+
+    @staticmethod
+    def match(i: _util.StringIter, text: str) -> bool:
+        """Consume `text` if it is next."""
+
+        index = i.index
+        for c in text:
+            try:
+                if next(i) == c:
+                    continue
+            except StopIteration:
+                pass
+            i.rewind(i.index - index)
+            return False
+        return True
+
+    @staticmethod
+    def consume(i: _util.StringIter, chars: frozenset[str], limit: int = -1) -> str:
+        """Consume characters while they are in `chars`, up to `limit` characters if given."""
+
+        value = []
+        while limit:
+            try:
+                c = next(i)
+            except StopIteration:
+                break
+            if c not in chars:
+                i.rewind(1)
+                break
+            value.append(c)
+            limit -= 1
+        return ''.join(value)
+
     def char_groups(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle character groups."""
 
-        current = []
-        pos = i.index - 1
-        found = 0
-        sub_first = 0
-        escaped = False
-        first = 0
-
+        start = i.index - 1
         try:
-            while True:
-                if not escaped and t == "\\":
-                    escaped = True
-                elif escaped:
-                    escaped = False
-                    current.extend(self.reference(t, i, True))
-                elif t == "[" and not found:
-                    found += 1
-                    first = pos
-                    current.append(t)
-                elif t == "[" and found and self.version == _regex.V1:
-                    # Start of sub char set found
-                    posix = None if self.is_bytes else self.get_posix(i)
-                    if posix:
-                        current.append(posix)
-                        pos = i.index - 2
-                    else:
-                        found += 1
-                        sub_first = pos
-                        current.append(t)
-                elif t == "[":
-                    posix = None if self.is_bytes else self.get_posix(i)
-                    if posix:
-                        current.append(posix)
-                        pos = i.index - 2
-                    else:
-                        current.append(t)
-                elif t == "^" and found == 1 and (pos == first + 1):
-                    # Found ^ at start of first char set; adjust 1st char position
-                    current.append(t)
-                    first = pos
-                elif self.version == _regex.V1 and t == "^" and found > 1 and (pos == sub_first + 1):
-                    # Found ^ at start of sub char set; adjust 1st char sub position
-                    current.append(t)
-                    sub_first = pos
-                elif t == "]" and found == 1 and (pos != first + 1):
-                    # First char set closed; log range
-                    current.append(t)
-                    found = 0
-                    break
-                elif self.version == _regex.V1 and t == "]" and found > 1 and (pos != sub_first + 1):
-                    # Sub char set closed; decrement depth counter
-                    found -= 1
-                    current.append(t)
-                else:
-                    current.append(t)
-                pos += 1
-                t = next(i)
+            self.parse_set(i)
         except StopIteration as e:
-            raise PatternError(f"Missing ']', unterminated character set at position {pos}") from e
+            raise PatternError(f"Missing ']', unterminated character set at position {start}") from e
+
+        return [self.text[start:i.index]]
+
+    def parse_set(self, i: _util.StringIter) -> None:
+        """Parse a character set, starting after its opening `[`, through its closing `]`."""
+
+        self.match(i, '^')
+        while True:
+            self.parse_set_operand(i)
+            if self.version == _regex.V1 and self.peek(i, 2) in _SET_OPS:
+                next(i)
+                next(i)
+                continue
+            # The operand only ends at an operator or `]`.
+            next(i)
+            return
+
+    def parse_set_operand(self, i: _util.StringIter) -> None:
+        """Parse the members of a set (or of an operand in a set operation) up to an operator or `]`."""
+
+        # The first member is always a member, even if it is `]`.
+        self.parse_set_member(i)
+        while True:
+            c = self.peek(i, 2)
+            if not c:
+                raise StopIteration
+            if c[0] == ']' or (self.version == _regex.V1 and c in _SET_OPS):
+                return
+            self.parse_set_member(i)
+
+    def parse_set_member(self, i: _util.StringIter) -> None:
+        """Parse a member of a character set, which may be a range."""
+
+        if not self.parse_set_item(i) or self.peek(i) != '-':
+            return
+
+        after = self.peek(i, 2)
+        if self.version == _regex.V1 and after == '--':
+            # A set difference, not a range.
+            return
+        next(i)
+        if after == '-]':
+            # A literal `-` at the end of the set.
+            return
+        self.parse_set_item(i)
+
+    def parse_set_item(self, i: _util.StringIter) -> bool:
+        """Parse a single item of a character set, and return whether it is a single character."""
 
-        return current
+        c = next(i)
+        if c == '\\':
+            return self.parse_set_escape(i)
+        if c == '[':
+            if self.parse_posix(i):
+                return False
+            if self.version == _regex.V1:
+                # A nested set.
+                self.parse_set(i)
+                return False
+        return True
+
+    def parse_set_escape(self, i: _util.StringIter) -> bool:
+        """Parse an escape in a character set, and return whether it is a single character."""
+
+        c = next(i)
+        if c in _HEX_ESCAPES:
+            self.consume(i, _HEX_DIGITS, _HEX_ESCAPES[c])
+        elif c == 'N':
+            # A named character, or otherwise a literal `N`.
+            index = i.index
+            if self.match(i, '{'):
+                self.consume(i, _NAMED_CHAR)
+                if self.match(i, '}'):
+                    return True
+            i.rewind(i.index - index)
+        elif c in ('p', 'P'):
+            index = i.index
+            if self.match(i, '{'):
+                self.match(i, '^')
+                self.parse_property_name(i)
+                if self.match(i, '}'):
+                    return False
+                # Not a property, so a literal `p` or `P`.
+                i.rewind(i.index - index)
+            elif self.peek(i) in _SHORT_PROPERTIES:
+                next(i)
+                return False
+        elif c in _CLASS_ESCAPES:
+            return False
+        elif c in _DIGIT:
+            self.consume(i, _OCTAL, 2)
+        return True
+
+    def parse_posix(self, i: _util.StringIter) -> bool:
+        """Parse a POSIX class after an opening `[`, or consume nothing if it isn't one."""
+
+        index = i.index
+        if self.match(i, ':'):
+            self.match(i, '^')
+            self.parse_property_name(i)
+            if self.match(i, ':]'):
+                return True
+        i.rewind(i.index - index)
+        return False
+
+    def parse_property_name(self, i: _util.StringIter) -> None:
+        """Parse a property name, which may be qualified with a value: `name`, `name=value`, or `name:value`."""
+
+        self.consume(i, _PROPERTY_NAME)
+        index = i.index
+        if self.peek(i) in (':', '='):
+            next(i)
+            if self.consume(i, _PROPERTY_VALUE).strip():
+                return
+        # No value, so the name is unqualified.
+        i.rewind(i.index - index)
 
     def normal(self, t: str, i: _util.StringIter) -> list[str]:
         """Handle normal chars."""
@@ -445,7 +529,8 @@ class _SearchParser(Generic[AnyStr]):
         self.verbose = bool(self.re_verbose)
         self.version = self.re_version if self.re_version else _regex.DEFAULT_VERSION
         new_pattern = []
-        i = _util.StringIter(self.process_quotes(search))
+        self.text = self.process_quotes(search)
+        i = _util.StringIter(self.text)
         new_pattern = self.main_group(i)
         return "".join(new_pattern)
 

{backrefs-8.2 → backrefs-8.2.1}/backrefs/bre.py

@@ -267,8 +267,9 @@ class Bre(_util.Immutable, Generic[AnyStr]):
         if isinstance(template, ReplaceTemplate):
             if use_format != template.use_format:
                 raise ValueError("Compiled replace cannot be a format object!")
-        elif isinstance(template, ReplaceTemplate) or (isinstance(template, (str, bytes)) and self.auto_compile):
-            return self.compile(template, (FORMAT if use_format and not isinstance(template, ReplaceTemplate) else 0))
+            return self.compile(template)
+        elif isinstance(template, (str, bytes)) and self.auto_compile:
+            return self.compile(template, (FORMAT if use_format else 0))
         elif isinstance(template, (str, bytes)) and use_format:
             # Reject an attempt to run format replace when auto-compiling
             # of template strings has been disabled and we are using a

{backrefs-8.2 → backrefs-8.2.1}/backrefs/bregex.py

@@ -282,8 +282,9 @@ class Bregex(_util.Immutable, Generic[AnyStr]):
         if isinstance(template, ReplaceTemplate):
             if use_format != template.use_format:
                 raise ValueError("Compiled replace cannot be a format object!")
-        elif isinstance(template, ReplaceTemplate) or (isinstance(template, (str, bytes)) and self.auto_compile):
-            return self.compile(template, (FORMAT if use_format and not isinstance(template, ReplaceTemplate) else 0))
+            return self.compile(template)
+        elif isinstance(template, (str, bytes)) and self.auto_compile:
+            return self.compile(template, (FORMAT if use_format else 0))
         elif isinstance(template, (str, bytes)) and use_format:
             # Reject an attempt to run format replace when auto-compiling
             # of template strings has been disabled and we are using a

{backrefs-8.2 → backrefs-8.2.1}/docs/src/markdown/about/changelog.md

@@ -3,6 +3,27 @@ icon: lucide/scroll-text
 ---
 # Changelog
 
+## 8.2.1
+
+-   **FIX**: Reject compiled replace templates from another search pattern in compiled search objects,
+    matching module-level substitutions and preventing incorrect capture references.
+-   **FIX**: `bre`: Fix a `-` after a POSIX class in a character set sometimes forming a range (`[[:blank:]-z]`).
+-   **FIX**: `bre`: Fix an escaped `-` before a POSIX or Unicode class in a character set being turned into a
+    range operator (`[\-\p{N}]`).
+-   **FIX**: `bre`: Fix `^` being treated as a negation when it follows properties in a character set that match
+    nothing, such as `\p{Han}` in a byte string (`[\p{Han}^a]`).
+-   **FIX**: `bre`: Fix ranges in byte string character sets with a named character beyond the byte range
+    (`[a-\N{CJK UNIFIED IDEOGRAPH-4E00}]`). They now match Regex: a range that ends beyond the byte range includes
+    everything up to `\xff`, and a range that starts beyond it matches nothing, or is an error if it is reversed.
+-   **FIX**: `bre`: Fix a literal `^` after the negating `^` in a character set (`[^^]`) causing an unterminated
+    character set error.
+-   **FIX**: `bregex`: Fix finding the end of character sets, which could cause false errors or wrongly handle what
+    follows a set (such as `\R`, groups, or verbose comments). Character sets now follow Regex's own rules:
+    -   POSIX classes are recognized in byte strings, and with qualified names or values (`[[:script=latin:]]`).
+    -   In V1, a `]` directly after a set operator is a literal (`[\w--]]`).
+    -   A literal `^` after the negating `^` (`[^^]`) no longer causes an unterminated character set error.
+-   **FIX**: `bregex`: POSIX classes in character sets are no longer rewritten before being passed to Regex.
+
 ## 8.2
 
 -   **NEW**: Drop Python 3.10.

{backrefs-8.2 → backrefs-8.2.1}/tests/test_bre.py

@@ -4,21 +4,15 @@ import unittest
 from backrefs import bre
 from backrefs.util import PatternError
 import re
+import re._constants as _constants
 import sys
 import pytest
 import random
 from backrefs import _bre_parse
 import copy
 
-PY39_PLUS = (3, 9) <= sys.version_info
-PY311_PLUS = (3, 11) <= sys.version_info
 PY313_PLUS = (3, 13) <= sys.version_info
 
-if PY311_PLUS:
-    import re._constants as _constants
-else:
-    import sre_constants as _constants
-
 
 class TestSearchTemplate(unittest.TestCase):
     """Search template tests."""
@@ -89,6 +83,40 @@ class TestSearchTemplate(unittest.TestCase):
         self.assertTrue(bre.match(r'[\p{HorizSpace}-Z]', '\x20') is not None)
         self.assertTrue(bre.match(r'[\p{HorizSpace}-Z]', '-') is not None)
 
+    def test_posix_followed_by_hyphen(self):
+        """A `-` after a POSIX class whose expansion ends in a single character should not form a range."""
+
+        for pattern in (r'[[:blank:]-z]', r'[[:space:]-~]'):
+            p = bre.compile(pattern)
+            self.assertTrue(p.fullmatch(' ') is not None)
+            self.assertTrue(p.fullmatch('-') is not None)
+            self.assertTrue(p.fullmatch(pattern[-2]) is not None)
+            self.assertTrue(p.fullmatch('a') is None)
+
+        p = bre.compile(r'[[:space:]--]')
+        self.assertTrue(p.fullmatch(' ') is not None)
+        self.assertTrue(p.fullmatch('-') is not None)
+
+    def test_escaped_hyphen_before_property(self):
+        """An escaped `-` before a POSIX or Unicode class is a literal, not a range operator."""
+
+        for pattern in (r'[\-\p{N}]', r'[\-[:digit:]]', r'[\Q-\E\p{N}]'):
+            p = bre.compile(pattern)
+            self.assertTrue(p.fullmatch('-') is not None)
+            self.assertTrue(p.fullmatch('1') is not None)
+            self.assertTrue(p.fullmatch('\\') is None)
+            self.assertTrue(p.fullmatch('A') is None)
+
+    def test_empty_property_before_caret(self):
+        """A `^` after a property that resolves to nothing must not negate the set."""
+
+        for pattern in (rb'[\p{Han}^a]', rb'[\N{CJK UNIFIED IDEOGRAPH-4E00}^a]', rb'[^\p{Han}^a]'):
+            p = bre.compile(pattern)
+            negated = pattern.startswith(b'[^')
+            self.assertEqual(p.fullmatch(b'^') is None, negated)
+            self.assertEqual(p.fullmatch(b'a') is None, negated)
+            self.assertEqual(p.fullmatch(b'b') is not None, negated)
+
     def test_non_raw_string_unicode(self):
         """Test non raw string Unicode notation."""
 
@@ -332,6 +360,65 @@ class TestSearchTemplate(unittest.TestCase):
         )
         self.assertTrue(pattern.match(b'a') is not None)
 
+    def test_byte_string_named_char_ranges(self):
+        """Test ranges in byte strings with named characters beyond the byte range, matching Regex."""
+
+        big = r'\N{CJK UNIFIED IDEOGRAPH-4E00}'
+        big2 = r'\N{CJK UNIFIED IDEOGRAPH-4E01}'
+
+        def matches(pattern):
+            p = bre.compile(pattern.encode('latin-1'))
+            return frozenset(c for c in range(256) if p.fullmatch(bytes([c])))
+
+        # A range that ends beyond the byte range includes everything up to the end of the byte range.
+        self.assertEqual(matches(f'[a-{big}]'), frozenset(range(0x61, 0x100)))
+        self.assertEqual(matches(f'[a-{big}c]'), frozenset(range(0x61, 0x100)))
+        self.assertEqual(matches(f'[^a-{big}]'), frozenset(range(0x61)))
+        self.assertEqual(matches(rf'[\N{{LATIN SMALL LETTER E WITH ACUTE}}-{big}]'), frozenset(range(0xE9, 0x100)))
+
+        # A range that starts beyond the byte range matches nothing, if it is a valid range.
+        self.assertEqual(matches(f'[{big}-{big2}]'), frozenset())
+        self.assertEqual(matches(f'[x{big}-{big2}]'), frozenset(b'x'))
+        for pattern in (f'[{big}-a]', f'[{big2}-{big}]', rf'[{big}-\x41]'):
+            with pytest.raises(PatternError, match='bad character range'):
+                bre.compile(pattern.encode('latin-1'))
+
+        # A `-` that isn't a range operator is a literal, and the named character matches nothing.
+        self.assertEqual(matches(f'[{big}-]'), frozenset(b'-'))
+        self.assertEqual(matches(f'[-{big}]'), frozenset(b'-'))
+        self.assertEqual(matches(f'[x{big}-]'), frozenset(b'-x'))
+        self.assertEqual(matches(f'[a-c-{big}]'), frozenset(b'-abc'))
+        self.assertEqual(matches(rf'[\x21-\x41-{big}]'), frozenset(range(0x21, 0x42)))
+
+    def test_escapes_in_char_groups(self):
+        """Escapes in a character set are single members, so ranges and named characters work with them."""
+
+        esc = '\\'
+        self.assertEqual([c for c in 'ABCD' if bre.fullmatch(f'[{esc}u0041-{esc}u0043]', c)], ['A', 'B', 'C'])
+        self.assertEqual([c for c in 'ABCD' if bre.fullmatch(r'[\U00000041-\U00000043]', c)], ['A', 'B', 'C'])
+
+        # An octal escape ends at the first character that isn't an octal digit.
+        self.assertEqual([c for c in ('\x01', '\x18', '8', '1') if bre.fullmatch(r'[\18]', c)], ['\x01', '8'])
+
+        # The `-` after an octal escape that ends a range is a literal.
+        p = bre.compile(rb'[\041-\101-\N{CJK UNIFIED IDEOGRAPH-4E00}]')
+        self.assertEqual(frozenset(c for c in range(256) if p.fullmatch(bytes([c]))), frozenset(range(0x21, 0x42)))
+
+        # Escapes with too few digits.
+        with pytest.raises(re.error, match='incomplete escape'):
+            bre.compile(r'[\x4]')
+        for pattern in (r'[\x4', f'[{esc}u004', r'[\10'):
+            with pytest.raises(PatternError, match='unterminated character set'):
+                bre.compile(pattern)
+
+    def test_caret_after_negation(self):
+        """A `^` after the negating `^` is a literal."""
+
+        self.assertEqual([c for c in '^]a' if bre.fullmatch(r'[^^]', c)], [']', 'a'])
+        self.assertEqual([c for c in '^]a' if bre.fullmatch(r'[^^^]', c)], [']', 'a'])
+        self.assertEqual([c for c in '^]ab-' if bre.fullmatch(r'[^^-a]', c)], [']', 'b', '-'])
+        self.assertEqual([c for c in '^]ab' if bre.fullmatch(r'[^^]]', c + ']')], [']', 'a', 'b'])
+
     def test_comments(self):
         """Test comments v0."""
 
@@ -387,6 +474,11 @@ class TestSearchTemplate(unittest.TestCase):
         pattern = bre.compile_search(r'(?x)test # \l \p{numbers}', re.UNICODE)
         self.assertEqual(pattern.pattern, r'(?x)test # \\l \\p{numbers}', re.UNICODE)
 
+        # Other escapes are left alone.
+        pattern = bre.compile_search('(?x)test # \\d comment\nok', re.UNICODE)
+        self.assertEqual(pattern.pattern, '(?x)test # \\d comment\nok')
+        self.assertTrue(pattern.fullmatch('testok') is not None)
+
     def test_char_group_nested_opening(self):
         """Test char group with nested opening [."""
 
@@ -891,7 +983,6 @@ class TestSearchTemplate(unittest.TestCase):
         m = pattern2.match('test')
         self.assertTrue(m is not None)
 
-    @unittest.skipUnless(PY39_PLUS, "Python 3.9 required")
     def test_emoji_property(self):
         """Test new emoji properties."""
 
@@ -2057,6 +2148,12 @@ class TestReplaceTemplate(unittest.TestCase):
         results = expand(pattern.match(b'Test'))
         self.assertEqual(b'\x00\x00\x00', results)
 
+        # An octal escape has at most three digits.
+        pattern = re.compile('Test')
+        for template, expected in ((r'\0000', '\x000'), (r'\0123', '\n3'), (r'\1234', 'S4')):
+            self.assertEqual(bre.compile_replace(pattern, template)(pattern.match('Test')), expected)
+            self.assertEqual(re.sub('Test', template, 'Test'), expected)
+
 
 class TestExceptions(unittest.TestCase):
     """Test Exceptions."""
@@ -2604,3 +2701,35 @@ class TestConvenienceFunctions(unittest.TestCase):
         # Fail due to `\l` being an invalid escape.
         with pytest.raises(re.error):
             p.sub(r'\ltest', 'tests')
+
+
+@pytest.mark.parametrize('method', ['sub', 'subn', 'subf', 'subfn'])
+@pytest.mark.parametrize('auto_compile', [False, True])
+def test_foreign_compiled_replacement(method, auto_compile):
+    """Reject templates whose named captures belong to another pattern."""
+
+    first = r'(?P<first>a)(?P<second>b)'
+    second = r'(?P<second>b)(?P<first>a)'
+    text = 'ba'
+    template = '{first}' if method in ('subf', 'subfn') else r'\g<first>'
+    source = bre.compile(first)
+    target = bre.compile(second, auto_compile=auto_compile)
+    replacement = source.compile(template, bre.FORMAT if method in ('subf', 'subfn') else 0)
+
+    with pytest.raises(ValueError, match="Pattern hash doesn't match"):
+        getattr(target, method)(replacement, text)
+
+
+@pytest.mark.parametrize('method', ['sub', 'subn', 'subf', 'subfn'])
+@pytest.mark.parametrize('auto_compile', [False, True])
+def test_matching_compiled_replacement(method, auto_compile):
+    """Matching templates still work without automatic string compilation."""
+
+    pattern = r'(?P<first>a)(?P<second>b)'
+    text, expected = 'ab', 'a'
+    template = '{first}' if method in ('subf', 'subfn') else r'\g<first>'
+    target = bre.compile(pattern, auto_compile=auto_compile)
+    replacement = target.compile(template, bre.FORMAT if method in ('subf', 'subfn') else 0)
+    if method in ('subn', 'subfn'):
+        expected = (expected, 1)
+    assert getattr(target, method)(replacement, text) == expected

{backrefs-8.2 → backrefs-8.2.1}/tests/test_bregex.py

@@ -9,14 +9,11 @@ import pytest
 import random
 import copy
 import time
-import sys
 try:
     import _regex_core
 except ImportError:
     from regex import _regex_core
 
-PY39_PLUS = (3, 9) <= sys.version_info
-
 
 class TestSearchTemplate(unittest.TestCase):
     """Search template tests."""
@@ -214,6 +211,11 @@ class TestSearchTemplate(unittest.TestCase):
         with pytest.raises(PatternError):
             bregex.compile_search('test[test', regex.UNICODE)
 
+        # Sets that end within an escape or POSIX class.
+        for pattern in (r'[\x4', r'[\N{LATIN', r'[[:al', r'[[:alpha:', r'[\p{L', '[\\'):
+            with pytest.raises(PatternError, match='unterminated character set'):
+                bregex.compile_search(pattern)
+
         pattern = bregex.compile_search('\\Qtest[test', regex.UNICODE)
         self.assertEqual(pattern.pattern, 'test\\[test')
 
@@ -256,6 +258,77 @@ class TestSearchTemplate(unittest.TestCase):
         pattern = bregex.compile_search(r'test [[:graph:]] \R', regex.V1)
         self.assertEqual(pattern.pattern, r'test [[:graph:]] (?>\r\n|[\n\v\f\r\x85\u2028\u2029])')
 
+    def test_char_set_end(self):
+        """Character sets should end exactly where Regex ends them."""
+
+        sets = [
+            # POSIX classes, including in byte strings and with qualified names or values.
+            '[[:digit:]]', '[^[:^alpha:]]', '[[:script=latin:]]', '[[:script:latin:]]', '[[:nv=-1/2:]]',
+            '[[:al pha:]]', '[[:a]]', '[[::]]',
+            # `]` and `^` at the start of a set.
+            '[]]', '[^]]', '[^^]', '[^^^]', '[[]]', '[[^]]]',
+            # Set operations: `]` is a literal at the start of an operand in V1.
+            '[a--]]]', '[\\w&&]]]', '[a||]]]', '[a~~]]]', '[a--[]]]]', '[[a]--[]]]]',
+            # Ranges, which consume the next item even if it looks like an operator.
+            '[!-&&]', '[\\x21-&&]', '[\\N{EXCLAMATION MARK}-&&]', '[\\d-&&]]]', '[\\p{L}-&&]]]', '[\\pL-&&]]]',
+            '[a-]', '[a--b]', '[\\p{L]', '[\\N{]', '[\\101]]', '[\\x5d]]'
+        ]
+        for version in (regex.V0, regex.V1):
+            for is_bytes in (False, True):
+                for text in sets:
+                    if is_bytes and ('\\p' in text or '\\N' in text):
+                        continue
+                    pattern = (text + ']]]').encode('latin-1') if is_bytes else text + ']]]'
+
+                    source = _regex_core.Source(pattern)
+                    source.get()
+                    try:
+                        _regex_core.parse_set(source, _regex_core.Info(version))
+                    except _regex_core.error:
+                        continue
+                    expected = pattern[:source.pos]
+
+                    parser = _bregex_parse._SearchParser(pattern, False, version)
+                    parser.version = version
+                    parser.text = text + ']]]'
+                    i = _bregex_parse._util.StringIter(parser.text)
+                    result = parser.char_groups(next(i), i)[0]
+                    self.assertEqual(
+                        result.encode('latin-1') if is_bytes else result,
+                        expected,
+                        f'{pattern!r} with {"V1" if version == regex.V1 else "V0"}'
+                    )
+
+    def test_char_set_end_symptoms(self):
+        """Finding the wrong end of a character set should not affect what follows it."""
+
+        line_break = r'(?>\r\n|[\n\v\f\r\x85\u2028\u2029])'
+        bytes_line_break = rb'(?>\r\n|[\n\v\f\r\x85])'
+        cases = [
+            (rb'[[:digit:](]', 0, rb'[[:digit:](]', [b'5', b'(', b'a']),
+            (rb'[[:digit:]#]\R', bregex.VERBOSE, rb'[[:digit:]#]' + bytes_line_break, [b'5\n', b'#\n', b'5\\R']),
+            (r'[[:script=latin:](]', 0, r'[[:script=latin:](]', ['a', '(', '1']),
+            (r'[[:script=latin:]#]\R', bregex.VERBOSE, r'[[:script=latin:]#]' + line_break, ['a\n', '#\n', 'a\\R']),
+            (r'[\w--](]', bregex.V1, r'[\w--](]', ['a', '(', ']']),
+            (r'[\W&&]#]\R', bregex.V1 | bregex.VERBOSE, r'[\W&&]#]' + line_break, [']\n', '#\n', '%\n']),
+            (r'[^^]\R', 0, r'[^^]' + line_break, ['a\n', '^\n']),
+            (r'[^^](a)', bregex.V1, r'[^^](a)', ['ba', '^a']),
+        ]
+        for pattern, flags, equivalent, tests in cases:
+            p = bregex.compile(pattern, flags)
+            for s in tests:
+                self.assertEqual(
+                    bool(p.fullmatch(s)),
+                    bool(regex.fullmatch(equivalent, s, flags=flags)),
+                    f'{pattern!r} with {s!r}'
+                )
+
+    def test_posix_not_normalized(self):
+        """POSIX classes should be passed to Regex unchanged."""
+
+        self.assertEqual(bregex.compile_search(r'[[:al_pha:]]').pattern, r'[[:al_pha:]]')
+        self.assertEqual(bregex.compile_search(r'[[:nv=-1/2:]]').pattern, r'[[:nv=-1/2:]]')
+
     def test_inline_comments(self):
         """Test that we properly find inline comments and avoid them."""
         pattern = bregex.compile_search(r'test(?#\l\p{^IsLatin})', regex.UNICODE)
@@ -587,7 +660,6 @@ class TestSearchTemplate(unittest.TestCase):
         m = pattern2.match('test')
         self.assertTrue(m is not None)
 
-    @unittest.skipUnless(PY39_PLUS, "Python 3.9 required")
     def test_emoji_property(self):
         """Test new emoji properties."""
 
@@ -2210,3 +2282,35 @@ class TestConvenienceFunctions(unittest.TestCase):
 
         with self.assertRaises(_regex_core.error):
             self.assertEqual(p.sub(r'\ltest', 'tests'), r'\ltest')
+
+
+@pytest.mark.parametrize('method', ['sub', 'subn', 'subf', 'subfn'])
+@pytest.mark.parametrize('auto_compile', [False, True])
+def test_foreign_compiled_replacement(method, auto_compile):
+    """Reject templates whose named captures belong to another pattern."""
+
+    first = r'(?P<first>a)(?P<second>b)'
+    second = r'(?P<second>b)(?P<first>a)'
+    text = 'ba'
+    template = '{first}' if method in ('subf', 'subfn') else r'\g<first>'
+    source = bregex.compile(first)
+    target = bregex.compile(second, auto_compile=auto_compile)
+    replacement = source.compile(template, bregex.FORMAT if method in ('subf', 'subfn') else 0)
+
+    with pytest.raises(ValueError, match="Pattern hash doesn't match"):
+        getattr(target, method)(replacement, text)
+
+
+@pytest.mark.parametrize('method', ['sub', 'subn', 'subf', 'subfn'])
+@pytest.mark.parametrize('auto_compile', [False, True])
+def test_matching_compiled_replacement(method, auto_compile):
+    """Matching templates still work without automatic string compilation."""
+
+    pattern = r'(?P<first>a)(?P<second>b)'
+    text, expected = 'ab', 'a'
+    template = '{first}' if method in ('subf', 'subfn') else r'\g<first>'
+    target = bregex.compile(pattern, auto_compile=auto_compile)
+    replacement = target.compile(template, bregex.FORMAT if method in ('subf', 'subfn') else 0)
+    if method in ('subn', 'subfn'):
+        expected = (expected, 1)
+    assert getattr(target, method)(replacement, text) == expected