Cython 3.2.0__cp39-abi3-win32.whl

Cython/Plex/Scanners.py

@@ -0,0 +1,328 @@
+"""
+Python Lexical Analyser
+
+Scanning an input stream
+"""
+
+import cython
+
+cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object)  # noqa:E402
+
+from . import Errors
+from .Regexps import BOL, EOL, EOF
+
+NOT_FOUND = object()
+
+
+class Scanner:
+    """
+    A Scanner is used to read tokens from a stream of characters
+    using the token set specified by a Plex.Lexicon.
+
+    Constructor:
+
+      Scanner(lexicon, stream, name = '')
+
+        See the docstring of the __init__ method for details.
+
+    Methods:
+
+      See the docstrings of the individual methods for more
+      information.
+
+      read() --> (value, text)
+        Reads the next lexical token from the stream.
+
+      position() --> (name, line, col)
+        Returns the position of the last token read using the
+        read() method.
+
+      begin(state_name)
+        Causes scanner to change state.
+
+      produce(value [, text])
+        Causes return of a token value to the caller of the
+        Scanner.
+
+    """
+
+    #  lexicon = None        # Lexicon
+    #  stream = None         # file-like object
+    #  name = ''
+    #  buffer = ''
+    #
+    #  These positions are used by the scanner to track its internal state:
+    #  buf_start_pos = 0     # position in input of start of buffer
+    #  next_pos = 0          # position in input of next char to read
+    #  cur_pos = 0           # position in input of current char
+    #  cur_line = 1          # line number of current char
+    #  cur_line_start = 0    # position in input of start of current line
+    #  start_pos = 0         # position in input of start of token
+    #  current_scanner_position_tuple = ("", 0, 0)
+    #        tuple of filename, line number and position in line, really mainly for error reporting
+    #
+    #  These positions are used to track what was read from the queue
+    #   (which may differ from the internal state when tokens are replaced onto the queue)
+    #  last_token_position_tuple = ("", 0, 0)  # tuple of filename, line number and position in line
+
+    #  text = None           # text of last token read
+    #  initial_state = None  # Node
+    #  state_name = ''       # Name of initial state
+    #  queue = None          # list of tokens and positions to be returned
+    #  trace = 0
+
+    def __init__(self, lexicon, stream, name='', initial_pos=None):
+        """
+        Scanner(lexicon, stream, name = '')
+
+          |lexicon| is a Plex.Lexicon instance specifying the lexical tokens
+          to be recognised.
+
+          |stream| can be a file object or anything which implements a
+          compatible read() method.
+
+          |name| is optional, and may be the name of the file being
+          scanned or any other identifying string.
+        """
+        self.trace = 0
+
+        self.buffer = ''
+        self.buf_start_pos = 0
+        self.next_pos = 0
+        self.cur_pos = 0
+        self.cur_line = 1
+        self.start_pos = 0
+        self.current_scanner_position_tuple = ("", 0, 0)
+        self.last_token_position_tuple = ("", 0, 0)
+        self.text = None
+        self.state_name = None
+
+        self.lexicon = lexicon
+        self.stream = stream
+        self.name = name
+        self.queue = []
+        self.initial_state = None
+        self.begin('')
+        self.next_pos = 0
+        self.cur_pos = 0
+        self.cur_line_start = 0
+        self.cur_char = BOL
+        self.input_state = 1
+        if initial_pos is not None:
+            self.cur_line, self.cur_line_start = initial_pos[1], -initial_pos[2]
+
+    def read(self):
+        """
+        Read the next lexical token from the stream and return a
+        tuple (value, text), where |value| is the value associated with
+        the token as specified by the Lexicon, and |text| is the actual
+        string read from the stream. Returns (None, '') on end of file.
+        """
+        queue = self.queue
+        while not queue:
+            self.text, action = self.scan_a_token()
+            if action is None:
+                self.produce(None)
+                self.eof()
+            else:
+                value = action.perform(self, self.text)
+                if value is not None:
+                    self.produce(value)
+        result, self.last_token_position_tuple = queue[0]
+        del queue[0]
+        return result
+
+    def unread(self, token, value, position):
+        self.queue.insert(0, ((token, value), position))
+
+    def get_current_scan_pos(self):
+        # distinct from the position of the last token due to the queue
+        return self.current_scanner_position_tuple
+
+    def scan_a_token(self):
+        """
+        Read the next input sequence recognised by the machine
+        and return (text, action). Returns ('', None) on end of
+        file.
+        """
+        self.start_pos = self.cur_pos
+        self.current_scanner_position_tuple = (
+            self.name, self.cur_line, self.cur_pos - self.cur_line_start
+        )
+        action = self.run_machine_inlined()
+        if action is not None:
+            if self.trace:
+                print("Scanner: read: Performing %s %d:%d" % (
+                    action, self.start_pos, self.cur_pos))
+            text = self.buffer[
+                self.start_pos - self.buf_start_pos:
+                self.cur_pos - self.buf_start_pos]
+            return (text, action)
+        else:
+            if self.cur_pos == self.start_pos:
+                if self.cur_char is None or self.cur_char is EOF:
+                    return ('', None)
+            raise Errors.UnrecognizedInput(self, self.state_name)
+
+    @cython.final
+    def run_machine_inlined(self):
+        """
+        Inlined version of run_machine for speed.
+        """
+        state: dict = self.initial_state
+        cur_pos: cython.Py_ssize_t = self.cur_pos
+        cur_line: cython.Py_ssize_t = self.cur_line
+        cur_line_start: cython.Py_ssize_t = self.cur_line_start
+        cur_char = self.cur_char
+        input_state: cython.long = self.input_state
+        next_pos: cython.Py_ssize_t = self.next_pos
+        data: str
+        buffer: str = self.buffer
+        buf_start_pos: cython.Py_ssize_t = self.buf_start_pos
+        buf_len: cython.Py_ssize_t = len(buffer)
+        buf_index: cython.Py_ssize_t
+        discard: cython.Py_ssize_t
+
+        b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \
+            None, 0, 0, 0, '', 0, 0
+
+        trace: cython.bint = self.trace
+        while 1:
+            if trace:
+                print("State %d, %d/%d:%s -->" % (
+                    state['number'], input_state, cur_pos, repr(cur_char)))
+
+            # Begin inlined self.save_for_backup()
+            action = state['action']
+            if action is not None:
+                b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \
+                    action, cur_pos, cur_line, cur_line_start, cur_char, input_state, next_pos
+            # End inlined self.save_for_backup()
+
+            c = cur_char
+            new_state = state.get(c, NOT_FOUND)
+            if new_state is NOT_FOUND:
+                new_state = c and state.get('else')
+
+            if new_state:
+                if trace:
+                    print("State %d" % new_state['number'])
+                state = new_state
+                # Begin inlined: self.next_char()
+                if input_state == 1:
+                    cur_pos = next_pos
+                    # Begin inlined: c = self.read_char()
+                    buf_index = next_pos - buf_start_pos
+                    if buf_index < buf_len:
+                        c = buffer[buf_index]
+                        next_pos += 1
+                    else:
+                        discard = self.start_pos - buf_start_pos
+                        data = self.stream.read(0x1000)
+                        buffer = self.buffer[discard:] + data
+                        self.buffer = buffer
+                        buf_start_pos += discard
+                        self.buf_start_pos = buf_start_pos
+                        buf_len = len(buffer)
+                        buf_index -= discard
+                        if data:
+                            c = buffer[buf_index]
+                            next_pos += 1
+                        else:
+                            c = ''
+                    # End inlined: c = self.read_char()
+                    if c == '\n':
+                        cur_char = EOL
+                        input_state = 2
+                    elif not c:
+                        cur_char = EOL
+                        input_state = 4
+                    else:
+                        cur_char = c
+                elif input_state == 2:  # after EoL (1) -> BoL (3)
+                    cur_char = '\n'
+                    input_state = 3
+                elif input_state == 3:  # start new code line
+                    cur_line += 1
+                    cur_line_start = cur_pos = next_pos
+                    cur_char = BOL
+                    input_state = 1
+                elif input_state == 4:  # after final line (1) -> EoF (5)
+                    cur_char = EOF
+                    input_state = 5
+                else:  # input_state == 5  (EoF)
+                    cur_char = ''
+                    # End inlined self.next_char()
+            else:  # not new_state
+                if trace:
+                    print("blocked")
+                # Begin inlined: action = self.back_up()
+                if b_action is not None:
+                    (action, cur_pos, cur_line, cur_line_start,
+                     cur_char, input_state, next_pos) = \
+                        (b_action, b_cur_pos, b_cur_line, b_cur_line_start,
+                         b_cur_char, b_input_state, b_next_pos)
+                else:
+                    action = None
+                break  # while 1
+                # End inlined: action = self.back_up()
+
+        self.cur_pos = cur_pos
+        self.cur_line = cur_line
+        self.cur_line_start = cur_line_start
+        self.cur_char = cur_char
+        self.input_state = input_state
+        self.next_pos = next_pos
+        if trace:
+            if action is not None:
+                print("Doing %s" % action)
+        return action
+
+    def position(self) -> tuple:
+        """
+        Return a tuple (name, line, col) representing the location of
+        the last token read using the read() method. |name| is the
+        name that was provided to the Scanner constructor; |line|
+        is the line number in the stream (1-based); |col| is the
+        position within the line of the first character of the token
+        (0-based).
+        """
+        return self.last_token_position_tuple
+
+    def get_position(self):
+        """
+        Python accessible wrapper around position(), only for error reporting.
+        """
+        return self.position()
+
+    def begin(self, state_name):
+        """Set the current state of the scanner to the named state."""
+        self.initial_state = (
+            self.lexicon.get_initial_state(state_name))
+        self.state_name = state_name
+
+    def produce(self, value, text=None):
+        """
+        Called from an action procedure, causes |value| to be returned
+        as the token value from read(). If |text| is supplied, it is
+        returned in place of the scanned text.
+
+        produce() can be called more than once during a single call to an action
+        procedure, in which case the tokens are queued up and returned one
+        at a time by subsequent calls to read(), until the queue is empty,
+        whereupon scanning resumes.
+        """
+        if text is None:
+            text = self.text
+        self.queue.append(((value, text), self.current_scanner_position_tuple))
+
+    def eof(self):
+        """
+        Override this method if you want something to be done at
+        end of file.
+        """
+        pass
+
+    @property
+    def start_line(self):
+        return self.last_token_position_tuple[1]

Cython/Plex/Transitions.pxd

@@ -0,0 +1,14 @@
+cimport cython
+
+cdef long maxint
+
+@cython.final
+cdef class TransitionMap:
+    cdef list map
+    cdef dict special
+
+    cpdef add(self, event, new_state)
+    cpdef add_set(self, event, new_set)
+    cpdef iteritems(self)
+    cdef Py_ssize_t split(self, long code)
+    cdef set get_special(self, event)

Cython/Plex/Transitions.py

@@ -0,0 +1,239 @@
+"""
+Plex - Transition Maps
+
+This version represents state sets directly as dicts for speed.
+"""
+import cython
+
+maxint = 2**31-1  # sentinel value
+
+
+class TransitionMap:
+    """
+    A TransitionMap maps an input event to a set of states.
+    An input event is one of: a range of character codes,
+    the empty string (representing an epsilon move), or one
+    of the special symbols BOL, EOL, EOF.
+
+    For characters, this implementation compactly represents
+    the map by means of a list:
+
+      [code_0, states_0, code_1, states_1, code_2, states_2,
+        ..., code_n-1, states_n-1, code_n]
+
+    where |code_i| is a character code, and |states_i| is a
+    set of states corresponding to characters with codes |c|
+    in the range |code_i| <= |c| <= |code_i+1|.
+
+    The following invariants hold:
+      n >= 1
+      code_0 == -maxint
+      code_n == maxint
+      code_i < code_i+1 for i in 0..n-1
+      states_0 == states_n-1
+
+    Mappings for the special events '', BOL, EOL, EOF are
+    kept separately in a dictionary.
+    """
+
+    def __init__(self, map=None, special=None):
+        if not map:
+            map = [-maxint, set(), maxint]
+        if not special:
+            special = {}
+        self.map = map          # The list of codes and states
+        self.special = special  # Mapping for special events
+
+    def add(self, event, new_state):
+        """
+        Add transition to |new_state| on |event|.
+        """
+        i: cython.Py_ssize_t
+        j: cython.Py_ssize_t
+        if type(event) is tuple:
+            code0, code1 = event
+            i = self.split(code0)
+            j = self.split(code1)
+            map = self.map
+            while i < j:
+                map[i + 1].add(new_state)
+                i += 2
+        else:
+            self.get_special(event).add(new_state)
+
+    def add_set(self, event, new_set):
+        """
+        Add transitions to the states in |new_set| on |event|.
+        """
+        i: cython.Py_ssize_t
+        j: cython.Py_ssize_t
+        if type(event) is tuple:
+            code0, code1 = event
+            i = self.split(code0)
+            j = self.split(code1)
+            map = self.map
+            while i < j:
+                map[i + 1].update(new_set)
+                i += 2
+        else:
+            self.get_special(event).update(new_set)
+
+    def get_epsilon(self):
+        """
+        Return the mapping for epsilon, or None.
+        """
+        return self.special.get('')
+
+    def iteritems(self):
+        """
+        Return the mapping as an iterable of ((code1, code2), state_set) and
+        (special_event, state_set) pairs.
+        """
+        result = []
+        map = self.map
+        else_set: cython.bint = map[1]
+        i: cython.Py_ssize_t = 0
+        n: cython.Py_ssize_t = len(map) - 1
+        code0 = map[0]
+        while i < n:
+            state_set = map[i + 1]
+            code1 = map[i + 2]
+            if state_set or else_set:
+                result.append(((code0, code1), state_set))
+            code0 = code1
+            i += 2
+        for event, state_set in self.special.items():
+            if state_set:
+                result.append((event, state_set))
+        return iter(result)
+
+    items = iteritems
+
+    # ------------------- Private methods --------------------
+
+    def split(self, code: cython.long):
+        """
+        Search the list for the position of the split point for |code|,
+        inserting a new split point if necessary. Returns index |i| such
+        that |code| == |map[i]|.
+        """
+        # We use a funky variation on binary search.
+        map = self.map
+        hi: cython.Py_ssize_t = len(map) - 1
+        # Special case: code == map[-1]
+        if code == maxint:
+            return hi
+
+        # General case
+        lo: cython.Py_ssize_t = 0
+        mid: cython.Py_ssize_t
+        # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2
+        while hi - lo >= 4:
+            # Find midpoint truncated to even index
+            mid = ((lo + hi) // 2) & ~1
+            if code < map[mid]:
+                hi = mid
+            else:
+                lo = mid
+        # map[lo] <= code < map[hi] and hi - lo == 2
+        if map[lo] == code:
+            return lo
+        else:
+            map[hi:hi] = [code, map[hi - 1].copy()]
+            return hi
+
+    def get_special(self, event) -> set:
+        """
+        Get state set for special event, adding a new entry if necessary.
+        """
+        special = self.special
+        state_set = special.get(event)
+        if state_set is None:
+            state_set = set()
+            special[event] = state_set
+        return state_set
+
+    # --------------------- Conversion methods -----------------------
+
+    def __str__(self):
+        map_strs = []
+        map = self.map
+        n: cython.Py_ssize_t = len(map)
+        i: cython.Py_ssize_t = 0
+        while i < n:
+            code = map[i]
+            if code == -maxint:
+                code_str = "-inf"
+            elif code == maxint:
+                code_str = "inf"
+            else:
+                code_str = str(code)
+            map_strs.append(code_str)
+            i += 1
+            if i < n:
+                map_strs.append(state_set_str(map[i]))
+            i += 1
+        special_strs = {}
+        for event, set in self.special.items():
+            special_strs[event] = state_set_str(set)
+        return "[%s]+%s" % (
+            ','.join(map_strs),
+            special_strs
+        )
+
+    # --------------------- Debugging methods -----------------------
+
+    def check(self):
+        """Check data structure integrity."""
+        if not self.map[-3] < self.map[-1]:
+            print(self)
+            assert 0
+
+    def dump(self, file):
+        map = self.map
+        i: cython.Py_ssize_t = 0
+        n: cython.Py_ssize_t = len(map) - 1
+        while i < n:
+            self.dump_range(map[i], map[i + 2], map[i + 1], file)
+            i += 2
+        for event, set in self.special.items():
+            if set:
+                if not event:
+                    event = 'empty'
+                self.dump_trans(event, set, file)
+
+    def dump_range(self, code0, code1, set, file):
+        if set:
+            if code0 == -maxint:
+                if code1 == maxint:
+                    k = "any"
+                else:
+                    k = "< %s" % self.dump_char(code1)
+            elif code1 == maxint:
+                k = "> %s" % self.dump_char(code0 - 1)
+            elif code0 == code1 - 1:
+                k = self.dump_char(code0)
+            else:
+                k = "%s..%s" % (self.dump_char(code0),
+                                self.dump_char(code1 - 1))
+            self.dump_trans(k, set, file)
+
+    def dump_char(self, code):
+        if 0 <= code <= 255:
+            return repr(chr(code))
+        else:
+            return "chr(%d)" % code
+
+    def dump_trans(self, key, set, file):
+        file.write("      %s --> %s\n" % (key, self.dump_set(set)))
+
+    def dump_set(self, set):
+        return state_set_str(set)
+
+
+#
+#   State set manipulation functions
+#
+
+def state_set_str(set):
+    return "[%s]" % ','.join(["S%d" % state.number for state in set])

Cython/Plex/__init__.py

@@ -0,0 +1,34 @@
+"""
+Python Lexical Analyser
+
+The Plex module provides lexical analysers with similar capabilities
+to GNU Flex. The following classes and functions are exported;
+see the attached docstrings for more information.
+
+   Scanner          For scanning a character stream under the
+                    direction of a Lexicon.
+
+   Lexicon          For constructing a lexical definition
+                    to be used by a Scanner.
+
+   Str, Any, AnyBut, AnyChar, Seq, Alt, Opt, Rep, Rep1,
+   Bol, Eol, Eof, Empty
+
+                    Regular expression constructors, for building pattern
+                    definitions for a Lexicon.
+
+   State            For defining scanner states when creating a
+                    Lexicon.
+
+   TEXT, IGNORE, Begin
+
+                    Actions for associating with patterns when
+        creating a Lexicon.
+"""
+# flake8: noqa:F401
+
+from .Actions import TEXT, IGNORE, Begin, Method
+from .Lexicons import Lexicon, State
+from .Regexps import RE, Seq, Alt, Rep1, Empty, Str, Any, AnyBut, AnyChar, Range
+from .Regexps import Opt, Rep, Bol, Eol, Eof, Case, NoCase
+from .Scanners import Scanner

Cython/Runtime/__init__.py

@@ -0,0 +1 @@
+# empty file