Cython 3.2.0__cp39-abi3-win32.whl

Cython/Plex/DFA.py

@@ -0,0 +1,164 @@
+# cython: auto_cpdef=True
+"""
+Python Lexical Analyser
+
+Converting NFA to DFA
+"""
+
+import cython
+from . import Machines
+from .Machines import LOWEST_PRIORITY
+from .Transitions import TransitionMap
+
+if cython.compiled:
+    from cython.cimports.Cython.Plex.Machines import Node, FastMachine
+    from cython.cimports.Cython.Plex.Transitions import TransitionMap as type_TransitionMap
+else:
+    from Cython.Plex.Machines import Node, FastMachine
+    from Cython.Plex.Transitions import TransitionMap as type_TransitionMap
+
+
+def nfa_to_dfa(old_machine, debug=None):
+    """
+    Given a nondeterministic Machine, return a new equivalent
+    Machine which is deterministic.
+    """
+    # We build a new machine whose states correspond to sets of states
+    # in the old machine. Initially we add a new state corresponding to
+    # the epsilon-closure of each initial old state. Then we give transitions
+    # to each new state which are the union of all transitions out of any
+    # of the corresponding old states. The new state reached on a given
+    # character is the one corresponding to the set of states reachable
+    # on that character from any of the old states. As new combinations of
+    # old states are created, new states are added as needed until closure
+    # is reached.
+    transitions: type_TransitionMap
+    new_machine: FastMachine = Machines.FastMachine()
+    state_map: StateMap = StateMap(new_machine)
+
+    # Seed the process using the initial states of the old machine.
+    # Make the corresponding new states into initial states of the new
+    # machine with the same names.
+    for (key, old_state) in old_machine.initial_states.items():
+        new_state = state_map.old_to_new(epsilon_closure(old_state))
+        new_machine.make_initial_state(key, new_state)
+
+    # Tricky bit here: we add things to the end of this list while we're
+    # iterating over it. The iteration stops when closure is achieved.
+    for new_state in new_machine.states:
+        transitions = TransitionMap()
+        for old_state in state_map.new_to_old(new_state):
+            for event, old_target_states in old_state.transitions.items():
+                if event and old_target_states:
+                    transitions.add_set(event, set_epsilon_closure(old_target_states))
+        for event, old_states in transitions.items():
+            new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states))
+
+    if debug:
+        debug.write("\n===== State Mapping =====\n")
+        state_map.dump(debug)
+    return new_machine
+
+
+@cython.cfunc
+def set_epsilon_closure(state_set: set) -> set:
+    """
+    Given a set of states, return the union of the epsilon
+    closures of its member states.
+    """
+    result = set()
+    for state1 in state_set:
+        for state2 in epsilon_closure(state1):
+            result.add(state2)
+    return result
+
+
+@cython.cfunc
+def epsilon_closure(state: Node) -> set:
+    """
+    Return the set of states reachable from the given state
+    by epsilon moves.
+    """
+    # Cache the result
+    result = state.epsilon_closure
+    if result is None:
+        result = set()
+        state.epsilon_closure = result
+        add_to_epsilon_closure(result, state)
+    return result
+
+
+@cython.cfunc
+def add_to_epsilon_closure(state_set: set, state: Node):
+    """
+    Recursively add to |state_set| states reachable from the given state
+    by epsilon moves.
+    """
+    state_set_2: set
+    state2: Node
+
+    if state not in state_set:
+        state_set.add(state)
+        state_set_2 = state.transitions.get_epsilon()
+        if state_set_2:
+            for state2 in state_set_2:
+                add_to_epsilon_closure(state_set, state2)
+
+
+class StateMap:
+    """
+    Helper class used by nfa_to_dfa() to map back and forth between
+    sets of states from the old machine and states of the new machine.
+    """
+
+    def __init__(self, new_machine):
+        self.new_machine = new_machine  # Machine
+        self.old_to_new_dict = {}  # {(old_state,...) : new_state}
+        self.new_to_old_dict = {}  # {id(new_state) : old_state_set}
+
+    def old_to_new(self, old_state_set: set):
+        """
+        Return the state of the new machine corresponding to the
+        set of old machine states represented by |state_set|. A new
+        state will be created if necessary. If any of the old states
+        are accepting states, the new state will be an accepting state
+        with the highest priority action from the old states.
+        """
+        key = self.make_key(old_state_set)
+        new_state = self.old_to_new_dict.get(key, None)
+        if not new_state:
+            action = self.highest_priority_action(old_state_set)
+            new_state = self.new_machine.new_state(action)
+            self.old_to_new_dict[key] = new_state
+            self.new_to_old_dict[id(new_state)] = old_state_set
+        return new_state
+
+    def highest_priority_action(self, state_set: set):
+        best_action = None
+        best_priority = LOWEST_PRIORITY
+        state: Node
+        for state in state_set:
+            priority = state.action_priority
+            if priority > best_priority:
+                best_action = state.action
+                best_priority = priority
+        return best_action
+
+    def new_to_old(self, new_state):
+        """Given a new state, return a set of corresponding old states."""
+        return self.new_to_old_dict[id(new_state)]
+
+    def make_key(self, state_set: set):
+        """
+        Convert a set of states into a uniquified
+        sorted tuple suitable for use as a dictionary key.
+        """
+        return tuple(sorted(state_set))
+
+    def dump(self, file):
+        from .Transitions import state_set_str
+
+        for new_state in self.new_machine.states:
+            old_state_set = self.new_to_old_dict[id(new_state)]
+            file.write("   State %s <-- %s\n" % (
+                new_state['number'], state_set_str(old_state_set)))

Cython/Plex/Errors.py

@@ -0,0 +1,48 @@
+"""
+Python Lexical Analyser
+
+Exception classes
+"""
+
+
+class PlexError(Exception):
+    message = ""
+
+
+class PlexTypeError(PlexError, TypeError):
+    pass
+
+
+class PlexValueError(PlexError, ValueError):
+    pass
+
+
+class InvalidToken(PlexError):
+    def __init__(self, token_number, message):
+        PlexError.__init__(self, "Token number %d: %s" % (token_number, message))
+
+
+class InvalidScanner(PlexError):
+    pass
+
+
+class AmbiguousAction(PlexError):
+    message = "Two tokens with different actions can match the same string"
+
+    def __init__(self):
+        pass
+
+
+class UnrecognizedInput(PlexError):
+    scanner = None
+    position = None
+    state_name = None
+
+    def __init__(self, scanner, state_name):
+        self.scanner = scanner
+        self.position = scanner.get_position()
+        self.state_name = state_name
+
+    def __str__(self):
+        return ("'%s', line %d, char %d: Token not recognised in state %r" % (
+            self.position + (self.state_name,)))

Cython/Plex/Lexicons.py

@@ -0,0 +1,178 @@
+"""
+Python Lexical Analyser
+
+Lexical Analyser Specification
+"""
+
+from . import Actions
+from . import DFA
+from . import Errors
+from . import Machines
+from . import Regexps
+
+# debug_flags for Lexicon constructor
+DUMP_NFA = 1
+DUMP_DFA = 2
+
+
+class State:
+    """
+    This class is used as part of a Plex.Lexicon specification to
+    introduce a user-defined state.
+
+    Constructor:
+
+       State(name, token_specifications)
+    """
+
+    name = None
+    tokens = None
+
+    def __init__(self, name, tokens):
+        self.name = name
+        self.tokens = tokens
+
+
+class Lexicon:
+    """
+    Lexicon(specification) builds a lexical analyser from the given
+    |specification|. The specification consists of a list of
+    specification items. Each specification item may be either:
+
+       1) A token definition, which is a tuple:
+
+             (pattern, action)
+
+          The |pattern| is a regular axpression built using the
+          constructors defined in the Plex module.
+
+          The |action| is the action to be performed when this pattern
+          is recognised (see below).
+
+       2) A state definition:
+
+             State(name, tokens)
+
+          where |name| is a character string naming the state,
+          and |tokens| is a list of token definitions as
+          above. The meaning and usage of states is described
+          below.
+
+    Actions
+    -------
+
+    The |action| in a token specification may be one of three things:
+
+       1) A function, which is called as follows:
+
+             function(scanner, text)
+
+          where |scanner| is the relevant Scanner instance, and |text|
+          is the matched text. If the function returns anything
+          other than None, that value is returned as the value of the
+          token. If it returns None, scanning continues as if the IGNORE
+          action were specified (see below).
+
+        2) One of the following special actions:
+
+           IGNORE means that the recognised characters will be treated as
+                  white space and ignored. Scanning will continue until
+                  the next non-ignored token is recognised before returning.
+
+           TEXT   causes the scanned text itself to be returned as the
+                  value of the token.
+
+        3) Any other value, which is returned as the value of the token.
+
+    States
+    ------
+
+    At any given time, the scanner is in one of a number of states.
+    Associated with each state is a set of possible tokens. When scanning,
+    only tokens associated with the current state are recognised.
+
+    There is a default state, whose name is the empty string. Token
+    definitions which are not inside any State definition belong to
+    the default state.
+
+    The initial state of the scanner is the default state. The state can
+    be changed in one of two ways:
+
+       1) Using Begin(state_name) as the action of a token.
+
+       2) Calling the begin(state_name) method of the Scanner.
+
+    To change back to the default state, use '' as the state name.
+    """
+
+    machine = None  # Machine
+    tables = None   # StateTableMachine
+
+    def __init__(self, specifications, debug=None, debug_flags=7):
+        if not isinstance(specifications, list):
+            raise Errors.InvalidScanner("Scanner definition is not a list")
+
+        nfa = Machines.Machine()
+        default_initial_state = nfa.new_initial_state('')
+        token_number = 1
+
+        for spec in specifications:
+            if isinstance(spec, State):
+                user_initial_state = nfa.new_initial_state(spec.name)
+                for token in spec.tokens:
+                    self.add_token_to_machine(
+                        nfa, user_initial_state, token, token_number)
+                    token_number += 1
+            elif isinstance(spec, tuple):
+                self.add_token_to_machine(
+                    nfa, default_initial_state, spec, token_number)
+                token_number += 1
+            else:
+                raise Errors.InvalidToken(
+                    token_number,
+                    "Expected a token definition (tuple) or State instance")
+
+        if debug and (debug_flags & 1):
+            debug.write("\n============= NFA ===========\n")
+            nfa.dump(debug)
+
+        dfa = DFA.nfa_to_dfa(nfa, debug=(debug_flags & 3) == 3 and debug)
+
+        if debug and (debug_flags & 2):
+            debug.write("\n============= DFA ===========\n")
+            dfa.dump(debug)
+
+        self.machine = dfa
+
+    def add_token_to_machine(self, machine, initial_state, token_spec, token_number):
+        try:
+            (re, action_spec) = self.parse_token_definition(token_spec)
+            if isinstance(action_spec, Actions.Action):
+                action = action_spec
+            else:
+                try:
+                    action_spec.__call__
+                except AttributeError:
+                    action = Actions.Return(action_spec)
+                else:
+                    action = Actions.Call(action_spec)
+            final_state = machine.new_state()
+            re.build_machine(machine, initial_state, final_state,
+                             match_bol=1, nocase=0)
+            final_state.set_action(action, priority=-token_number)
+        except Errors.PlexError as e:
+            raise e.__class__("Token number %d: %s" % (token_number, e))
+
+    def parse_token_definition(self, token_spec):
+        if not isinstance(token_spec, tuple):
+            raise Errors.InvalidToken("Token definition is not a tuple")
+        if len(token_spec) != 2:
+            raise Errors.InvalidToken("Wrong number of items in token definition")
+
+        pattern, action = token_spec
+        if not isinstance(pattern, Regexps.RE):
+            raise Errors.InvalidToken("Pattern is not an RE instance")
+        return (pattern, action)
+
+    def get_initial_state(self, name):
+        return self.machine.get_initial_state(name)

Cython/Plex/Machines.pxd

@@ -0,0 +1,36 @@
+cimport cython
+
+from .Actions cimport Action
+from .Transitions cimport TransitionMap
+
+cdef int maxint
+
+
+@cython.final
+cdef class Machine:
+    cdef readonly list states
+    cdef readonly dict  initial_states
+    cdef readonly Py_ssize_t next_state_number
+
+    cpdef new_state(self)
+    cpdef new_initial_state(self, name)
+    cpdef make_initial_state(self, name, state)
+
+
+@cython.final
+cdef class Node:
+    cdef readonly TransitionMap transitions
+    cdef readonly Action action
+    cdef public set epsilon_closure
+    cdef readonly Py_ssize_t number
+    cdef readonly int action_priority
+
+
+@cython.final
+cdef class FastMachine:
+    cdef readonly dict initial_states
+    cdef readonly dict new_state_template
+    cdef readonly list states
+    cdef readonly Py_ssize_t next_number
+
+    cpdef make_initial_state(self, name, state)

Cython/Plex/Machines.py

@@ -0,0 +1,238 @@
+"""
+Python Lexical Analyser
+
+Classes for building NFAs and DFAs
+"""
+
+import cython
+from .Transitions import TransitionMap
+
+maxint = 2**31-1  # sentinel value
+
+LOWEST_PRIORITY = -maxint
+
+
+class Machine:
+    """A collection of Nodes representing an NFA or DFA."""
+    def __init__(self):
+        self.states = []  # [Node]
+        self.initial_states = {}  # {(name, bol): Node}
+        self.next_state_number = 1
+
+    def __del__(self):
+        for state in self.states:
+            state.destroy()
+
+    def new_state(self):
+        """Add a new state to the machine and return it."""
+        s = Node()
+        n: cython.Py_ssize_t = self.next_state_number
+        self.next_state_number = n + 1
+        s.number = n
+        self.states.append(s)
+        return s
+
+    def new_initial_state(self, name):
+        state = self.new_state()
+        self.make_initial_state(name, state)
+        return state
+
+    def make_initial_state(self, name, state):
+        self.initial_states[name] = state
+
+    def get_initial_state(self, name):
+        return self.initial_states[name]
+
+    def dump(self, file):
+        file.write("Plex.Machine:\n")
+        if self.initial_states is not None:
+            file.write("   Initial states:\n")
+            for (name, state) in sorted(self.initial_states.items()):
+                file.write("      '%s': %d\n" % (name, state.number))
+        for s in self.states:
+            s.dump(file)
+
+
+class Node:
+    """A state of an NFA or DFA."""
+
+    def __init__(self):
+        # Preinitialise the list of empty transitions, because
+        # the nfa-to-dfa algorithm needs it
+        self.transitions = TransitionMap()      # TransitionMap
+        self.action_priority = LOWEST_PRIORITY  # integer
+        self.action = None  # Action
+        self.number = 0     # for debug output
+        self.epsilon_closure = None  # used by nfa_to_dfa()
+
+    def destroy(self):
+        self.transitions = None
+        self.action = None
+        self.epsilon_closure = None
+
+    def add_transition(self, event, new_state):
+        self.transitions.add(event, new_state)
+
+    def link_to(self, state):
+        """Add an epsilon-move from this state to another state."""
+        self.add_transition('', state)
+
+    def set_action(self, action, priority):
+        """Make this an accepting state with the given action. If
+        there is already an action, choose the action with highest
+        priority."""
+        if priority > self.action_priority:
+            self.action = action
+            self.action_priority = priority
+
+    def get_action(self):
+        return self.action
+
+    def get_action_priority(self):
+        return self.action_priority
+
+    def is_accepting(self):
+        return self.action is not None
+
+    def __str__(self):
+        return "State %d" % self.number
+
+    def dump(self, file):
+        # Header
+        file.write("   State %d:\n" % self.number)
+        # Transitions
+        #        self.dump_transitions(file)
+        self.transitions.dump(file)
+        # Action
+        action = self.action
+        priority = self.action_priority
+        if action is not None:
+            file.write("      %s [priority %d]\n" % (action, priority))
+
+    def __lt__(self, other):
+        return self.number < other.number
+
+    def __hash__(self):
+        # Prevent overflowing hash values due to arbitrarily large unsigned addresses.
+        return id(self) & maxint
+
+
+class FastMachine:
+    """
+    FastMachine is a deterministic machine represented in a way that
+    allows fast scanning.
+    """
+    def __init__(self):
+        self.initial_states = {}  # {state_name:state}
+        self.states = []          # [state]  where state = {event:state, 'else':state, 'action':Action}
+        self.next_number = 1      # for debugging
+        self.new_state_template = {
+            '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None
+        }
+
+    def __del__(self):
+        for state in self.states:
+            state.clear()
+
+    def new_state(self, action=None):
+        number: cython.Py_ssize_t = self.next_number
+        self.next_number = number + 1
+        result = self.new_state_template.copy()
+        result['number'] = number
+        result['action'] = action
+        self.states.append(result)
+        return result
+
+    def make_initial_state(self, name, state):
+        self.initial_states[name] = state
+
+    def add_transitions(self, state: dict, event, new_state, maxint: cython.int = maxint):
+        code:  cython.int
+        code0: cython.int
+        code1: cython.int
+
+        if type(event) is tuple:
+            code0, code1 = event
+            if code0 == -maxint:
+                state['else'] = new_state
+            elif code1 != maxint:
+                for code in range(code0, code1):
+                    state[chr(code)] = new_state
+        else:
+            state[event] = new_state
+
+    def get_initial_state(self, name):
+        return self.initial_states[name]
+
+    def dump(self, file):
+        file.write("Plex.FastMachine:\n")
+        file.write("   Initial states:\n")
+        for name, state in sorted(self.initial_states.items()):
+            file.write("      %s: %s\n" % (repr(name), state['number']))
+        for state in self.states:
+            self.dump_state(state, file)
+
+    def dump_state(self, state, file):
+        # Header
+        file.write("   State %d:\n" % state['number'])
+        # Transitions
+        self.dump_transitions(state, file)
+        # Action
+        action = state['action']
+        if action is not None:
+            file.write("      %s\n" % action)
+
+    def dump_transitions(self, state, file):
+        chars_leading_to_state = {}
+        special_to_state = {}
+        for (c, s) in state.items():
+            if len(c) == 1:
+                chars = chars_leading_to_state.get(id(s))
+                if chars is None:
+                    chars = []
+                    chars_leading_to_state[id(s)] = chars
+                chars.append(c)
+            elif len(c) <= 4:
+                special_to_state[c] = s
+        ranges_to_state = {}
+        for state in self.states:
+            char_list = chars_leading_to_state.get(id(state))
+            if char_list:
+                ranges = self.chars_to_ranges(char_list)
+                ranges_to_state[ranges] = state
+        for ranges in sorted(ranges_to_state):
+            key = self.ranges_to_string(ranges)
+            state = ranges_to_state[ranges]
+            file.write("      %s --> State %d\n" % (key, state['number']))
+        for key in ('bol', 'eol', 'eof', 'else'):
+            state = special_to_state.get(key)
+            if state:
+                file.write("      %s --> State %d\n" % (key, state['number']))
+
+    def chars_to_ranges(self, char_list: list) -> tuple:
+        char_list.sort()
+
+        c1: cython.Py_UCS4
+        c2: cython.Py_UCS4
+        i: cython.Py_ssize_t = 0
+        n: cython.Py_ssize_t = len(char_list)
+        result = []
+        while i < n:
+            c1 = ord(char_list[i])
+            c2 = c1
+            i += 1
+            while i < n and ord(char_list[i]) == c2 + 1:
+                i += 1
+                c2 += 1
+            result.append((chr(c1), chr(c2)))
+        return tuple(result)
+
+    def ranges_to_string(self, range_list) -> str:
+        return ','.join(map(self.range_to_string, range_list))
+
+    def range_to_string(self, range_tuple: tuple):
+        (c1, c2) = range_tuple
+        if c1 == c2:
+            return repr(c1)
+        else:
+            return f"{c1!r}..{c2!r}"