Cython 3.3.0a1__cp315-cp315-win_amd64.whl

Cython/Plex/Actions.py

@@ -0,0 +1,119 @@
+"""
+Python Lexical Analyser
+
+Actions for use in token specifications
+"""
+
+class Action:
+    def perform(self, token_stream, text):
+        pass  # abstract
+
+    def __copy__(self):
+        return self  # immutable, no need to copy
+
+    def __deepcopy__(self, memo):
+        return self  # immutable, no need to copy
+
+
+class Return(Action):
+    """
+    Internal Plex action which causes |value| to
+    be returned as the value of the associated token
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+    def perform(self, token_stream, text):
+        return self.value
+
+    def __repr__(self):
+        return "Return(%r)" % self.value
+
+
+class Call(Action):
+    """
+    Internal Plex action which causes a function to be called.
+    """
+
+    def __init__(self, function):
+        self.function = function
+
+    def perform(self, token_stream, text):
+        return self.function(token_stream, text)
+
+    def __repr__(self):
+        return "Call(%s)" % self.function.__name__
+
+
+class Method(Action):
+    """
+    Plex action that calls a specific method on the token stream,
+    passing the matched text and any provided constant keyword arguments.
+    """
+
+    def __init__(self, name, **kwargs):
+        self.name = name
+        self.kwargs = kwargs or None
+
+    def perform(self, token_stream, text):
+        method = getattr(token_stream, self.name)
+        # self.kwargs is almost always unused => avoid call overhead
+        return method(text, **self.kwargs) if self.kwargs is not None else method(text)
+
+    def __repr__(self):
+        kwargs = (
+            ', '.join(sorted(['%s=%r' % item for item in self.kwargs.items()]))
+            if self.kwargs is not None else '')
+        return "Method(%s%s%s)" % (self.name, ', ' if kwargs else '', kwargs)
+
+
+class Begin(Action):
+    """
+    Begin(state_name) is a Plex action which causes the Scanner to
+    enter the state |state_name|. See the docstring of Plex.Lexicon
+    for more information.
+    """
+
+    def __init__(self, state_name):
+        self.state_name = state_name
+
+    def perform(self, token_stream, text):
+        token_stream.begin(self.state_name)
+
+    def __repr__(self):
+        return "Begin(%s)" % self.state_name
+
+
+class Ignore(Action):
+    """
+    IGNORE is a Plex action which causes its associated token
+    to be ignored. See the docstring of Plex.Lexicon  for more
+    information.
+    """
+
+    def perform(self, token_stream, text):
+        return None
+
+    def __repr__(self):
+        return "IGNORE"
+
+
+IGNORE = Ignore()
+
+
+class Text(Action):
+    """
+    TEXT is a Plex action which causes the text of a token to
+    be returned as the value of the token. See the docstring of
+    Plex.Lexicon  for more information.
+    """
+
+    def perform(self, token_stream, text):
+        return text
+
+    def __repr__(self):
+        return "TEXT"
+
+
+TEXT = Text()

Cython/Plex/DFA.pxd

@@ -0,0 +1,14 @@
+cimport cython
+
+from . cimport Machines
+
+
+@cython.final
+cdef class StateMap:
+    cdef Machines.FastMachine new_machine
+    cdef dict old_to_new_dict
+    cdef dict new_to_old_dict
+
+    cdef old_to_new(self, set old_state_set)
+    cdef highest_priority_action(self, set state_set)
+    cdef make_key(self, set state_set)

Cython/Plex/DFA.py

@@ -0,0 +1,164 @@
+# cython: auto_cpdef=True
+"""
+Python Lexical Analyser
+
+Converting NFA to DFA
+"""
+
+import cython
+from . import Machines
+from .Machines import LOWEST_PRIORITY
+from .Transitions import TransitionMap
+
+if cython.compiled:
+    from cython.cimports.Cython.Plex.Machines import Node, FastMachine
+    from cython.cimports.Cython.Plex.Transitions import TransitionMap as type_TransitionMap
+else:
+    from Cython.Plex.Machines import Node, FastMachine
+    from Cython.Plex.Transitions import TransitionMap as type_TransitionMap
+
+
+def nfa_to_dfa(old_machine, debug=None):
+    """
+    Given a nondeterministic Machine, return a new equivalent
+    Machine which is deterministic.
+    """
+    # We build a new machine whose states correspond to sets of states
+    # in the old machine. Initially we add a new state corresponding to
+    # the epsilon-closure of each initial old state. Then we give transitions
+    # to each new state which are the union of all transitions out of any
+    # of the corresponding old states. The new state reached on a given
+    # character is the one corresponding to the set of states reachable
+    # on that character from any of the old states. As new combinations of
+    # old states are created, new states are added as needed until closure
+    # is reached.
+    transitions: type_TransitionMap
+    new_machine: FastMachine = Machines.FastMachine()
+    state_map: StateMap = StateMap(new_machine)
+
+    # Seed the process using the initial states of the old machine.
+    # Make the corresponding new states into initial states of the new
+    # machine with the same names.
+    for (key, old_state) in old_machine.initial_states.items():
+        new_state = state_map.old_to_new(epsilon_closure(old_state))
+        new_machine.make_initial_state(key, new_state)
+
+    # Tricky bit here: we add things to the end of this list while we're
+    # iterating over it. The iteration stops when closure is achieved.
+    for new_state in new_machine.states:
+        transitions = TransitionMap()
+        for old_state in state_map.new_to_old(new_state):
+            for event, old_target_states in old_state.transitions.items():
+                if event and old_target_states:
+                    transitions.add_set(event, set_epsilon_closure(old_target_states))
+        for event, old_states in transitions.items():
+            new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states))
+
+    if debug:
+        debug.write("\n===== State Mapping =====\n")
+        state_map.dump(debug)
+    return new_machine
+
+
+@cython.cfunc
+def set_epsilon_closure(state_set: set) -> set:
+    """
+    Given a set of states, return the union of the epsilon
+    closures of its member states.
+    """
+    result = set()
+    for state1 in state_set:
+        for state2 in epsilon_closure(state1):
+            result.add(state2)
+    return result
+
+
+@cython.cfunc
+def epsilon_closure(state: Node) -> set:
+    """
+    Return the set of states reachable from the given state
+    by epsilon moves.
+    """
+    # Cache the result
+    result = state.epsilon_closure
+    if result is None:
+        result = set()
+        state.epsilon_closure = result
+        add_to_epsilon_closure(result, state)
+    return result
+
+
+@cython.cfunc
+def add_to_epsilon_closure(state_set: set, state: Node):
+    """
+    Recursively add to |state_set| states reachable from the given state
+    by epsilon moves.
+    """
+    state_set_2: set
+    state2: Node
+
+    if state not in state_set:
+        state_set.add(state)
+        state_set_2 = state.transitions.get_epsilon()
+        if state_set_2:
+            for state2 in state_set_2:
+                add_to_epsilon_closure(state_set, state2)
+
+
+class StateMap:
+    """
+    Helper class used by nfa_to_dfa() to map back and forth between
+    sets of states from the old machine and states of the new machine.
+    """
+
+    def __init__(self, new_machine):
+        self.new_machine = new_machine  # Machine
+        self.old_to_new_dict = {}  # {(old_state,...) : new_state}
+        self.new_to_old_dict = {}  # {id(new_state) : old_state_set}
+
+    def old_to_new(self, old_state_set: set):
+        """
+        Return the state of the new machine corresponding to the
+        set of old machine states represented by |state_set|. A new
+        state will be created if necessary. If any of the old states
+        are accepting states, the new state will be an accepting state
+        with the highest priority action from the old states.
+        """
+        key = self.make_key(old_state_set)
+        new_state = self.old_to_new_dict.get(key, None)
+        if not new_state:
+            action = self.highest_priority_action(old_state_set)
+            new_state = self.new_machine.new_state(action)
+            self.old_to_new_dict[key] = new_state
+            self.new_to_old_dict[id(new_state)] = old_state_set
+        return new_state
+
+    def highest_priority_action(self, state_set: set):
+        best_action = None
+        best_priority = LOWEST_PRIORITY
+        state: Node
+        for state in state_set:
+            priority = state.action_priority
+            if priority > best_priority:
+                best_action = state.action
+                best_priority = priority
+        return best_action
+
+    def new_to_old(self, new_state):
+        """Given a new state, return a set of corresponding old states."""
+        return self.new_to_old_dict[id(new_state)]
+
+    def make_key(self, state_set: set):
+        """
+        Convert a set of states into a uniquified
+        sorted tuple suitable for use as a dictionary key.
+        """
+        return tuple(sorted(state_set))
+
+    def dump(self, file):
+        from .Transitions import state_set_str
+
+        for new_state in self.new_machine.states:
+            old_state_set = self.new_to_old_dict[id(new_state)]
+            file.write("   State %s <-- %s\n" % (
+                new_state['number'], state_set_str(old_state_set)))

Cython/Plex/Errors.py

@@ -0,0 +1,48 @@
+"""
+Python Lexical Analyser
+
+Exception classes
+"""
+
+
+class PlexError(Exception):
+    message = ""
+
+
+class PlexTypeError(PlexError, TypeError):
+    pass
+
+
+class PlexValueError(PlexError, ValueError):
+    pass
+
+
+class InvalidToken(PlexError):
+    def __init__(self, token_number, message):
+        PlexError.__init__(self, "Token number %d: %s" % (token_number, message))
+
+
+class InvalidScanner(PlexError):
+    pass
+
+
+class AmbiguousAction(PlexError):
+    message = "Two tokens with different actions can match the same string"
+
+    def __init__(self):
+        pass
+
+
+class UnrecognizedInput(PlexError):
+    scanner = None
+    position = None
+    state_name = None
+
+    def __init__(self, scanner, state_name):
+        self.scanner = scanner
+        self.position = scanner.get_position()
+        self.state_name = state_name
+
+    def __str__(self):
+        return ("'%s', line %d, char %d: Token not recognised in state %r" % (
+            self.position + (self.state_name,)))

Cython/Plex/Lexicons.py

@@ -0,0 +1,178 @@
+"""
+Python Lexical Analyser
+
+Lexical Analyser Specification
+"""
+
+from . import Actions
+from . import DFA
+from . import Errors
+from . import Machines
+from . import Regexps
+
+# debug_flags for Lexicon constructor
+DUMP_NFA = 1
+DUMP_DFA = 2
+
+
+class State:
+    """
+    This class is used as part of a Plex.Lexicon specification to
+    introduce a user-defined state.
+
+    Constructor:
+
+       State(name, token_specifications)
+    """
+
+    name = None
+    tokens = None
+
+    def __init__(self, name, tokens):
+        self.name = name
+        self.tokens = tokens
+
+
+class Lexicon:
+    """
+    Lexicon(specification) builds a lexical analyser from the given
+    |specification|. The specification consists of a list of
+    specification items. Each specification item may be either:
+
+       1) A token definition, which is a tuple:
+
+             (pattern, action)
+
+          The |pattern| is a regular axpression built using the
+          constructors defined in the Plex module.
+
+          The |action| is the action to be performed when this pattern
+          is recognised (see below).
+
+       2) A state definition:
+
+             State(name, tokens)
+
+          where |name| is a character string naming the state,
+          and |tokens| is a list of token definitions as
+          above. The meaning and usage of states is described
+          below.
+
+    Actions
+    -------
+
+    The |action| in a token specification may be one of three things:
+
+       1) A function, which is called as follows:
+
+             function(scanner, text)
+
+          where |scanner| is the relevant Scanner instance, and |text|
+          is the matched text. If the function returns anything
+          other than None, that value is returned as the value of the
+          token. If it returns None, scanning continues as if the IGNORE
+          action were specified (see below).
+
+        2) One of the following special actions:
+
+           IGNORE means that the recognised characters will be treated as
+                  white space and ignored. Scanning will continue until
+                  the next non-ignored token is recognised before returning.
+
+           TEXT   causes the scanned text itself to be returned as the
+                  value of the token.
+
+        3) Any other value, which is returned as the value of the token.
+
+    States
+    ------
+
+    At any given time, the scanner is in one of a number of states.
+    Associated with each state is a set of possible tokens. When scanning,
+    only tokens associated with the current state are recognised.
+
+    There is a default state, whose name is the empty string. Token
+    definitions which are not inside any State definition belong to
+    the default state.
+
+    The initial state of the scanner is the default state. The state can
+    be changed in one of two ways:
+
+       1) Using Begin(state_name) as the action of a token.
+
+       2) Calling the begin(state_name) method of the Scanner.
+
+    To change back to the default state, use '' as the state name.
+    """
+
+    machine = None  # Machine
+    tables = None   # StateTableMachine
+
+    def __init__(self, specifications, debug=None, debug_flags=7):
+        if not isinstance(specifications, list):
+            raise Errors.InvalidScanner("Scanner definition is not a list")
+
+        nfa = Machines.Machine()
+        default_initial_state = nfa.new_initial_state('')
+        token_number = 1
+
+        for spec in specifications:
+            if isinstance(spec, State):
+                user_initial_state = nfa.new_initial_state(spec.name)
+                for token in spec.tokens:
+                    self.add_token_to_machine(
+                        nfa, user_initial_state, token, token_number)
+                    token_number += 1
+            elif isinstance(spec, tuple):
+                self.add_token_to_machine(
+                    nfa, default_initial_state, spec, token_number)
+                token_number += 1
+            else:
+                raise Errors.InvalidToken(
+                    token_number,
+                    "Expected a token definition (tuple) or State instance")
+
+        if debug and (debug_flags & 1):
+            debug.write("\n============= NFA ===========\n")
+            nfa.dump(debug)
+
+        dfa = DFA.nfa_to_dfa(nfa, debug=(debug_flags & 3) == 3 and debug)
+
+        if debug and (debug_flags & 2):
+            debug.write("\n============= DFA ===========\n")
+            dfa.dump(debug)
+
+        self.machine = dfa
+
+    def add_token_to_machine(self, machine, initial_state, token_spec, token_number):
+        try:
+            (re, action_spec) = self.parse_token_definition(token_spec)
+            if isinstance(action_spec, Actions.Action):
+                action = action_spec
+            else:
+                try:
+                    action_spec.__call__
+                except AttributeError:
+                    action = Actions.Return(action_spec)
+                else:
+                    action = Actions.Call(action_spec)
+            final_state = machine.new_state()
+            re.build_machine(machine, initial_state, final_state,
+                             match_bol=1, nocase=0)
+            final_state.set_action(action, priority=-token_number)
+        except Errors.PlexError as e:
+            raise e.__class__("Token number %d: %s" % (token_number, e))
+
+    def parse_token_definition(self, token_spec):
+        if not isinstance(token_spec, tuple):
+            raise Errors.InvalidToken("Token definition is not a tuple")
+        if len(token_spec) != 2:
+            raise Errors.InvalidToken("Wrong number of items in token definition")
+
+        pattern, action = token_spec
+        if not isinstance(pattern, Regexps.RE):
+            raise Errors.InvalidToken("Pattern is not an RE instance")
+        return (pattern, action)
+
+    def get_initial_state(self, name):
+        return self.machine.get_initial_state(name)

Cython/Plex/Machines.pxd

@@ -0,0 +1,36 @@
+cimport cython
+
+from .Actions cimport Action
+from .Transitions cimport TransitionMap
+
+cdef int maxint
+
+
+@cython.final
+cdef class Machine:
+    cdef readonly list states
+    cdef readonly dict  initial_states
+    cdef readonly Py_ssize_t next_state_number
+
+    cpdef new_state(self)
+    cpdef new_initial_state(self, name)
+    cpdef make_initial_state(self, name, state)
+
+
+@cython.final
+cdef class Node:
+    cdef readonly TransitionMap transitions
+    cdef readonly Action action
+    cdef public set epsilon_closure
+    cdef readonly Py_ssize_t number
+    cdef readonly int action_priority
+
+
+@cython.final
+cdef class FastMachine:
+    cdef readonly dict initial_states
+    cdef readonly dict new_state_template
+    cdef readonly list states
+    cdef readonly Py_ssize_t next_number
+
+    cpdef make_initial_state(self, name, state)