omlish 0.0.0.dev57__py3-none-any.whl → 0.0.0.dev59__py3-none-any.whl

omlish/antlr/_runtime/TokenStreamRewriter.py

@@ -0,0 +1,258 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#
+
+from io import StringIO
+from .Token import Token
+
+from .CommonTokenStream import CommonTokenStream
+
+
+class TokenStreamRewriter(object):
+    __slots__ = ('tokens', 'programs', 'lastRewriteTokenIndexes')
+
+    DEFAULT_PROGRAM_NAME = "default"
+    PROGRAM_INIT_SIZE = 100
+    MIN_TOKEN_INDEX = 0
+
+    def __init__(self, tokens):
+        """
+        :type  tokens: antlr4.BufferedTokenStream.BufferedTokenStream
+        :param tokens:
+        :return:
+        """
+        super(TokenStreamRewriter, self).__init__()
+        self.tokens = tokens
+        self.programs = {self.DEFAULT_PROGRAM_NAME: []}
+        self.lastRewriteTokenIndexes = {}
+
+    def getTokenStream(self):
+        return self.tokens
+
+    def rollback(self, instruction_index, program_name):
+        ins = self.programs.get(program_name, None)
+        if ins:
+            self.programs[program_name] = ins[self.MIN_TOKEN_INDEX: instruction_index]
+
+    def deleteProgram(self, program_name=DEFAULT_PROGRAM_NAME):
+        self.rollback(self.MIN_TOKEN_INDEX, program_name)
+
+    def insertAfterToken(self, token, text, program_name=DEFAULT_PROGRAM_NAME):
+        self.insertAfter(token.tokenIndex, text, program_name)
+
+    def insertAfter(self, index, text, program_name=DEFAULT_PROGRAM_NAME):
+        op = self.InsertAfterOp(self.tokens, index + 1, text)
+        rewrites = self.getProgram(program_name)
+        op.instructionIndex = len(rewrites)
+        rewrites.append(op)
+
+    def insertBeforeIndex(self, index, text):
+        self.insertBefore(self.DEFAULT_PROGRAM_NAME, index, text)
+
+    def insertBeforeToken(self, token, text, program_name=DEFAULT_PROGRAM_NAME):
+        self.insertBefore(program_name, token.tokenIndex, text)
+
+    def insertBefore(self, program_name, index, text):
+        op = self.InsertBeforeOp(self.tokens, index, text)
+        rewrites = self.getProgram(program_name)
+        op.instructionIndex = len(rewrites)
+        rewrites.append(op)
+
+    def replaceIndex(self, index, text):
+        self.replace(self.DEFAULT_PROGRAM_NAME, index, index, text)
+
+    def replaceRange(self, from_idx, to_idx, text):
+        self.replace(self.DEFAULT_PROGRAM_NAME, from_idx, to_idx, text)
+
+    def replaceSingleToken(self, token, text):
+        self.replace(self.DEFAULT_PROGRAM_NAME, token.tokenIndex, token.tokenIndex, text)
+
+    def replaceRangeTokens(self, from_token, to_token, text, program_name=DEFAULT_PROGRAM_NAME):
+        self.replace(program_name, from_token.tokenIndex, to_token.tokenIndex, text)
+
+    def replace(self, program_name, from_idx, to_idx, text):
+        if any((from_idx > to_idx, from_idx < 0, to_idx < 0, to_idx >= len(self.tokens.tokens))):
+            raise ValueError(
+                'replace: range invalid: {}..{}(size={})'.format(from_idx, to_idx, len(self.tokens.tokens)))
+        op = self.ReplaceOp(from_idx, to_idx, self.tokens, text)
+        rewrites = self.getProgram(program_name)
+        op.instructionIndex = len(rewrites)
+        rewrites.append(op)
+
+    def deleteToken(self, token):
+        self.delete(self.DEFAULT_PROGRAM_NAME, token, token)
+
+    def deleteIndex(self, index):
+        self.delete(self.DEFAULT_PROGRAM_NAME, index, index)
+
+    def delete(self, program_name, from_idx, to_idx):
+        if isinstance(from_idx, Token):
+            self.replace(program_name, from_idx.tokenIndex, to_idx.tokenIndex, "")
+        else:
+            self.replace(program_name, from_idx, to_idx, "")
+
+    def lastRewriteTokenIndex(self, program_name=DEFAULT_PROGRAM_NAME):
+        return self.lastRewriteTokenIndexes.get(program_name, -1)
+
+    def setLastRewriteTokenIndex(self, program_name, i):
+        self.lastRewriteTokenIndexes[program_name] = i
+
+    def getProgram(self, program_name):
+        return self.programs.setdefault(program_name, [])
+
+    def getDefaultText(self):
+        return self.getText(self.DEFAULT_PROGRAM_NAME, 0, len(self.tokens.tokens) - 1)
+
+    def getText(self, program_name, start:int, stop:int):
+        """
+        :return: the text in tokens[start, stop](closed interval)
+        """
+        rewrites = self.programs.get(program_name)
+
+        # ensure start/end are in range
+        if stop > len(self.tokens.tokens) - 1:
+            stop = len(self.tokens.tokens) - 1
+        if start < 0:
+            start = 0
+
+        # if no instructions to execute
+        if not rewrites: return self.tokens.getText(start, stop)
+        buf = StringIO()
+        indexToOp = self._reduceToSingleOperationPerIndex(rewrites)
+        i = start
+        while all((i <= stop, i < len(self.tokens.tokens))):
+            op = indexToOp.pop(i, None)
+            token = self.tokens.get(i)
+            if op is None:
+                if token.type != Token.EOF: buf.write(token.text)
+                i += 1
+            else:
+                i = op.execute(buf)
+
+        if stop == len(self.tokens.tokens)-1:
+            for op in indexToOp.values():
+                if op.index >= len(self.tokens.tokens)-1: buf.write(op.text)
+
+        return buf.getvalue()
+
+    def _reduceToSingleOperationPerIndex(self, rewrites):
+        # Walk replaces
+        for i, rop in enumerate(rewrites):
+            if any((rop is None, not isinstance(rop, TokenStreamRewriter.ReplaceOp))):
+                continue
+            # Wipe prior inserts within range
+            inserts = [op for op in rewrites[:i] if isinstance(op, TokenStreamRewriter.InsertBeforeOp)]
+            for iop in inserts:
+                if iop.index == rop.index:
+                    rewrites[iop.instructionIndex] = None
+                    rop.text = '{}{}'.format(iop.text, rop.text)
+                elif all((iop.index > rop.index, iop.index <= rop.last_index)):
+                    rewrites[iop.instructionIndex] = None
+
+            # Drop any prior replaces contained within
+            prevReplaces = [op for op in rewrites[:i] if isinstance(op, TokenStreamRewriter.ReplaceOp)]
+            for prevRop in prevReplaces:
+                if all((prevRop.index >= rop.index, prevRop.last_index <= rop.last_index)):
+                    rewrites[prevRop.instructionIndex] = None
+                    continue
+                isDisjoint = any((prevRop.last_index<rop.index, prevRop.index>rop.last_index))
+                if all((prevRop.text is None, rop.text is None, not isDisjoint)):
+                    rewrites[prevRop.instructionIndex] = None
+                    rop.index = min(prevRop.index, rop.index)
+                    rop.last_index = min(prevRop.last_index, rop.last_index)
+                    print('New rop {}'.format(rop))
+                elif (not(isDisjoint)):
+                    raise ValueError("replace op boundaries of {} overlap with previous {}".format(rop, prevRop))
+
+        # Walk inserts
+        for i, iop in enumerate(rewrites):
+            if any((iop is None, not isinstance(iop, TokenStreamRewriter.InsertBeforeOp))):
+                continue
+            prevInserts = [op for op in rewrites[:i] if isinstance(op, TokenStreamRewriter.InsertBeforeOp)]
+            for prev_index, prevIop in enumerate(prevInserts):
+                if prevIop.index == iop.index and type(prevIop) is TokenStreamRewriter.InsertBeforeOp:
+                    iop.text += prevIop.text
+                    rewrites[prev_index] = None
+                elif prevIop.index == iop.index and type(prevIop) is TokenStreamRewriter.InsertAfterOp:
+                    iop.text = prevIop.text + iop.text
+                    rewrites[prev_index] = None
+            # look for replaces where iop.index is in range; error
+            prevReplaces = [op for op in rewrites[:i] if isinstance(op, TokenStreamRewriter.ReplaceOp)]
+            for rop in prevReplaces:
+                if iop.index == rop.index:
+                    rop.text = iop.text + rop.text
+                    rewrites[i] = None
+                    continue
+                if all((iop.index >= rop.index, iop.index <= rop.last_index)):
+                    raise ValueError("insert op {} within boundaries of previous {}".format(iop, rop))
+
+        reduced = {}
+        for i, op in enumerate(rewrites):
+            if op is None: continue
+            if reduced.get(op.index): raise ValueError('should be only one op per index')
+            reduced[op.index] = op
+
+        return reduced
+
+    class RewriteOperation(object):
+        __slots__ = ('tokens', 'index', 'text', 'instructionIndex')
+
+        def __init__(self, tokens, index, text=""):
+            """
+            :type tokens: CommonTokenStream
+            :param tokens:
+            :param index:
+            :param text:
+            :return:
+            """
+            self.tokens = tokens
+            self.index = index
+            self.text = text
+            self.instructionIndex = 0
+
+        def execute(self, buf):
+            """
+            :type buf: StringIO.StringIO
+            :param buf:
+            :return:
+            """
+            return self.index
+
+        def __str__(self):
+            return '<{}@{}:"{}">'.format(self.__class__.__name__, self.tokens.get(self.index), self.text)
+
+    class InsertBeforeOp(RewriteOperation):
+
+        def __init__(self, tokens, index, text=""):
+            super(TokenStreamRewriter.InsertBeforeOp, self).__init__(tokens, index, text)
+
+        def execute(self, buf):
+            buf.write(self.text)
+            if self.tokens.get(self.index).type != Token.EOF:
+                buf.write(self.tokens.get(self.index).text)
+            return self.index + 1
+
+    class InsertAfterOp(InsertBeforeOp):
+        pass
+
+    class ReplaceOp(RewriteOperation):
+        __slots__ = 'last_index'
+
+        def __init__(self, from_idx, to_idx, tokens, text):
+            super(TokenStreamRewriter.ReplaceOp, self).__init__(tokens, from_idx, text)
+            self.last_index = to_idx
+
+        def execute(self, buf):
+            if self.text:
+                buf.write(self.text)
+            return self.last_index + 1
+
+        def __str__(self):
+            if self.text:
+                return '<ReplaceOp@{}..{}:"{}">'.format(self.tokens.get(self.index), self.tokens.get(self.last_index),
+                                                        self.text)

omlish/antlr/_runtime/Utils.py

@@ -0,0 +1,36 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#
+
+from io import StringIO
+
+def str_list(val):
+    with StringIO() as buf:
+        buf.write('[')
+        first = True
+        for item in val:
+            if not first:
+                buf.write(', ')
+            buf.write(str(item))
+            first = False
+        buf.write(']')
+        return buf.getvalue()
+
+def escapeWhitespace(s:str, escapeSpaces:bool):
+    with StringIO() as buf:
+        for c in s:
+            if c==' ' and escapeSpaces:
+                buf.write('\u00B7')
+            elif c=='\t':
+                buf.write("\\t")
+            elif c=='\n':
+                buf.write("\\n")
+            elif c=='\r':
+                buf.write("\\r")
+            else:
+                buf.write(c)
+        return buf.getvalue()

omlish/antlr/_runtime/__init__.py

@@ -0,0 +1,24 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+from .Token import Token
+from .InputStream import InputStream
+from .FileStream import FileStream
+from .StdinStream import StdinStream
+from .BufferedTokenStream import TokenStream
+from .CommonTokenStream import CommonTokenStream
+from .Lexer import Lexer
+from .Parser import Parser
+from .dfa.DFA import DFA
+from .atn.ATN import ATN
+from .atn.ATNDeserializer import ATNDeserializer
+from .atn.LexerATNSimulator import LexerATNSimulator
+from .atn.ParserATNSimulator import ParserATNSimulator
+from .atn.PredictionMode import PredictionMode
+from .PredictionContext import PredictionContextCache
+from .ParserRuleContext import RuleContext, ParserRuleContext
+from .tree.Tree import ParseTreeListener, ParseTreeVisitor, ParseTreeWalker, TerminalNode, ErrorNode, RuleNode
+from .error.Errors import RecognitionException, IllegalStateException, NoViableAltException
+from .error.ErrorStrategy import BailErrorStrategy
+from .error.DiagnosticErrorListener import DiagnosticErrorListener
+from .Utils import str_list

omlish/antlr/_runtime/_pygrun.py

@@ -0,0 +1,174 @@
+#!python
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+__author__ = 'jszheng'
+import optparse
+import sys
+import os
+import importlib
+from .. import *
+
+
+# this is a python version of TestRig
+def beautify_lisp_string(in_string):
+    indent_size = 3
+    add_indent = ' '*indent_size
+    out_string = in_string[0]  # no indent for 1st (
+    indent = ''
+    for i in range(1, len(in_string)):
+        if in_string[i] == '(' and in_string[i+1] != ' ':
+            indent += add_indent
+            out_string += "\n" + indent + '('
+        elif in_string[i] == ')':
+            out_string += ')'
+            if len(indent) > 0:
+                indent = indent.replace(add_indent, '', 1)
+        else:
+            out_string += in_string[i]
+    return out_string
+
+
+def main():
+
+    #############################################################
+    # parse options
+    # not support -gui -encoding -ps
+    #############################################################
+    usage = "Usage: %prog [options] Grammar_Name Start_Rule"
+    parser = optparse.OptionParser(usage=usage)
+    # parser.add_option('-t', '--tree',
+    #                   dest="out_file",
+    #                   default="default.out",
+    #                   help='set output file name',
+    #                   )
+    parser.add_option('-t', '--tree',
+                      default=False,
+                      action='store_true',
+                      help='Print AST tree'
+                      )
+    parser.add_option('-k', '--tokens',
+                      dest="token",
+                      default=False,
+                      action='store_true',
+                      help='Show Tokens'
+                      )
+    parser.add_option('-s', '--sll',
+                      dest="sll",
+                      default=False,
+                      action='store_true',
+                      help='Show SLL'
+                      )
+    parser.add_option('-d', '--diagnostics',
+                      dest="diagnostics",
+                      default=False,
+                      action='store_true',
+                      help='Enable diagnostics error listener'
+                      )
+    parser.add_option('-a', '--trace',
+                      dest="trace",
+                      default=False,
+                      action='store_true',
+                      help='Enable Trace'
+                      )
+
+    options, remainder = parser.parse_args()
+    if len(remainder) < 2:
+        print('ERROR: You have to provide at least 2 arguments!')
+        parser.print_help()
+        exit(1)
+    else:
+        grammar = remainder.pop(0)
+        start_rule = remainder.pop(0)
+        file_list = remainder
+
+    #############################################################
+    # check and load antlr generated files
+    #############################################################
+    # dynamic load the module and class
+    lexerName = grammar + 'Lexer'
+    parserName = grammar + 'Parser'
+    # check if the generate file exist
+    lexer_file = lexerName + '.py'
+    parser_file = parserName + '.py'
+    if not os.path.exists(lexer_file):
+        print("[ERROR] Can't find lexer file {}!".format(lexer_file))
+        print(os.path.realpath('.'))
+        exit(1)
+    if not os.path.exists(parser_file):
+        print("[ERROR] Can't find parser file {}!".format(lexer_file))
+        print(os.path.realpath('.'))
+        exit(1)
+
+    # current directory is where the generated file loaded
+    # the script might be in different place.
+    sys.path.append('.')
+    # print(sys.path)
+
+    # add current directory to python global namespace in case of relative imports
+    globals().update({'__package__': os.path.basename(os.getcwd())})
+
+    # print("Load Lexer {}".format(lexerName))
+    module_lexer = __import__(lexerName, globals(), locals(), lexerName)
+    class_lexer = getattr(module_lexer, lexerName)
+    # print(class_lexer)
+
+    # print("Load Parser {}".format(parserName))
+    module_parser = __import__(parserName, globals(), locals(), parserName)
+    class_parser = getattr(module_parser, parserName)
+    # print(class_parser)
+
+    #############################################################
+    # main process steps.
+    #############################################################
+    def process(input_stream, class_lexer, class_parser):
+        lexer = class_lexer(input_stream)
+        token_stream = CommonTokenStream(lexer)
+        token_stream.fill()
+        if options.token:  # need to show token
+            for tok in token_stream.tokens:
+                print(tok)
+        if start_rule == 'tokens':
+            return
+
+        parser = class_parser(token_stream)
+
+        if options.diagnostics:
+            parser.addErrorListener(DiagnosticErrorListener())
+            parser._interp.predictionMode = PredictionMode.LL_EXACT_AMBIG_DETECTION
+        if options.tree:
+            parser.buildParseTrees = True
+        if options.sll:
+            parser._interp.predictionMode = PredictionMode.SLL
+        #parser.setTokenStream(token_stream)
+        parser.setTrace(options.trace)
+        if hasattr(parser, start_rule):
+            func_start_rule = getattr(parser, start_rule)
+            parser_ret = func_start_rule()
+            if options.tree:
+                lisp_tree_str = parser_ret.toStringTree(recog=parser)
+                print(beautify_lisp_string(lisp_tree_str))
+        else:
+            print("[ERROR] Can't find start rule '{}' in parser '{}'".format(start_rule, parserName))
+
+    #############################################################
+    # use stdin if not provide file as input stream
+    #############################################################
+    if len(file_list) == 0:
+        input_stream = InputStream(sys.stdin.read())
+        process(input_stream, class_lexer, class_parser)
+        exit(0)
+
+    #############################################################
+    # iterate all input file
+    #############################################################
+    for file_name in file_list:
+        if os.path.exists(file_name) and os.path.isfile(file_name):
+            input_stream = FileStream(file_name)
+            process(input_stream, class_lexer, class_parser)
+        else:
+            print("[ERROR] file {} not exist".format(os.path.normpath(file_name)))
+
+
+if __name__ == '__main__':
+    main()

omlish/antlr/_runtime/atn/ATN.py

@@ -0,0 +1,135 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#/
+from ..IntervalSet import IntervalSet
+
+from ..RuleContext import RuleContext
+
+from ..Token import Token
+from .ATNType import ATNType
+from .ATNState import ATNState, DecisionState
+
+
+class ATN(object):
+    __slots__ = (
+        'grammarType', 'maxTokenType', 'states', 'decisionToState',
+        'ruleToStartState', 'ruleToStopState', 'modeNameToStartState',
+        'ruleToTokenType', 'lexerActions', 'modeToStartState'
+    )
+
+    INVALID_ALT_NUMBER = 0
+
+    # Used for runtime deserialization of ATNs from strings#/
+    def __init__(self, grammarType:ATNType , maxTokenType:int ):
+        # The type of the ATN.
+        self.grammarType = grammarType
+        # The maximum value for any symbol recognized by a transition in the ATN.
+        self.maxTokenType = maxTokenType
+        self.states = []
+        # Each subrule/rule is a decision point and we must track them so we
+        #  can go back later and build DFA predictors for them.  This includes
+        #  all the rules, subrules, optional blocks, ()+, ()* etc...
+        self.decisionToState = []
+        # Maps from rule index to starting state number.
+        self.ruleToStartState = []
+        # Maps from rule index to stop state number.
+        self.ruleToStopState = None
+        self.modeNameToStartState = dict()
+        # For lexer ATNs, this maps the rule index to the resulting token type.
+        # For parser ATNs, this maps the rule index to the generated bypass token
+        # type if the
+        # {@link ATNDeserializationOptions#isGenerateRuleBypassTransitions}
+        # deserialization option was specified; otherwise, this is {@code null}.
+        self.ruleToTokenType = None
+        # For lexer ATNs, this is an array of {@link LexerAction} objects which may
+        # be referenced by action transitions in the ATN.
+        self.lexerActions = None
+        self.modeToStartState = []
+
+    # Compute the set of valid tokens that can occur starting in state {@code s}.
+    #  If {@code ctx} is null, the set of tokens will not include what can follow
+    #  the rule surrounding {@code s}. In other words, the set will be
+    #  restricted to tokens reachable staying within {@code s}'s rule.
+    def nextTokensInContext(self, s:ATNState, ctx:RuleContext):
+        from ..LL1Analyzer import LL1Analyzer
+        anal = LL1Analyzer(self)
+        return anal.LOOK(s, ctx=ctx)
+
+    # Compute the set of valid tokens that can occur starting in {@code s} and
+    # staying in same rule. {@link Token#EPSILON} is in set if we reach end of
+    # rule.
+    def nextTokensNoContext(self, s:ATNState):
+        if s.nextTokenWithinRule is not None:
+            return s.nextTokenWithinRule
+        s.nextTokenWithinRule = self.nextTokensInContext(s, None)
+        s.nextTokenWithinRule.readonly = True
+        return s.nextTokenWithinRule
+
+    def nextTokens(self, s:ATNState, ctx:RuleContext = None):
+        if ctx==None:
+            return self.nextTokensNoContext(s)
+        else:
+            return self.nextTokensInContext(s, ctx)
+
+    def addState(self, state:ATNState):
+        if state is not None:
+            state.atn = self
+            state.stateNumber = len(self.states)
+        self.states.append(state)
+
+    def removeState(self, state:ATNState):
+        self.states[state.stateNumber] = None # just free mem, don't shift states in list
+
+    def defineDecisionState(self, s:DecisionState):
+        self.decisionToState.append(s)
+        s.decision = len(self.decisionToState)-1
+        return s.decision
+
+    def getDecisionState(self, decision:int):
+        if len(self.decisionToState)==0:
+            return None
+        else:
+            return self.decisionToState[decision]
+
+    # Computes the set of input symbols which could follow ATN state number
+    # {@code stateNumber} in the specified full {@code context}. This method
+    # considers the complete parser context, but does not evaluate semantic
+    # predicates (i.e. all predicates encountered during the calculation are
+    # assumed true). If a path in the ATN exists from the starting state to the
+    # {@link RuleStopState} of the outermost context without matching any
+    # symbols, {@link Token#EOF} is added to the returned set.
+    #
+    # <p>If {@code context} is {@code null}, it is treated as
+    # {@link ParserRuleContext#EMPTY}.</p>
+    #
+    # @param stateNumber the ATN state number
+    # @param context the full parse context
+    # @return The set of potentially valid input symbols which could follow the
+    # specified state in the specified context.
+    # @throws IllegalArgumentException if the ATN does not contain a state with
+    # number {@code stateNumber}
+    #/
+    def getExpectedTokens(self, stateNumber:int, ctx:RuleContext ):
+        if stateNumber < 0 or stateNumber >= len(self.states):
+            raise Exception("Invalid state number.")
+        s = self.states[stateNumber]
+        following = self.nextTokens(s)
+        if Token.EPSILON not in following:
+            return following
+        expected = IntervalSet()
+        expected.addSet(following)
+        expected.removeOne(Token.EPSILON)
+        while (ctx != None and ctx.invokingState >= 0 and Token.EPSILON in following):
+            invokingState = self.states[ctx.invokingState]
+            rt = invokingState.transitions[0]
+            following = self.nextTokens(rt.followState)
+            expected.addSet(following)
+            expected.removeOne(Token.EPSILON)
+            ctx = ctx.parentCtx
+        if Token.EPSILON in following:
+            expected.addOne(Token.EOF)
+        return expected