omextra 0.0.0.dev437__py3-none-any.whl → 0.0.0.dev439__py3-none-any.whl

omextra/formats/json5/parsing.py

@@ -0,0 +1,101 @@
+# ruff: noqa: N802 N803
+import typing as ta
+
+from omlish.formats.json5.errors import Json5Error
+from omlish.formats.json5.literals import LITERAL_VALUES
+from omlish.formats.json5.literals import parse_number_literal
+from omlish.formats.json5.literals import parse_string_literal
+
+from ...text import antlr
+from ._antlr.Json5Lexer import Json5Lexer  # type: ignore
+from ._antlr.Json5Parser import Json5Parser  # type: ignore
+from ._antlr.Json5Visitor import Json5Visitor  # type: ignore
+
+
+##
+
+
+class Json5ParseVisitor(antlr.parsing.StandardParseTreeVisitor, Json5Visitor):
+    def visitArr(self, ctx: Json5Parser.ArrContext):
+        return [self.visit(e) for e in ctx.value()]
+
+    def visitKey(self, ctx: Json5Parser.KeyContext):
+        if (s := ctx.STRING()) is not None:
+            return parse_string_literal(s.getText())
+
+        elif (i := ctx.IDENTIFIER()) is not None:
+            return parse_string_literal(''.join(['"', i.getText(), '"']))
+
+        elif (l := ctx.LITERAL()) is not None:
+            return LITERAL_VALUES[l.getText()]
+
+        elif (n := ctx.NUMERIC_LITERAL()) is not None:
+            return n.getText()
+
+        else:
+            raise RuntimeError(ctx)
+
+    def visitNumber(self, ctx: Json5Parser.NumberContext):
+        return parse_number_literal(ctx.getText())
+
+    def visitObj(self, ctx: Json5Parser.ObjContext):
+        dct: dict[ta.Any, ta.Any] = {}
+        for pair in ctx.pair():
+            key, value = self.visit(pair)
+            dct[key] = value
+        return dct
+
+    def visitPair(self, ctx: Json5Parser.PairContext):
+        key = self.visit(ctx.key())
+        value = self.visit(ctx.value())
+        return (key, value)
+
+    def visitValue(self, ctx: Json5Parser.ValueContext):
+        if (s := ctx.STRING()) is not None:
+            return parse_string_literal(s.getText())
+
+        elif (n := ctx.LITERAL()) is not None:
+            return LITERAL_VALUES[n.getText()]
+
+        else:
+            return super().visitChildren(ctx)
+
+
+def _make_parser(buf: str) -> Json5Parser:
+    return antlr.parsing.make_parser(
+        buf,
+        Json5Lexer,
+        Json5Parser,
+        silent_errors=True,
+    )
+
+
+def parse(buf: str) -> ta.Any:
+    try:
+        root = _make_parser(buf).json5()
+
+    except antlr.errors.ParseError as e:
+        raise Json5Error from e
+
+    if antlr.parsing.is_eof_context(root):
+        raise Json5Error('Empty input')
+
+    visitor = Json5ParseVisitor()
+    return visitor.visit(root)
+
+
+def parse_many(buf: str) -> ta.Iterator[ta.Any]:
+    try:
+        parser = _make_parser(buf)
+
+        while True:
+            if parser.getInputStream().LT(1).type == antlr.runtime.Token.EOF:
+                break
+
+            value = parser.value()
+
+            visitor = Json5ParseVisitor()
+            yield visitor.visit(value)
+
+    except antlr.errors.ParseError as e:
+        raise Json5Error from e

omextra/specs/proto/_antlr/Protobuf3Lexer.py

@@ -3,7 +3,7 @@
 # flake8: noqa
 # @omlish-generated
 # Generated from Protobuf3.g4 by ANTLR 4.13.2
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 from io import StringIO
 import sys
 if sys.version_info[1] > 5:

omextra/specs/proto/_antlr/Protobuf3Listener.py

@@ -3,7 +3,7 @@
 # flake8: noqa
 # @omlish-generated
 # Generated from Protobuf3.g4 by ANTLR 4.13.2
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 if "." in __name__:
     from .Protobuf3Parser import Protobuf3Parser
 else:

omextra/specs/proto/_antlr/Protobuf3Parser.py

@@ -4,7 +4,7 @@
 # @omlish-generated
 # Generated from Protobuf3.g4 by ANTLR 4.13.2
 # encoding: utf-8
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 from io import StringIO
 import sys
 if sys.version_info[1] > 5:

omextra/specs/proto/_antlr/Protobuf3Visitor.py

@@ -3,7 +3,7 @@
 # flake8: noqa
 # @omlish-generated
 # Generated from Protobuf3.g4 by ANTLR 4.13.2
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 if "." in __name__:
     from .Protobuf3Parser import Protobuf3Parser
 else:

omextra/specs/proto/parsing.py

@@ -2,9 +2,9 @@
 import typing as ta
 
 from omlish import check
-from omlish.text.antlr import runtime as antlr4
-from omlish.text.antlr.errors import SilentRaisingErrorListener
 
+from ...text.antlr import runtime as antlr4
+from ...text.antlr.errors import SilentRaisingErrorListener
 from . import nodes as no
 from ._antlr.Protobuf3Lexer import Protobuf3Lexer  # type: ignore
 from ._antlr.Protobuf3Parser import Protobuf3Parser  # type: ignore

omextra/sql/parsing/_antlr/MinisqlLexer.py

@@ -3,7 +3,7 @@
 # flake8: noqa
 # @omlish-generated
 # Generated from Minisql.g4 by ANTLR 4.13.2
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 from io import StringIO
 import sys
 if sys.version_info[1] > 5:

omextra/sql/parsing/_antlr/MinisqlListener.py

@@ -3,7 +3,7 @@
 # flake8: noqa
 # @omlish-generated
 # Generated from Minisql.g4 by ANTLR 4.13.2
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 if "." in __name__:
     from .MinisqlParser import MinisqlParser
 else:

omextra/sql/parsing/_antlr/MinisqlParser.py

@@ -4,7 +4,7 @@
 # @omlish-generated
 # Generated from Minisql.g4 by ANTLR 4.13.2
 # encoding: utf-8
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 from io import StringIO
 import sys
 if sys.version_info[1] > 5:

omextra/sql/parsing/_antlr/MinisqlVisitor.py

@@ -3,7 +3,7 @@
 # flake8: noqa
 # @omlish-generated
 # Generated from Minisql.g4 by ANTLR 4.13.2
-from omlish.text.antlr._runtime._all import *
+from ....text.antlr._runtime._all import *
 if "." in __name__:
     from .MinisqlParser import MinisqlParser
 else:

omextra/sql/parsing/parsing.py

@@ -3,10 +3,10 @@ import typing as ta
 
 from omlish import check
 from omlish.sql import queries as no
-from omlish.text.antlr import runtime as antlr4
-from omlish.text.antlr.delimit import DelimitingLexer
-from omlish.text.antlr.errors import SilentRaisingErrorListener
 
+from ...text.antlr import runtime as antlr4
+from ...text.antlr.delimit import DelimitingLexer
+from ...text.antlr.errors import SilentRaisingErrorListener
 from ._antlr.MinisqlLexer import MinisqlLexer  # type: ignore
 from ._antlr.MinisqlParser import MinisqlParser  # type: ignore
 from ._antlr.MinisqlVisitor import MinisqlVisitor  # type: ignore

omextra/text/antlr/__init__.py

@@ -0,0 +1,3 @@
+from . import errors  # noqa
+from . import parsing  # noqa
+from . import runtime  # noqa

omextra/text/antlr/_runtime/BufferedTokenStream.py

@@ -0,0 +1,305 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+
+# This implementation of {@link TokenStream} loads tokens from a
+# {@link TokenSource} on-demand, and places the tokens in a buffer to provide
+# access to any previous token by index.
+#
+# <p>
+# This token stream ignores the value of {@link Token#getChannel}. If your
+# parser requires the token stream filter tokens to only those on a particular
+# channel, such as {@link Token#DEFAULT_CHANNEL} or
+# {@link Token#HIDDEN_CHANNEL}, use a filtering token stream such a
+# {@link CommonTokenStream}.</p>
+from io import StringIO
+from .Token import Token
+from .error.Errors import IllegalStateException
+
+# need forward declaration
+Lexer = None
+
+# this is just to keep meaningful parameter types to Parser
+class TokenStream(object):
+
+    pass
+
+
+class BufferedTokenStream(TokenStream):
+    __slots__ = ('tokenSource', 'tokens', 'index', 'fetchedEOF')
+
+    def __init__(self, tokenSource:Lexer):
+        # The {@link TokenSource} from which tokens for this stream are fetched.
+        self.tokenSource = tokenSource
+
+        # A collection of all tokens fetched from the token source. The list is
+        # considered a complete view of the input once {@link #fetchedEOF} is set
+        # to {@code true}.
+        self.tokens = []
+
+        # The index into {@link #tokens} of the current token (next token to
+        # {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be
+        # {@link #LT LT(1)}.
+        #
+        # <p>This field is set to -1 when the stream is first constructed or when
+        # {@link #setTokenSource} is called, indicating that the first token has
+        # not yet been fetched from the token source. For additional information,
+        # see the documentation of {@link IntStream} for a description of
+        # Initializing Methods.</p>
+        self.index = -1
+
+        # Indicates whether the {@link Token#EOF} token has been fetched from
+        # {@link #tokenSource} and added to {@link #tokens}. This field improves
+        # performance for the following cases:
+        #
+        # <ul>
+        # <li>{@link #consume}: The lookahead check in {@link #consume} to prevent
+        # consuming the EOF symbol is optimized by checking the values of
+        # {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.</li>
+        # <li>{@link #fetch}: The check to prevent adding multiple EOF symbols into
+        # {@link #tokens} is trivial with this field.</li>
+        # <ul>
+        self.fetchedEOF = False
+
+    def mark(self):
+        return 0
+
+    def release(self, marker:int):
+        # no resources to release
+        pass
+
+    def reset(self):
+        self.seek(0)
+
+    def seek(self, index:int):
+        self.lazyInit()
+        self.index = self.adjustSeekIndex(index)
+
+    def get(self, index:int):
+        self.lazyInit()
+        return self.tokens[index]
+
+    def consume(self):
+        skipEofCheck = False
+        if self.index >= 0:
+            if self.fetchedEOF:
+                # the last token in tokens is EOF. skip check if p indexes any
+                # fetched token except the last.
+                skipEofCheck = self.index < len(self.tokens) - 1
+            else:
+               # no EOF token in tokens. skip check if p indexes a fetched token.
+                skipEofCheck = self.index < len(self.tokens)
+        else:
+            # not yet initialized
+            skipEofCheck = False
+
+        if not skipEofCheck and self.LA(1) == Token.EOF:
+            raise IllegalStateException("cannot consume EOF")
+
+        if self.sync(self.index + 1):
+            self.index = self.adjustSeekIndex(self.index + 1)
+
+    # Make sure index {@code i} in tokens has a token.
+    #
+    # @return {@code true} if a token is located at index {@code i}, otherwise
+    #    {@code false}.
+    # @see #get(int i)
+    #/
+    def sync(self, i:int):
+        n = i - len(self.tokens) + 1 # how many more elements we need?
+        if n > 0 :
+            fetched = self.fetch(n)
+            return fetched >= n
+        return True
+
+    # Add {@code n} elements to buffer.
+    #
+    # @return The actual number of elements added to the buffer.
+    #/
+    def fetch(self, n:int):
+        if self.fetchedEOF:
+            return 0
+        for i in range(0, n):
+            t = self.tokenSource.nextToken()
+            t.tokenIndex = len(self.tokens)
+            self.tokens.append(t)
+            if t.type==Token.EOF:
+                self.fetchedEOF = True
+                return i + 1
+        return n
+
+
+    # Get all tokens from start..stop inclusively#/
+    def getTokens(self, start:int, stop:int, types:set=None):
+        if start<0 or stop<0:
+            return None
+        self.lazyInit()
+        subset = []
+        if stop >= len(self.tokens):
+            stop = len(self.tokens)-1
+        for i in range(start, stop):
+            t = self.tokens[i]
+            if t.type==Token.EOF:
+                break
+            if types is None or t.type in types:
+                subset.append(t)
+        return subset
+
+    def LA(self, i:int):
+        return self.LT(i).type
+
+    def LB(self, k:int):
+        if (self.index-k) < 0:
+            return None
+        return self.tokens[self.index-k]
+
+    def LT(self, k:int):
+        self.lazyInit()
+        if k==0:
+            return None
+        if k < 0:
+            return self.LB(-k)
+        i = self.index + k - 1
+        self.sync(i)
+        if i >= len(self.tokens): # return EOF token
+            # EOF must be last token
+            return self.tokens[len(self.tokens)-1]
+        return self.tokens[i]
+
+    # Allowed derived classes to modify the behavior of operations which change
+    # the current stream position by adjusting the target token index of a seek
+    # operation. The default implementation simply returns {@code i}. If an
+    # exception is thrown in this method, the current stream index should not be
+    # changed.
+    #
+    # <p>For example, {@link CommonTokenStream} overrides this method to ensure that
+    # the seek target is always an on-channel token.</p>
+    #
+    # @param i The target token index.
+    # @return The adjusted target token index.
+
+    def adjustSeekIndex(self, i:int):
+        return i
+
+    def lazyInit(self):
+        if self.index == -1:
+            self.setup()
+
+    def setup(self):
+        self.sync(0)
+        self.index = self.adjustSeekIndex(0)
+
+    # Reset this token stream by setting its token source.#/
+    def setTokenSource(self, tokenSource:Lexer):
+        self.tokenSource = tokenSource
+        self.tokens = []
+        self.index = -1
+        self.fetchedEOF = False
+
+
+    # Given a starting index, return the index of the next token on channel.
+    #  Return i if tokens[i] is on channel.  Return the index of the EOF token
+    # if there are no tokens on channel between i and EOF.
+    #/
+    def nextTokenOnChannel(self, i:int, channel:int):
+        self.sync(i)
+        if i>=len(self.tokens):
+            return len(self.tokens) - 1
+        token = self.tokens[i]
+        while token.channel!=channel:
+            if token.type==Token.EOF:
+                return i
+            i += 1
+            self.sync(i)
+            token = self.tokens[i]
+        return i
+
+    # Given a starting index, return the index of the previous token on channel.
+    #  Return i if tokens[i] is on channel. Return -1 if there are no tokens
+    #  on channel between i and 0.
+    def previousTokenOnChannel(self, i:int, channel:int):
+        while i>=0 and self.tokens[i].channel!=channel:
+            i -= 1
+        return i
+
+    # Collect all tokens on specified channel to the right of
+    #  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
+    #  EOF. If channel is -1, find any non default channel token.
+    def getHiddenTokensToRight(self, tokenIndex:int, channel:int=-1):
+        self.lazyInit()
+        if tokenIndex<0 or tokenIndex>=len(self.tokens):
+            raise Exception(str(tokenIndex) + " not in 0.." + str(len(self.tokens)-1))
+        from .Lexer import Lexer
+        nextOnChannel = self.nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL)
+        from_ = tokenIndex+1
+        # if none onchannel to right, nextOnChannel=-1 so set to = last token
+        to = (len(self.tokens)-1) if nextOnChannel==-1 else nextOnChannel
+        return self.filterForChannel(from_, to, channel)
+
+
+    # Collect all tokens on specified channel to the left of
+    #  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
+    #  If channel is -1, find any non default channel token.
+    def getHiddenTokensToLeft(self, tokenIndex:int, channel:int=-1):
+        self.lazyInit()
+        if tokenIndex<0 or tokenIndex>=len(self.tokens):
+            raise Exception(str(tokenIndex) + " not in 0.." + str(len(self.tokens)-1))
+        from .Lexer import Lexer
+        prevOnChannel = self.previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL)
+        if prevOnChannel == tokenIndex - 1:
+            return None
+        # if none on channel to left, prevOnChannel=-1 then from=0
+        from_ = prevOnChannel+1
+        to = tokenIndex-1
+        return self.filterForChannel(from_, to, channel)
+
+
+    def filterForChannel(self, left:int, right:int, channel:int):
+        hidden = []
+        for i in range(left, right+1):
+            t = self.tokens[i]
+            if channel==-1:
+                from .Lexer import Lexer
+                if t.channel!= Lexer.DEFAULT_TOKEN_CHANNEL:
+                    hidden.append(t)
+            elif t.channel==channel:
+                    hidden.append(t)
+        if len(hidden)==0:
+            return None
+        return hidden
+
+    def getSourceName(self):
+        return self.tokenSource.getSourceName()
+
+    # Get the text of all tokens in this buffer.#/
+    def getText(self, start:int=None, stop:int=None):
+        self.lazyInit()
+        self.fill()
+        if isinstance(start, Token):
+            start = start.tokenIndex
+        elif start is None:
+            start = 0
+        if isinstance(stop, Token):
+            stop = stop.tokenIndex
+        elif stop is None or stop >= len(self.tokens):
+            stop = len(self.tokens) - 1
+        if start < 0 or stop < 0 or stop < start:
+            return ""
+        with StringIO() as buf:
+            for i in range(start, stop+1):
+                t = self.tokens[i]
+                if t.type==Token.EOF:
+                    break
+                buf.write(t.text)
+            return buf.getvalue()
+
+
+    # Get all tokens from lexer until EOF#/
+    def fill(self):
+        self.lazyInit()
+        while self.fetch(1000)==1000:
+            pass

omextra/text/antlr/_runtime/CommonTokenFactory.py

@@ -0,0 +1,64 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#
+
+#
+# This default implementation of {@link TokenFactory} creates
+# {@link CommonToken} objects.
+#
+from .Token import CommonToken
+
+class TokenFactory(object):
+
+    pass
+
+class CommonTokenFactory(TokenFactory):
+    __slots__ = 'copyText'
+
+    #
+    # The default {@link CommonTokenFactory} instance.
+    #
+    # <p>
+    # This token factory does not explicitly copy token text when constructing
+    # tokens.</p>
+    #
+    DEFAULT = None
+
+    def __init__(self, copyText:bool=False):
+        # Indicates whether {@link CommonToken#setText} should be called after
+        # constructing tokens to explicitly set the text. This is useful for cases
+        # where the input stream might not be able to provide arbitrary substrings
+        # of text from the input after the lexer creates a token (e.g. the
+        # implementation of {@link CharStream#getText} in
+        # {@link UnbufferedCharStream} throws an
+        # {@link UnsupportedOperationException}). Explicitly setting the token text
+        # allows {@link Token#getText} to be called at any time regardless of the
+        # input stream implementation.
+        #
+        # <p>
+        # The default value is {@code false} to avoid the performance and memory
+        # overhead of copying text for every token unless explicitly requested.</p>
+        #
+        self.copyText = copyText
+
+    def create(self, source, type:int, text:str, channel:int, start:int, stop:int, line:int, column:int):
+        t = CommonToken(source, type, channel, start, stop)
+        t.line = line
+        t.column = column
+        if text is not None:
+            t.text = text
+        elif self.copyText and source[1] is not None:
+            t.text = source[1].getText(start,stop)
+        return t
+
+    def createThin(self, type:int, text:str):
+        t = CommonToken(type=type)
+        t.text = text
+        return t
+
+CommonTokenFactory.DEFAULT = CommonTokenFactory()

omextra/text/antlr/_runtime/CommonTokenStream.py

@@ -0,0 +1,90 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#/
+
+#
+# This class extends {@link BufferedTokenStream} with functionality to filter
+# token streams to tokens on a particular channel (tokens where
+# {@link Token#getChannel} returns a particular value).
+#
+# <p>
+# This token stream provides access to all tokens by index or when calling
+# methods like {@link #getText}. The channel filtering is only used for code
+# accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and
+# {@link #LB}.</p>
+#
+# <p>
+# By default, tokens are placed on the default channel
+# ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the
+# {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to
+# call {@link Lexer#setChannel}.
+# </p>
+#
+# <p>
+# Note: lexer rules which use the {@code ->skip} lexer command or call
+# {@link Lexer#skip} do not produce tokens at all, so input text matched by
+# such a rule will not be available as part of the token stream, regardless of
+# channel.</p>
+#/
+
+from .BufferedTokenStream import BufferedTokenStream
+from .Lexer import Lexer
+from .Token import Token
+
+
+class CommonTokenStream(BufferedTokenStream):
+    __slots__ = 'channel'
+
+    def __init__(self, lexer:Lexer, channel:int=Token.DEFAULT_CHANNEL):
+        super().__init__(lexer)
+        self.channel = channel
+
+    def adjustSeekIndex(self, i:int):
+        return self.nextTokenOnChannel(i, self.channel)
+
+    def LB(self, k:int):
+        if k==0 or (self.index-k)<0:
+            return None
+        i = self.index
+        n = 1
+        # find k good tokens looking backwards
+        while n <= k:
+            # skip off-channel tokens
+            i = self.previousTokenOnChannel(i - 1, self.channel)
+            n += 1
+        if i < 0:
+            return None
+        return self.tokens[i]
+
+    def LT(self, k:int):
+        self.lazyInit()
+        if k == 0:
+            return None
+        if k < 0:
+            return self.LB(-k)
+        i = self.index
+        n = 1 # we know tokens[pos] is a good one
+        # find k good tokens
+        while n < k:
+            # skip off-channel tokens, but make sure to not look past EOF
+            if self.sync(i + 1):
+                i = self.nextTokenOnChannel(i + 1, self.channel)
+            n += 1
+        return self.tokens[i]
+
+    # Count EOF just once.#/
+    def getNumberOfOnChannelTokens(self):
+        n = 0
+        self.fill()
+        for i in range(0, len(self.tokens)):
+            t = self.tokens[i]
+            if t.channel==self.channel:
+                n += 1
+            if t.type==Token.EOF:
+                break
+        return n