omlish 0.0.0.dev57__tar.gz → 0.0.0.dev58__tar.gz

{omlish-0.0.0.dev57/omlish.egg-info → omlish-0.0.0.dev58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev57
+Version: 0.0.0.dev58
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev57 → omlish-0.0.0.dev58}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev57'
-__revision__ = '6bc82bd3fa6f9a7e33d9f3a69e8d3e1855aa6417'
+__version__ = '0.0.0.dev58'
+__revision__ = 'aae9a0f7439a053a6e5c2105ff299991d37335ac'
 
 
 #

omlish-0.0.0.dev58/omlish/antlr/_runtime/BufferedTokenStream.py

@@ -0,0 +1,305 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+
+# This implementation of {@link TokenStream} loads tokens from a
+# {@link TokenSource} on-demand, and places the tokens in a buffer to provide
+# access to any previous token by index.
+#
+# <p>
+# This token stream ignores the value of {@link Token#getChannel}. If your
+# parser requires the token stream filter tokens to only those on a particular
+# channel, such as {@link Token#DEFAULT_CHANNEL} or
+# {@link Token#HIDDEN_CHANNEL}, use a filtering token stream such a
+# {@link CommonTokenStream}.</p>
+from io import StringIO
+from .Token import Token
+from .error.Errors import IllegalStateException
+
+# need forward declaration
+Lexer = None
+
+# this is just to keep meaningful parameter types to Parser
+class TokenStream(object):
+
+    pass
+
+
+class BufferedTokenStream(TokenStream):
+    __slots__ = ('tokenSource', 'tokens', 'index', 'fetchedEOF')
+
+    def __init__(self, tokenSource:Lexer):
+        # The {@link TokenSource} from which tokens for this stream are fetched.
+        self.tokenSource = tokenSource
+
+        # A collection of all tokens fetched from the token source. The list is
+        # considered a complete view of the input once {@link #fetchedEOF} is set
+        # to {@code true}.
+        self.tokens = []
+
+        # The index into {@link #tokens} of the current token (next token to
+        # {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be
+        # {@link #LT LT(1)}.
+        #
+        # <p>This field is set to -1 when the stream is first constructed or when
+        # {@link #setTokenSource} is called, indicating that the first token has
+        # not yet been fetched from the token source. For additional information,
+        # see the documentation of {@link IntStream} for a description of
+        # Initializing Methods.</p>
+        self.index = -1
+
+        # Indicates whether the {@link Token#EOF} token has been fetched from
+        # {@link #tokenSource} and added to {@link #tokens}. This field improves
+        # performance for the following cases:
+        #
+        # <ul>
+        # <li>{@link #consume}: The lookahead check in {@link #consume} to prevent
+        # consuming the EOF symbol is optimized by checking the values of
+        # {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.</li>
+        # <li>{@link #fetch}: The check to prevent adding multiple EOF symbols into
+        # {@link #tokens} is trivial with this field.</li>
+        # <ul>
+        self.fetchedEOF = False
+
+    def mark(self):
+        return 0
+
+    def release(self, marker:int):
+        # no resources to release
+        pass
+
+    def reset(self):
+        self.seek(0)
+
+    def seek(self, index:int):
+        self.lazyInit()
+        self.index = self.adjustSeekIndex(index)
+
+    def get(self, index:int):
+        self.lazyInit()
+        return self.tokens[index]
+
+    def consume(self):
+        skipEofCheck = False
+        if self.index >= 0:
+            if self.fetchedEOF:
+                # the last token in tokens is EOF. skip check if p indexes any
+                # fetched token except the last.
+                skipEofCheck = self.index < len(self.tokens) - 1
+            else:
+               # no EOF token in tokens. skip check if p indexes a fetched token.
+                skipEofCheck = self.index < len(self.tokens)
+        else:
+            # not yet initialized
+            skipEofCheck = False
+
+        if not skipEofCheck and self.LA(1) == Token.EOF:
+            raise IllegalStateException("cannot consume EOF")
+
+        if self.sync(self.index + 1):
+            self.index = self.adjustSeekIndex(self.index + 1)
+
+    # Make sure index {@code i} in tokens has a token.
+    #
+    # @return {@code true} if a token is located at index {@code i}, otherwise
+    #    {@code false}.
+    # @see #get(int i)
+    #/
+    def sync(self, i:int):
+        n = i - len(self.tokens) + 1 # how many more elements we need?
+        if n > 0 :
+            fetched = self.fetch(n)
+            return fetched >= n
+        return True
+
+    # Add {@code n} elements to buffer.
+    #
+    # @return The actual number of elements added to the buffer.
+    #/
+    def fetch(self, n:int):
+        if self.fetchedEOF:
+            return 0
+        for i in range(0, n):
+            t = self.tokenSource.nextToken()
+            t.tokenIndex = len(self.tokens)
+            self.tokens.append(t)
+            if t.type==Token.EOF:
+                self.fetchedEOF = True
+                return i + 1
+        return n
+
+
+    # Get all tokens from start..stop inclusively#/
+    def getTokens(self, start:int, stop:int, types:set=None):
+        if start<0 or stop<0:
+            return None
+        self.lazyInit()
+        subset = []
+        if stop >= len(self.tokens):
+            stop = len(self.tokens)-1
+        for i in range(start, stop):
+            t = self.tokens[i]
+            if t.type==Token.EOF:
+                break
+            if types is None or t.type in types:
+                subset.append(t)
+        return subset
+
+    def LA(self, i:int):
+        return self.LT(i).type
+
+    def LB(self, k:int):
+        if (self.index-k) < 0:
+            return None
+        return self.tokens[self.index-k]
+
+    def LT(self, k:int):
+        self.lazyInit()
+        if k==0:
+            return None
+        if k < 0:
+            return self.LB(-k)
+        i = self.index + k - 1
+        self.sync(i)
+        if i >= len(self.tokens): # return EOF token
+            # EOF must be last token
+            return self.tokens[len(self.tokens)-1]
+        return self.tokens[i]
+
+    # Allowed derived classes to modify the behavior of operations which change
+    # the current stream position by adjusting the target token index of a seek
+    # operation. The default implementation simply returns {@code i}. If an
+    # exception is thrown in this method, the current stream index should not be
+    # changed.
+    #
+    # <p>For example, {@link CommonTokenStream} overrides this method to ensure that
+    # the seek target is always an on-channel token.</p>
+    #
+    # @param i The target token index.
+    # @return The adjusted target token index.
+
+    def adjustSeekIndex(self, i:int):
+        return i
+
+    def lazyInit(self):
+        if self.index == -1:
+            self.setup()
+
+    def setup(self):
+        self.sync(0)
+        self.index = self.adjustSeekIndex(0)
+
+    # Reset this token stream by setting its token source.#/
+    def setTokenSource(self, tokenSource:Lexer):
+        self.tokenSource = tokenSource
+        self.tokens = []
+        self.index = -1
+        self.fetchedEOF = False
+
+
+    # Given a starting index, return the index of the next token on channel.
+    #  Return i if tokens[i] is on channel.  Return the index of the EOF token
+    # if there are no tokens on channel between i and EOF.
+    #/
+    def nextTokenOnChannel(self, i:int, channel:int):
+        self.sync(i)
+        if i>=len(self.tokens):
+            return len(self.tokens) - 1
+        token = self.tokens[i]
+        while token.channel!=channel:
+            if token.type==Token.EOF:
+                return i
+            i += 1
+            self.sync(i)
+            token = self.tokens[i]
+        return i
+
+    # Given a starting index, return the index of the previous token on channel.
+    #  Return i if tokens[i] is on channel. Return -1 if there are no tokens
+    #  on channel between i and 0.
+    def previousTokenOnChannel(self, i:int, channel:int):
+        while i>=0 and self.tokens[i].channel!=channel:
+            i -= 1
+        return i
+
+    # Collect all tokens on specified channel to the right of
+    #  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
+    #  EOF. If channel is -1, find any non default channel token.
+    def getHiddenTokensToRight(self, tokenIndex:int, channel:int=-1):
+        self.lazyInit()
+        if tokenIndex<0 or tokenIndex>=len(self.tokens):
+            raise Exception(str(tokenIndex) + " not in 0.." + str(len(self.tokens)-1))
+        from .Lexer import Lexer
+        nextOnChannel = self.nextTokenOnChannel(tokenIndex + 1, Lexer.DEFAULT_TOKEN_CHANNEL)
+        from_ = tokenIndex+1
+        # if none onchannel to right, nextOnChannel=-1 so set to = last token
+        to = (len(self.tokens)-1) if nextOnChannel==-1 else nextOnChannel
+        return self.filterForChannel(from_, to, channel)
+
+
+    # Collect all tokens on specified channel to the left of
+    #  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
+    #  If channel is -1, find any non default channel token.
+    def getHiddenTokensToLeft(self, tokenIndex:int, channel:int=-1):
+        self.lazyInit()
+        if tokenIndex<0 or tokenIndex>=len(self.tokens):
+            raise Exception(str(tokenIndex) + " not in 0.." + str(len(self.tokens)-1))
+        from .Lexer import Lexer
+        prevOnChannel = self.previousTokenOnChannel(tokenIndex - 1, Lexer.DEFAULT_TOKEN_CHANNEL)
+        if prevOnChannel == tokenIndex - 1:
+            return None
+        # if none on channel to left, prevOnChannel=-1 then from=0
+        from_ = prevOnChannel+1
+        to = tokenIndex-1
+        return self.filterForChannel(from_, to, channel)
+
+
+    def filterForChannel(self, left:int, right:int, channel:int):
+        hidden = []
+        for i in range(left, right+1):
+            t = self.tokens[i]
+            if channel==-1:
+                from .Lexer import Lexer
+                if t.channel!= Lexer.DEFAULT_TOKEN_CHANNEL:
+                    hidden.append(t)
+            elif t.channel==channel:
+                    hidden.append(t)
+        if len(hidden)==0:
+            return None
+        return hidden
+
+    def getSourceName(self):
+        return self.tokenSource.getSourceName()
+
+    # Get the text of all tokens in this buffer.#/
+    def getText(self, start:int=None, stop:int=None):
+        self.lazyInit()
+        self.fill()
+        if isinstance(start, Token):
+            start = start.tokenIndex
+        elif start is None:
+            start = 0
+        if isinstance(stop, Token):
+            stop = stop.tokenIndex
+        elif stop is None or stop >= len(self.tokens):
+            stop = len(self.tokens) - 1
+        if start < 0 or stop < 0 or stop < start:
+            return ""
+        with StringIO() as buf:
+            for i in range(start, stop+1):
+                t = self.tokens[i]
+                if t.type==Token.EOF:
+                    break
+                buf.write(t.text)
+            return buf.getvalue()
+
+
+    # Get all tokens from lexer until EOF#/
+    def fill(self):
+        self.lazyInit()
+        while self.fetch(1000)==1000:
+            pass

omlish-0.0.0.dev58/omlish/antlr/_runtime/CommonTokenFactory.py

@@ -0,0 +1,64 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#
+
+#
+# This default implementation of {@link TokenFactory} creates
+# {@link CommonToken} objects.
+#
+from .Token import CommonToken
+
+class TokenFactory(object):
+
+    pass
+
+class CommonTokenFactory(TokenFactory):
+    __slots__ = 'copyText'
+
+    #
+    # The default {@link CommonTokenFactory} instance.
+    #
+    # <p>
+    # This token factory does not explicitly copy token text when constructing
+    # tokens.</p>
+    #
+    DEFAULT = None
+
+    def __init__(self, copyText:bool=False):
+        # Indicates whether {@link CommonToken#setText} should be called after
+        # constructing tokens to explicitly set the text. This is useful for cases
+        # where the input stream might not be able to provide arbitrary substrings
+        # of text from the input after the lexer creates a token (e.g. the
+        # implementation of {@link CharStream#getText} in
+        # {@link UnbufferedCharStream} throws an
+        # {@link UnsupportedOperationException}). Explicitly setting the token text
+        # allows {@link Token#getText} to be called at any time regardless of the
+        # input stream implementation.
+        #
+        # <p>
+        # The default value is {@code false} to avoid the performance and memory
+        # overhead of copying text for every token unless explicitly requested.</p>
+        #
+        self.copyText = copyText
+
+    def create(self, source, type:int, text:str, channel:int, start:int, stop:int, line:int, column:int):
+        t = CommonToken(source, type, channel, start, stop)
+        t.line = line
+        t.column = column
+        if text is not None:
+            t.text = text
+        elif self.copyText and source[1] is not None:
+            t.text = source[1].getText(start,stop)
+        return t
+
+    def createThin(self, type:int, text:str):
+        t = CommonToken(type=type)
+        t.text = text
+        return t
+
+CommonTokenFactory.DEFAULT = CommonTokenFactory()

omlish-0.0.0.dev58/omlish/antlr/_runtime/CommonTokenStream.py

@@ -0,0 +1,90 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#/
+
+#
+# This class extends {@link BufferedTokenStream} with functionality to filter
+# token streams to tokens on a particular channel (tokens where
+# {@link Token#getChannel} returns a particular value).
+#
+# <p>
+# This token stream provides access to all tokens by index or when calling
+# methods like {@link #getText}. The channel filtering is only used for code
+# accessing tokens via the lookahead methods {@link #LA}, {@link #LT}, and
+# {@link #LB}.</p>
+#
+# <p>
+# By default, tokens are placed on the default channel
+# ({@link Token#DEFAULT_CHANNEL}), but may be reassigned by using the
+# {@code ->channel(HIDDEN)} lexer command, or by using an embedded action to
+# call {@link Lexer#setChannel}.
+# </p>
+#
+# <p>
+# Note: lexer rules which use the {@code ->skip} lexer command or call
+# {@link Lexer#skip} do not produce tokens at all, so input text matched by
+# such a rule will not be available as part of the token stream, regardless of
+# channel.</p>
+#/
+
+from .BufferedTokenStream import BufferedTokenStream
+from .Lexer import Lexer
+from .Token import Token
+
+
+class CommonTokenStream(BufferedTokenStream):
+    __slots__ = 'channel'
+
+    def __init__(self, lexer:Lexer, channel:int=Token.DEFAULT_CHANNEL):
+        super().__init__(lexer)
+        self.channel = channel
+
+    def adjustSeekIndex(self, i:int):
+        return self.nextTokenOnChannel(i, self.channel)
+
+    def LB(self, k:int):
+        if k==0 or (self.index-k)<0:
+            return None
+        i = self.index
+        n = 1
+        # find k good tokens looking backwards
+        while n <= k:
+            # skip off-channel tokens
+            i = self.previousTokenOnChannel(i - 1, self.channel)
+            n += 1
+        if i < 0:
+            return None
+        return self.tokens[i]
+
+    def LT(self, k:int):
+        self.lazyInit()
+        if k == 0:
+            return None
+        if k < 0:
+            return self.LB(-k)
+        i = self.index
+        n = 1 # we know tokens[pos] is a good one
+        # find k good tokens
+        while n < k:
+            # skip off-channel tokens, but make sure to not look past EOF
+            if self.sync(i + 1):
+                i = self.nextTokenOnChannel(i + 1, self.channel)
+            n += 1
+        return self.tokens[i]
+
+    # Count EOF just once.#/
+    def getNumberOfOnChannelTokens(self):
+        n = 0
+        self.fill()
+        for i in range(0, len(self.tokens)):
+            t = self.tokens[i]
+            if t.channel==self.channel:
+                n += 1
+            if t.type==Token.EOF:
+                break
+        return n

omlish-0.0.0.dev58/omlish/antlr/_runtime/FileStream.py

@@ -0,0 +1,30 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#
+
+#
+#  This is an InputStream that is loaded from a file all at once
+#  when you construct the object.
+#
+
+import codecs
+from .InputStream import InputStream
+
+
+class FileStream(InputStream):
+    __slots__ = 'fileName'
+
+    def __init__(self, fileName:str, encoding:str='ascii', errors:str='strict'):
+        super().__init__(self.readDataFrom(fileName, encoding, errors))
+        self.fileName = fileName
+
+    def readDataFrom(self, fileName:str, encoding:str, errors:str='strict'):
+        # read binary to avoid line ending conversion
+        with open(fileName, 'rb') as file:
+            bytes = file.read()
+            return codecs.decode(bytes, encoding, errors)

omlish-0.0.0.dev58/omlish/antlr/_runtime/InputStream.py

@@ -0,0 +1,90 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+#
+# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+# Use of this file is governed by the BSD 3-clause license that
+# can be found in the LICENSE.txt file in the project root.
+#
+
+
+#
+#  Vacuum all input from a string and then treat it like a buffer.
+#
+from .Token import Token
+
+
+class InputStream (object):
+    __slots__ = ('name', 'strdata', '_index', 'data', '_size')
+
+    def __init__(self, data: str):
+        self.name = "<empty>"
+        self.strdata = data
+        self._loadString()
+
+    def _loadString(self):
+        self._index = 0
+        self.data = [ord(c) for c in self.strdata]
+        self._size = len(self.data)
+
+    @property
+    def index(self):
+        return self._index
+
+    @property
+    def size(self):
+        return self._size
+
+    # Reset the stream so that it's in the same state it was
+    #  when the object was created *except* the data array is not
+    #  touched.
+    #
+    def reset(self):
+        self._index = 0
+
+    def consume(self):
+        if self._index >= self._size:
+            assert self.LA(1) == Token.EOF
+            raise Exception("cannot consume EOF")
+        self._index += 1
+
+    def LA(self, offset: int):
+        if offset==0:
+            return 0 # undefined
+        if offset<0:
+            offset += 1 # e.g., translate LA(-1) to use offset=0
+        pos = self._index + offset - 1
+        if pos < 0 or pos >= self._size: # invalid
+            return Token.EOF
+        return self.data[pos]
+
+    def LT(self, offset: int):
+        return self.LA(offset)
+
+    # mark/release do nothing; we have entire buffer
+    def mark(self):
+        return -1
+
+    def release(self, marker: int):
+        pass
+
+    # consume() ahead until p==_index; can't just set p=_index as we must
+    # update line and column. If we seek backwards, just set p
+    #
+    def seek(self, _index: int):
+        if _index<=self._index:
+            self._index = _index # just jump; don't update stream state (line, ...)
+            return
+        # seek forward
+        self._index = min(_index, self._size)
+
+    def getText(self, start :int, stop: int):
+        if stop >= self._size:
+            stop = self._size-1
+        if start >= self._size:
+            return ""
+        else:
+            return self.strdata[start:stop+1]
+
+    def __str__(self):
+        return self.strdata