omlish 0.0.0.dev56__py3-none-any.whl → 0.0.0.dev58__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. omlish/__about__.py +2 -2
  2. omlish/antlr/__init__.py +0 -0
  3. omlish/antlr/_runtime/BufferedTokenStream.py +305 -0
  4. omlish/antlr/_runtime/CommonTokenFactory.py +64 -0
  5. omlish/antlr/_runtime/CommonTokenStream.py +90 -0
  6. omlish/antlr/_runtime/FileStream.py +30 -0
  7. omlish/antlr/_runtime/InputStream.py +90 -0
  8. omlish/antlr/_runtime/IntervalSet.py +183 -0
  9. omlish/antlr/_runtime/LL1Analyzer.py +176 -0
  10. omlish/antlr/_runtime/Lexer.py +332 -0
  11. omlish/antlr/_runtime/ListTokenSource.py +147 -0
  12. omlish/antlr/_runtime/Parser.py +583 -0
  13. omlish/antlr/_runtime/ParserInterpreter.py +173 -0
  14. omlish/antlr/_runtime/ParserRuleContext.py +189 -0
  15. omlish/antlr/_runtime/PredictionContext.py +632 -0
  16. omlish/antlr/_runtime/Recognizer.py +150 -0
  17. omlish/antlr/_runtime/RuleContext.py +230 -0
  18. omlish/antlr/_runtime/StdinStream.py +14 -0
  19. omlish/antlr/_runtime/Token.py +158 -0
  20. omlish/antlr/_runtime/TokenStreamRewriter.py +258 -0
  21. omlish/antlr/_runtime/Utils.py +36 -0
  22. omlish/antlr/_runtime/__init__.py +24 -0
  23. omlish/antlr/_runtime/_pygrun.py +174 -0
  24. omlish/antlr/_runtime/atn/ATN.py +135 -0
  25. omlish/antlr/_runtime/atn/ATNConfig.py +162 -0
  26. omlish/antlr/_runtime/atn/ATNConfigSet.py +215 -0
  27. omlish/antlr/_runtime/atn/ATNDeserializationOptions.py +27 -0
  28. omlish/antlr/_runtime/atn/ATNDeserializer.py +449 -0
  29. omlish/antlr/_runtime/atn/ATNSimulator.py +50 -0
  30. omlish/antlr/_runtime/atn/ATNState.py +267 -0
  31. omlish/antlr/_runtime/atn/ATNType.py +20 -0
  32. omlish/antlr/_runtime/atn/LexerATNSimulator.py +573 -0
  33. omlish/antlr/_runtime/atn/LexerAction.py +301 -0
  34. omlish/antlr/_runtime/atn/LexerActionExecutor.py +146 -0
  35. omlish/antlr/_runtime/atn/ParserATNSimulator.py +1664 -0
  36. omlish/antlr/_runtime/atn/PredictionMode.py +502 -0
  37. omlish/antlr/_runtime/atn/SemanticContext.py +333 -0
  38. omlish/antlr/_runtime/atn/Transition.py +271 -0
  39. omlish/antlr/_runtime/atn/__init__.py +4 -0
  40. omlish/antlr/_runtime/dfa/DFA.py +136 -0
  41. omlish/antlr/_runtime/dfa/DFASerializer.py +76 -0
  42. omlish/antlr/_runtime/dfa/DFAState.py +129 -0
  43. omlish/antlr/_runtime/dfa/__init__.py +4 -0
  44. omlish/antlr/_runtime/error/DiagnosticErrorListener.py +110 -0
  45. omlish/antlr/_runtime/error/ErrorListener.py +75 -0
  46. omlish/antlr/_runtime/error/ErrorStrategy.py +712 -0
  47. omlish/antlr/_runtime/error/Errors.py +176 -0
  48. omlish/antlr/_runtime/error/__init__.py +4 -0
  49. omlish/antlr/_runtime/tree/Chunk.py +33 -0
  50. omlish/antlr/_runtime/tree/ParseTreeMatch.py +121 -0
  51. omlish/antlr/_runtime/tree/ParseTreePattern.py +75 -0
  52. omlish/antlr/_runtime/tree/ParseTreePatternMatcher.py +377 -0
  53. omlish/antlr/_runtime/tree/RuleTagToken.py +53 -0
  54. omlish/antlr/_runtime/tree/TokenTagToken.py +50 -0
  55. omlish/antlr/_runtime/tree/Tree.py +194 -0
  56. omlish/antlr/_runtime/tree/Trees.py +114 -0
  57. omlish/antlr/_runtime/tree/__init__.py +2 -0
  58. omlish/antlr/_runtime/xpath/XPath.py +272 -0
  59. omlish/antlr/_runtime/xpath/XPathLexer.py +98 -0
  60. omlish/antlr/_runtime/xpath/__init__.py +4 -0
  61. omlish/formats/json/cli.py +76 -7
  62. omlish/formats/props.py +6 -2
  63. {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/METADATA +1 -1
  64. {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/RECORD +68 -9
  65. {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/LICENSE +0 -0
  66. {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/WHEEL +0 -0
  67. {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/entry_points.txt +0 -0
  68. {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,183 @@
1
+ # type: ignore
2
+ # ruff: noqa
3
+ # flake8: noqa
4
+ #
5
+ # Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
6
+ # Use of this file is governed by the BSD 3-clause license that
7
+ # can be found in the LICENSE.txt file in the project root.
8
+ #
9
+
10
+ from io import StringIO
11
+ from .Token import Token
12
+
13
+ # need forward declarations
14
+ IntervalSet = None
15
+
16
+ class IntervalSet(object):
17
+ __slots__ = ('intervals', 'readonly')
18
+
19
+ def __init__(self):
20
+ self.intervals = None
21
+ self.readonly = False
22
+
23
+ def __iter__(self):
24
+ if self.intervals is not None:
25
+ for i in self.intervals:
26
+ for c in i:
27
+ yield c
28
+
29
+ def __getitem__(self, item):
30
+ i = 0
31
+ for k in self:
32
+ if i==item:
33
+ return k
34
+ else:
35
+ i += 1
36
+ return Token.INVALID_TYPE
37
+
38
+ def addOne(self, v:int):
39
+ self.addRange(range(v, v+1))
40
+
41
+ def addRange(self, v:range):
42
+ if self.intervals is None:
43
+ self.intervals = list()
44
+ self.intervals.append(v)
45
+ else:
46
+ # find insert pos
47
+ k = 0
48
+ for i in self.intervals:
49
+ # distinct range -> insert
50
+ if v.stop<i.start:
51
+ self.intervals.insert(k, v)
52
+ return
53
+ # contiguous range -> adjust
54
+ elif v.stop==i.start:
55
+ self.intervals[k] = range(v.start, i.stop)
56
+ return
57
+ # overlapping range -> adjust and reduce
58
+ elif v.start<=i.stop:
59
+ self.intervals[k] = range(min(i.start,v.start), max(i.stop,v.stop))
60
+ self.reduce(k)
61
+ return
62
+ k += 1
63
+ # greater than any existing
64
+ self.intervals.append(v)
65
+
66
+ def addSet(self, other:IntervalSet):
67
+ if other.intervals is not None:
68
+ for i in other.intervals:
69
+ self.addRange(i)
70
+ return self
71
+
72
+ def reduce(self, k:int):
73
+ # only need to reduce if k is not the last
74
+ if k<len(self.intervals)-1:
75
+ l = self.intervals[k]
76
+ r = self.intervals[k+1]
77
+ # if r contained in l
78
+ if l.stop >= r.stop:
79
+ self.intervals.pop(k+1)
80
+ self.reduce(k)
81
+ elif l.stop >= r.start:
82
+ self.intervals[k] = range(l.start, r.stop)
83
+ self.intervals.pop(k+1)
84
+
85
+ def complement(self, start, stop):
86
+ result = IntervalSet()
87
+ result.addRange(range(start,stop+1))
88
+ for i in self.intervals:
89
+ result.removeRange(i)
90
+ return result
91
+
92
+ def __contains__(self, item):
93
+ if self.intervals is None:
94
+ return False
95
+ else:
96
+ return any(item in i for i in self.intervals)
97
+
98
+ def __len__(self):
99
+ return sum(len(i) for i in self.intervals)
100
+
101
+ def removeRange(self, v):
102
+ if v.start==v.stop-1:
103
+ self.removeOne(v.start)
104
+ elif self.intervals is not None:
105
+ k = 0
106
+ for i in self.intervals:
107
+ # intervals are ordered
108
+ if v.stop<=i.start:
109
+ return
110
+ # check for including range, split it
111
+ elif v.start>i.start and v.stop<i.stop:
112
+ self.intervals[k] = range(i.start, v.start)
113
+ x = range(v.stop, i.stop)
114
+ self.intervals.insert(k, x)
115
+ return
116
+ # check for included range, remove it
117
+ elif v.start<=i.start and v.stop>=i.stop:
118
+ self.intervals.pop(k)
119
+ k -= 1 # need another pass
120
+ # check for lower boundary
121
+ elif v.start<i.stop:
122
+ self.intervals[k] = range(i.start, v.start)
123
+ # check for upper boundary
124
+ elif v.stop<i.stop:
125
+ self.intervals[k] = range(v.stop, i.stop)
126
+ k += 1
127
+
128
+ def removeOne(self, v):
129
+ if self.intervals is not None:
130
+ k = 0
131
+ for i in self.intervals:
132
+ # intervals is ordered
133
+ if v<i.start:
134
+ return
135
+ # check for single value range
136
+ elif v==i.start and v==i.stop-1:
137
+ self.intervals.pop(k)
138
+ return
139
+ # check for lower boundary
140
+ elif v==i.start:
141
+ self.intervals[k] = range(i.start+1, i.stop)
142
+ return
143
+ # check for upper boundary
144
+ elif v==i.stop-1:
145
+ self.intervals[k] = range(i.start, i.stop-1)
146
+ return
147
+ # split existing range
148
+ elif v<i.stop-1:
149
+ x = range(i.start, v)
150
+ self.intervals[k] = range(v + 1, i.stop)
151
+ self.intervals.insert(k, x)
152
+ return
153
+ k += 1
154
+
155
+
156
+ def toString(self, literalNames:list, symbolicNames:list):
157
+ if self.intervals is None:
158
+ return "{}"
159
+ with StringIO() as buf:
160
+ if len(self)>1:
161
+ buf.write("{")
162
+ first = True
163
+ for i in self.intervals:
164
+ for j in i:
165
+ if not first:
166
+ buf.write(", ")
167
+ buf.write(self.elementName(literalNames, symbolicNames, j))
168
+ first = False
169
+ if len(self)>1:
170
+ buf.write("}")
171
+ return buf.getvalue()
172
+
173
+ def elementName(self, literalNames:list, symbolicNames:list, a:int):
174
+ if a==Token.EOF:
175
+ return "<EOF>"
176
+ elif a==Token.EPSILON:
177
+ return "<EPSILON>"
178
+ else:
179
+ if a<len(literalNames) and literalNames[a] != "<INVALID>":
180
+ return literalNames[a]
181
+ if a<len(symbolicNames):
182
+ return symbolicNames[a]
183
+ return "<UNKNOWN>"
@@ -0,0 +1,176 @@
1
+ # type: ignore
2
+ # ruff: noqa
3
+ # flake8: noqa
4
+ #
5
+ # Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
6
+ # Use of this file is governed by the BSD 3-clause license that
7
+ # can be found in the LICENSE.txt file in the project root.
8
+ #/
9
+ from .IntervalSet import IntervalSet
10
+ from .Token import Token
11
+ from .PredictionContext import PredictionContext, SingletonPredictionContext, PredictionContextFromRuleContext
12
+ from .RuleContext import RuleContext
13
+ from .atn.ATN import ATN
14
+ from .atn.ATNConfig import ATNConfig
15
+ from .atn.ATNState import ATNState, RuleStopState
16
+ from .atn.Transition import WildcardTransition, NotSetTransition, AbstractPredicateTransition, RuleTransition
17
+
18
+
19
+ class LL1Analyzer (object):
20
+ __slots__ = 'atn'
21
+
22
+ #* Special value added to the lookahead sets to indicate that we hit
23
+ # a predicate during analysis if {@code seeThruPreds==false}.
24
+ #/
25
+ HIT_PRED = Token.INVALID_TYPE
26
+
27
+ def __init__(self, atn:ATN):
28
+ self.atn = atn
29
+
30
+ #*
31
+ # Calculates the SLL(1) expected lookahead set for each outgoing transition
32
+ # of an {@link ATNState}. The returned array has one element for each
33
+ # outgoing transition in {@code s}. If the closure from transition
34
+ # <em>i</em> leads to a semantic predicate before matching a symbol, the
35
+ # element at index <em>i</em> of the result will be {@code null}.
36
+ #
37
+ # @param s the ATN state
38
+ # @return the expected symbols for each outgoing transition of {@code s}.
39
+ #/
40
+ def getDecisionLookahead(self, s:ATNState):
41
+ if s is None:
42
+ return None
43
+
44
+ count = len(s.transitions)
45
+ look = [] * count
46
+ for alt in range(0, count):
47
+ look[alt] = set()
48
+ lookBusy = set()
49
+ seeThruPreds = False # fail to get lookahead upon pred
50
+ self._LOOK(s.transition(alt).target, None, PredictionContext.EMPTY,
51
+ look[alt], lookBusy, set(), seeThruPreds, False)
52
+ # Wipe out lookahead for this alternative if we found nothing
53
+ # or we had a predicate when we !seeThruPreds
54
+ if len(look[alt])==0 or self.HIT_PRED in look[alt]:
55
+ look[alt] = None
56
+ return look
57
+
58
+ #*
59
+ # Compute set of tokens that can follow {@code s} in the ATN in the
60
+ # specified {@code ctx}.
61
+ #
62
+ # <p>If {@code ctx} is {@code null} and the end of the rule containing
63
+ # {@code s} is reached, {@link Token#EPSILON} is added to the result set.
64
+ # If {@code ctx} is not {@code null} and the end of the outermost rule is
65
+ # reached, {@link Token#EOF} is added to the result set.</p>
66
+ #
67
+ # @param s the ATN state
68
+ # @param stopState the ATN state to stop at. This can be a
69
+ # {@link BlockEndState} to detect epsilon paths through a closure.
70
+ # @param ctx the complete parser context, or {@code null} if the context
71
+ # should be ignored
72
+ #
73
+ # @return The set of tokens that can follow {@code s} in the ATN in the
74
+ # specified {@code ctx}.
75
+ #/
76
+ def LOOK(self, s:ATNState, stopState:ATNState=None, ctx:RuleContext=None):
77
+ r = IntervalSet()
78
+ seeThruPreds = True # ignore preds; get all lookahead
79
+ lookContext = PredictionContextFromRuleContext(s.atn, ctx) if ctx is not None else None
80
+ self._LOOK(s, stopState, lookContext, r, set(), set(), seeThruPreds, True)
81
+ return r
82
+
83
+ #*
84
+ # Compute set of tokens that can follow {@code s} in the ATN in the
85
+ # specified {@code ctx}.
86
+ #
87
+ # <p>If {@code ctx} is {@code null} and {@code stopState} or the end of the
88
+ # rule containing {@code s} is reached, {@link Token#EPSILON} is added to
89
+ # the result set. If {@code ctx} is not {@code null} and {@code addEOF} is
90
+ # {@code true} and {@code stopState} or the end of the outermost rule is
91
+ # reached, {@link Token#EOF} is added to the result set.</p>
92
+ #
93
+ # @param s the ATN state.
94
+ # @param stopState the ATN state to stop at. This can be a
95
+ # {@link BlockEndState} to detect epsilon paths through a closure.
96
+ # @param ctx The outer context, or {@code null} if the outer context should
97
+ # not be used.
98
+ # @param look The result lookahead set.
99
+ # @param lookBusy A set used for preventing epsilon closures in the ATN
100
+ # from causing a stack overflow. Outside code should pass
101
+ # {@code new HashSet<ATNConfig>} for this argument.
102
+ # @param calledRuleStack A set used for preventing left recursion in the
103
+ # ATN from causing a stack overflow. Outside code should pass
104
+ # {@code new BitSet()} for this argument.
105
+ # @param seeThruPreds {@code true} to true semantic predicates as
106
+ # implicitly {@code true} and "see through them", otherwise {@code false}
107
+ # to treat semantic predicates as opaque and add {@link #HIT_PRED} to the
108
+ # result if one is encountered.
109
+ # @param addEOF Add {@link Token#EOF} to the result if the end of the
110
+ # outermost context is reached. This parameter has no effect if {@code ctx}
111
+ # is {@code null}.
112
+ #/
113
+ def _LOOK(self, s:ATNState, stopState:ATNState , ctx:PredictionContext, look:IntervalSet, lookBusy:set,
114
+ calledRuleStack:set, seeThruPreds:bool, addEOF:bool):
115
+ c = ATNConfig(s, 0, ctx)
116
+
117
+ if c in lookBusy:
118
+ return
119
+ lookBusy.add(c)
120
+
121
+ if s == stopState:
122
+ if ctx is None:
123
+ look.addOne(Token.EPSILON)
124
+ return
125
+ elif ctx.isEmpty() and addEOF:
126
+ look.addOne(Token.EOF)
127
+ return
128
+
129
+ if isinstance(s, RuleStopState ):
130
+ if ctx is None:
131
+ look.addOne(Token.EPSILON)
132
+ return
133
+ elif ctx.isEmpty() and addEOF:
134
+ look.addOne(Token.EOF)
135
+ return
136
+
137
+ if ctx != PredictionContext.EMPTY:
138
+ removed = s.ruleIndex in calledRuleStack
139
+ try:
140
+ calledRuleStack.discard(s.ruleIndex)
141
+ # run thru all possible stack tops in ctx
142
+ for i in range(0, len(ctx)):
143
+ returnState = self.atn.states[ctx.getReturnState(i)]
144
+ self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
145
+ finally:
146
+ if removed:
147
+ calledRuleStack.add(s.ruleIndex)
148
+ return
149
+
150
+ for t in s.transitions:
151
+ if type(t) == RuleTransition:
152
+ if t.target.ruleIndex in calledRuleStack:
153
+ continue
154
+
155
+ newContext = SingletonPredictionContext.create(ctx, t.followState.stateNumber)
156
+
157
+ try:
158
+ calledRuleStack.add(t.target.ruleIndex)
159
+ self._LOOK(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
160
+ finally:
161
+ calledRuleStack.remove(t.target.ruleIndex)
162
+ elif isinstance(t, AbstractPredicateTransition ):
163
+ if seeThruPreds:
164
+ self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
165
+ else:
166
+ look.addOne(self.HIT_PRED)
167
+ elif t.isEpsilon:
168
+ self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
169
+ elif type(t) == WildcardTransition:
170
+ look.addRange( range(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) )
171
+ else:
172
+ set_ = t.label
173
+ if set_ is not None:
174
+ if isinstance(t, NotSetTransition):
175
+ set_ = set_.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
176
+ look.addSet(set_)
@@ -0,0 +1,332 @@
1
+ # type: ignore
2
+ # ruff: noqa
3
+ # flake8: noqa
4
+ # Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
5
+ # Use of this file is governed by the BSD 3-clause license that
6
+ # can be found in the LICENSE.txt file in the project root.
7
+ #/
8
+
9
+ # A lexer is recognizer that draws input symbols from a character stream.
10
+ # lexer grammars result in a subclass of self object. A Lexer object
11
+ # uses simplified match() and error recovery mechanisms in the interest
12
+ # of speed.
13
+ #/
14
+ from io import StringIO
15
+
16
+ import sys
17
+ if sys.version_info[1] > 5:
18
+ from typing import TextIO
19
+ else:
20
+ from typing.io import TextIO
21
+ from .CommonTokenFactory import CommonTokenFactory
22
+ from .atn.LexerATNSimulator import LexerATNSimulator
23
+ from .InputStream import InputStream
24
+ from .Recognizer import Recognizer
25
+ from .Token import Token
26
+ from .error.Errors import IllegalStateException, LexerNoViableAltException, RecognitionException
27
+
28
+ class TokenSource(object):
29
+
30
+ pass
31
+
32
+
33
+ class Lexer(Recognizer, TokenSource):
34
+ __slots__ = (
35
+ '_input', '_output', '_factory', '_tokenFactorySourcePair', '_token',
36
+ '_tokenStartCharIndex', '_tokenStartLine', '_tokenStartColumn',
37
+ '_hitEOF', '_channel', '_type', '_modeStack', '_mode', '_text'
38
+ )
39
+
40
+ DEFAULT_MODE = 0
41
+ MORE = -2
42
+ SKIP = -3
43
+
44
+ DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL
45
+ HIDDEN = Token.HIDDEN_CHANNEL
46
+ MIN_CHAR_VALUE = 0x0000
47
+ MAX_CHAR_VALUE = 0x10FFFF
48
+
49
+ def __init__(self, input:InputStream, output:TextIO = sys.stdout):
50
+ super().__init__()
51
+ self._input = input
52
+ self._output = output
53
+ self._factory = CommonTokenFactory.DEFAULT
54
+ self._tokenFactorySourcePair = (self, input)
55
+
56
+ self._interp = None # child classes must populate this
57
+
58
+ # The goal of all lexer rules/methods is to create a token object.
59
+ # self is an instance variable as multiple rules may collaborate to
60
+ # create a single token. nextToken will return self object after
61
+ # matching lexer rule(s). If you subclass to allow multiple token
62
+ # emissions, then set self to the last token to be matched or
63
+ # something nonnull so that the auto token emit mechanism will not
64
+ # emit another token.
65
+ self._token = None
66
+
67
+ # What character index in the stream did the current token start at?
68
+ # Needed, for example, to get the text for current token. Set at
69
+ # the start of nextToken.
70
+ self._tokenStartCharIndex = -1
71
+
72
+ # The line on which the first character of the token resides#/
73
+ self._tokenStartLine = -1
74
+
75
+ # The character position of first character within the line#/
76
+ self._tokenStartColumn = -1
77
+
78
+ # Once we see EOF on char stream, next token will be EOF.
79
+ # If you have DONE : EOF ; then you see DONE EOF.
80
+ self._hitEOF = False
81
+
82
+ # The channel number for the current token#/
83
+ self._channel = Token.DEFAULT_CHANNEL
84
+
85
+ # The token type for the current token#/
86
+ self._type = Token.INVALID_TYPE
87
+
88
+ self._modeStack = []
89
+ self._mode = self.DEFAULT_MODE
90
+
91
+ # You can set the text for the current token to override what is in
92
+ # the input char buffer. Use setText() or can set self instance var.
93
+ #/
94
+ self._text = None
95
+
96
+
97
+ def reset(self):
98
+ # wack Lexer state variables
99
+ if self._input is not None:
100
+ self._input.seek(0) # rewind the input
101
+ self._token = None
102
+ self._type = Token.INVALID_TYPE
103
+ self._channel = Token.DEFAULT_CHANNEL
104
+ self._tokenStartCharIndex = -1
105
+ self._tokenStartColumn = -1
106
+ self._tokenStartLine = -1
107
+ self._text = None
108
+
109
+ self._hitEOF = False
110
+ self._mode = Lexer.DEFAULT_MODE
111
+ self._modeStack = []
112
+
113
+ self._interp.reset()
114
+
115
+ # Return a token from self source; i.e., match a token on the char
116
+ # stream.
117
+ def nextToken(self):
118
+ if self._input is None:
119
+ raise IllegalStateException("nextToken requires a non-null input stream.")
120
+
121
+ # Mark start location in char stream so unbuffered streams are
122
+ # guaranteed at least have text of current token
123
+ tokenStartMarker = self._input.mark()
124
+ try:
125
+ while True:
126
+ if self._hitEOF:
127
+ self.emitEOF()
128
+ return self._token
129
+ self._token = None
130
+ self._channel = Token.DEFAULT_CHANNEL
131
+ self._tokenStartCharIndex = self._input.index
132
+ self._tokenStartColumn = self._interp.column
133
+ self._tokenStartLine = self._interp.line
134
+ self._text = None
135
+ continueOuter = False
136
+ while True:
137
+ self._type = Token.INVALID_TYPE
138
+ ttype = self.SKIP
139
+ try:
140
+ ttype = self._interp.match(self._input, self._mode)
141
+ except LexerNoViableAltException as e:
142
+ self.notifyListeners(e) # report error
143
+ self.recover(e)
144
+ if self._input.LA(1)==Token.EOF:
145
+ self._hitEOF = True
146
+ if self._type == Token.INVALID_TYPE:
147
+ self._type = ttype
148
+ if self._type == self.SKIP:
149
+ continueOuter = True
150
+ break
151
+ if self._type!=self.MORE:
152
+ break
153
+ if continueOuter:
154
+ continue
155
+ if self._token is None:
156
+ self.emit()
157
+ return self._token
158
+ finally:
159
+ # make sure we release marker after match or
160
+ # unbuffered char stream will keep buffering
161
+ self._input.release(tokenStartMarker)
162
+
163
+ # Instruct the lexer to skip creating a token for current lexer rule
164
+ # and look for another token. nextToken() knows to keep looking when
165
+ # a lexer rule finishes with token set to SKIP_TOKEN. Recall that
166
+ # if token==null at end of any token rule, it creates one for you
167
+ # and emits it.
168
+ #/
169
+ def skip(self):
170
+ self._type = self.SKIP
171
+
172
+ def more(self):
173
+ self._type = self.MORE
174
+
175
+ def mode(self, m:int):
176
+ self._mode = m
177
+
178
+ def pushMode(self, m:int):
179
+ if self._interp.debug:
180
+ print("pushMode " + str(m), file=self._output)
181
+ self._modeStack.append(self._mode)
182
+ self.mode(m)
183
+
184
+ def popMode(self):
185
+ if len(self._modeStack)==0:
186
+ raise Exception("Empty Stack")
187
+ if self._interp.debug:
188
+ print("popMode back to "+ self._modeStack[:-1], file=self._output)
189
+ self.mode( self._modeStack.pop() )
190
+ return self._mode
191
+
192
+ # Set the char stream and reset the lexer#/
193
+ @property
194
+ def inputStream(self):
195
+ return self._input
196
+
197
+ @inputStream.setter
198
+ def inputStream(self, input:InputStream):
199
+ self._input = None
200
+ self._tokenFactorySourcePair = (self, self._input)
201
+ self.reset()
202
+ self._input = input
203
+ self._tokenFactorySourcePair = (self, self._input)
204
+
205
+ @property
206
+ def sourceName(self):
207
+ return self._input.sourceName
208
+
209
+ # By default does not support multiple emits per nextToken invocation
210
+ # for efficiency reasons. Subclass and override self method, nextToken,
211
+ # and getToken (to push tokens into a list and pull from that list
212
+ # rather than a single variable as self implementation does).
213
+ #/
214
+ def emitToken(self, token:Token):
215
+ self._token = token
216
+
217
+ # The standard method called to automatically emit a token at the
218
+ # outermost lexical rule. The token object should point into the
219
+ # char buffer start..stop. If there is a text override in 'text',
220
+ # use that to set the token's text. Override self method to emit
221
+ # custom Token objects or provide a new factory.
222
+ #/
223
+ def emit(self):
224
+ t = self._factory.create(self._tokenFactorySourcePair, self._type, self._text, self._channel, self._tokenStartCharIndex,
225
+ self.getCharIndex()-1, self._tokenStartLine, self._tokenStartColumn)
226
+ self.emitToken(t)
227
+ return t
228
+
229
+ def emitEOF(self):
230
+ cpos = self.column
231
+ lpos = self.line
232
+ eof = self._factory.create(self._tokenFactorySourcePair, Token.EOF, None, Token.DEFAULT_CHANNEL, self._input.index,
233
+ self._input.index-1, lpos, cpos)
234
+ self.emitToken(eof)
235
+ return eof
236
+
237
+ @property
238
+ def type(self):
239
+ return self._type
240
+
241
+ @type.setter
242
+ def type(self, type:int):
243
+ self._type = type
244
+
245
+ @property
246
+ def line(self):
247
+ return self._interp.line
248
+
249
+ @line.setter
250
+ def line(self, line:int):
251
+ self._interp.line = line
252
+
253
+ @property
254
+ def column(self):
255
+ return self._interp.column
256
+
257
+ @column.setter
258
+ def column(self, column:int):
259
+ self._interp.column = column
260
+
261
+ # What is the index of the current character of lookahead?#/
262
+ def getCharIndex(self):
263
+ return self._input.index
264
+
265
+ # Return the text matched so far for the current token or any
266
+ # text override.
267
+ @property
268
+ def text(self):
269
+ if self._text is not None:
270
+ return self._text
271
+ else:
272
+ return self._interp.getText(self._input)
273
+
274
+ # Set the complete text of self token; it wipes any previous
275
+ # changes to the text.
276
+ @text.setter
277
+ def text(self, txt:str):
278
+ self._text = txt
279
+
280
+ # Return a list of all Token objects in input char stream.
281
+ # Forces load of all tokens. Does not include EOF token.
282
+ #/
283
+ def getAllTokens(self):
284
+ tokens = []
285
+ t = self.nextToken()
286
+ while t.type!=Token.EOF:
287
+ tokens.append(t)
288
+ t = self.nextToken()
289
+ return tokens
290
+
291
+ def notifyListeners(self, e:LexerNoViableAltException):
292
+ start = self._tokenStartCharIndex
293
+ stop = self._input.index
294
+ text = self._input.getText(start, stop)
295
+ msg = "token recognition error at: '" + self.getErrorDisplay(text) + "'"
296
+ listener = self.getErrorListenerDispatch()
297
+ listener.syntaxError(self, None, self._tokenStartLine, self._tokenStartColumn, msg, e)
298
+
299
+ def getErrorDisplay(self, s:str):
300
+ with StringIO() as buf:
301
+ for c in s:
302
+ buf.write(self.getErrorDisplayForChar(c))
303
+ return buf.getvalue()
304
+
305
+ def getErrorDisplayForChar(self, c:str):
306
+ if ord(c[0])==Token.EOF:
307
+ return "<EOF>"
308
+ elif c=='\n':
309
+ return "\\n"
310
+ elif c=='\t':
311
+ return "\\t"
312
+ elif c=='\r':
313
+ return "\\r"
314
+ else:
315
+ return c
316
+
317
+ def getCharErrorDisplay(self, c:str):
318
+ return "'" + self.getErrorDisplayForChar(c) + "'"
319
+
320
+ # Lexers can normally match any char in it's vocabulary after matching
321
+ # a token, so do the easy thing and just kill a character and hope
322
+ # it all works out. You can instead use the rule invocation stack
323
+ # to do sophisticated error recovery if you are in a fragment rule.
324
+ #/
325
+ def recover(self, re:RecognitionException):
326
+ if self._input.LA(1) != Token.EOF:
327
+ if isinstance(re, LexerNoViableAltException):
328
+ # skip a char and try again
329
+ self._interp.consume(self._input)
330
+ else:
331
+ # TODO: Do we lose character or line position information?
332
+ self._input.consume()