omlish 0.0.0.dev56__py3-none-any.whl → 0.0.0.dev58__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omlish/__about__.py +2 -2
- omlish/antlr/__init__.py +0 -0
- omlish/antlr/_runtime/BufferedTokenStream.py +305 -0
- omlish/antlr/_runtime/CommonTokenFactory.py +64 -0
- omlish/antlr/_runtime/CommonTokenStream.py +90 -0
- omlish/antlr/_runtime/FileStream.py +30 -0
- omlish/antlr/_runtime/InputStream.py +90 -0
- omlish/antlr/_runtime/IntervalSet.py +183 -0
- omlish/antlr/_runtime/LL1Analyzer.py +176 -0
- omlish/antlr/_runtime/Lexer.py +332 -0
- omlish/antlr/_runtime/ListTokenSource.py +147 -0
- omlish/antlr/_runtime/Parser.py +583 -0
- omlish/antlr/_runtime/ParserInterpreter.py +173 -0
- omlish/antlr/_runtime/ParserRuleContext.py +189 -0
- omlish/antlr/_runtime/PredictionContext.py +632 -0
- omlish/antlr/_runtime/Recognizer.py +150 -0
- omlish/antlr/_runtime/RuleContext.py +230 -0
- omlish/antlr/_runtime/StdinStream.py +14 -0
- omlish/antlr/_runtime/Token.py +158 -0
- omlish/antlr/_runtime/TokenStreamRewriter.py +258 -0
- omlish/antlr/_runtime/Utils.py +36 -0
- omlish/antlr/_runtime/__init__.py +24 -0
- omlish/antlr/_runtime/_pygrun.py +174 -0
- omlish/antlr/_runtime/atn/ATN.py +135 -0
- omlish/antlr/_runtime/atn/ATNConfig.py +162 -0
- omlish/antlr/_runtime/atn/ATNConfigSet.py +215 -0
- omlish/antlr/_runtime/atn/ATNDeserializationOptions.py +27 -0
- omlish/antlr/_runtime/atn/ATNDeserializer.py +449 -0
- omlish/antlr/_runtime/atn/ATNSimulator.py +50 -0
- omlish/antlr/_runtime/atn/ATNState.py +267 -0
- omlish/antlr/_runtime/atn/ATNType.py +20 -0
- omlish/antlr/_runtime/atn/LexerATNSimulator.py +573 -0
- omlish/antlr/_runtime/atn/LexerAction.py +301 -0
- omlish/antlr/_runtime/atn/LexerActionExecutor.py +146 -0
- omlish/antlr/_runtime/atn/ParserATNSimulator.py +1664 -0
- omlish/antlr/_runtime/atn/PredictionMode.py +502 -0
- omlish/antlr/_runtime/atn/SemanticContext.py +333 -0
- omlish/antlr/_runtime/atn/Transition.py +271 -0
- omlish/antlr/_runtime/atn/__init__.py +4 -0
- omlish/antlr/_runtime/dfa/DFA.py +136 -0
- omlish/antlr/_runtime/dfa/DFASerializer.py +76 -0
- omlish/antlr/_runtime/dfa/DFAState.py +129 -0
- omlish/antlr/_runtime/dfa/__init__.py +4 -0
- omlish/antlr/_runtime/error/DiagnosticErrorListener.py +110 -0
- omlish/antlr/_runtime/error/ErrorListener.py +75 -0
- omlish/antlr/_runtime/error/ErrorStrategy.py +712 -0
- omlish/antlr/_runtime/error/Errors.py +176 -0
- omlish/antlr/_runtime/error/__init__.py +4 -0
- omlish/antlr/_runtime/tree/Chunk.py +33 -0
- omlish/antlr/_runtime/tree/ParseTreeMatch.py +121 -0
- omlish/antlr/_runtime/tree/ParseTreePattern.py +75 -0
- omlish/antlr/_runtime/tree/ParseTreePatternMatcher.py +377 -0
- omlish/antlr/_runtime/tree/RuleTagToken.py +53 -0
- omlish/antlr/_runtime/tree/TokenTagToken.py +50 -0
- omlish/antlr/_runtime/tree/Tree.py +194 -0
- omlish/antlr/_runtime/tree/Trees.py +114 -0
- omlish/antlr/_runtime/tree/__init__.py +2 -0
- omlish/antlr/_runtime/xpath/XPath.py +272 -0
- omlish/antlr/_runtime/xpath/XPathLexer.py +98 -0
- omlish/antlr/_runtime/xpath/__init__.py +4 -0
- omlish/formats/json/cli.py +76 -7
- omlish/formats/props.py +6 -2
- {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/METADATA +1 -1
- {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/RECORD +68 -9
- {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/LICENSE +0 -0
- {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/WHEEL +0 -0
- {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/entry_points.txt +0 -0
- {omlish-0.0.0.dev56.dist-info → omlish-0.0.0.dev58.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,183 @@
|
|
1
|
+
# type: ignore
|
2
|
+
# ruff: noqa
|
3
|
+
# flake8: noqa
|
4
|
+
#
|
5
|
+
# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
6
|
+
# Use of this file is governed by the BSD 3-clause license that
|
7
|
+
# can be found in the LICENSE.txt file in the project root.
|
8
|
+
#
|
9
|
+
|
10
|
+
from io import StringIO
|
11
|
+
from .Token import Token
|
12
|
+
|
13
|
+
# need forward declarations
|
14
|
+
IntervalSet = None
|
15
|
+
|
16
|
+
class IntervalSet(object):
|
17
|
+
__slots__ = ('intervals', 'readonly')
|
18
|
+
|
19
|
+
def __init__(self):
|
20
|
+
self.intervals = None
|
21
|
+
self.readonly = False
|
22
|
+
|
23
|
+
def __iter__(self):
|
24
|
+
if self.intervals is not None:
|
25
|
+
for i in self.intervals:
|
26
|
+
for c in i:
|
27
|
+
yield c
|
28
|
+
|
29
|
+
def __getitem__(self, item):
|
30
|
+
i = 0
|
31
|
+
for k in self:
|
32
|
+
if i==item:
|
33
|
+
return k
|
34
|
+
else:
|
35
|
+
i += 1
|
36
|
+
return Token.INVALID_TYPE
|
37
|
+
|
38
|
+
def addOne(self, v:int):
|
39
|
+
self.addRange(range(v, v+1))
|
40
|
+
|
41
|
+
def addRange(self, v:range):
|
42
|
+
if self.intervals is None:
|
43
|
+
self.intervals = list()
|
44
|
+
self.intervals.append(v)
|
45
|
+
else:
|
46
|
+
# find insert pos
|
47
|
+
k = 0
|
48
|
+
for i in self.intervals:
|
49
|
+
# distinct range -> insert
|
50
|
+
if v.stop<i.start:
|
51
|
+
self.intervals.insert(k, v)
|
52
|
+
return
|
53
|
+
# contiguous range -> adjust
|
54
|
+
elif v.stop==i.start:
|
55
|
+
self.intervals[k] = range(v.start, i.stop)
|
56
|
+
return
|
57
|
+
# overlapping range -> adjust and reduce
|
58
|
+
elif v.start<=i.stop:
|
59
|
+
self.intervals[k] = range(min(i.start,v.start), max(i.stop,v.stop))
|
60
|
+
self.reduce(k)
|
61
|
+
return
|
62
|
+
k += 1
|
63
|
+
# greater than any existing
|
64
|
+
self.intervals.append(v)
|
65
|
+
|
66
|
+
def addSet(self, other:IntervalSet):
|
67
|
+
if other.intervals is not None:
|
68
|
+
for i in other.intervals:
|
69
|
+
self.addRange(i)
|
70
|
+
return self
|
71
|
+
|
72
|
+
def reduce(self, k:int):
|
73
|
+
# only need to reduce if k is not the last
|
74
|
+
if k<len(self.intervals)-1:
|
75
|
+
l = self.intervals[k]
|
76
|
+
r = self.intervals[k+1]
|
77
|
+
# if r contained in l
|
78
|
+
if l.stop >= r.stop:
|
79
|
+
self.intervals.pop(k+1)
|
80
|
+
self.reduce(k)
|
81
|
+
elif l.stop >= r.start:
|
82
|
+
self.intervals[k] = range(l.start, r.stop)
|
83
|
+
self.intervals.pop(k+1)
|
84
|
+
|
85
|
+
def complement(self, start, stop):
|
86
|
+
result = IntervalSet()
|
87
|
+
result.addRange(range(start,stop+1))
|
88
|
+
for i in self.intervals:
|
89
|
+
result.removeRange(i)
|
90
|
+
return result
|
91
|
+
|
92
|
+
def __contains__(self, item):
|
93
|
+
if self.intervals is None:
|
94
|
+
return False
|
95
|
+
else:
|
96
|
+
return any(item in i for i in self.intervals)
|
97
|
+
|
98
|
+
def __len__(self):
|
99
|
+
return sum(len(i) for i in self.intervals)
|
100
|
+
|
101
|
+
def removeRange(self, v):
|
102
|
+
if v.start==v.stop-1:
|
103
|
+
self.removeOne(v.start)
|
104
|
+
elif self.intervals is not None:
|
105
|
+
k = 0
|
106
|
+
for i in self.intervals:
|
107
|
+
# intervals are ordered
|
108
|
+
if v.stop<=i.start:
|
109
|
+
return
|
110
|
+
# check for including range, split it
|
111
|
+
elif v.start>i.start and v.stop<i.stop:
|
112
|
+
self.intervals[k] = range(i.start, v.start)
|
113
|
+
x = range(v.stop, i.stop)
|
114
|
+
self.intervals.insert(k, x)
|
115
|
+
return
|
116
|
+
# check for included range, remove it
|
117
|
+
elif v.start<=i.start and v.stop>=i.stop:
|
118
|
+
self.intervals.pop(k)
|
119
|
+
k -= 1 # need another pass
|
120
|
+
# check for lower boundary
|
121
|
+
elif v.start<i.stop:
|
122
|
+
self.intervals[k] = range(i.start, v.start)
|
123
|
+
# check for upper boundary
|
124
|
+
elif v.stop<i.stop:
|
125
|
+
self.intervals[k] = range(v.stop, i.stop)
|
126
|
+
k += 1
|
127
|
+
|
128
|
+
def removeOne(self, v):
|
129
|
+
if self.intervals is not None:
|
130
|
+
k = 0
|
131
|
+
for i in self.intervals:
|
132
|
+
# intervals is ordered
|
133
|
+
if v<i.start:
|
134
|
+
return
|
135
|
+
# check for single value range
|
136
|
+
elif v==i.start and v==i.stop-1:
|
137
|
+
self.intervals.pop(k)
|
138
|
+
return
|
139
|
+
# check for lower boundary
|
140
|
+
elif v==i.start:
|
141
|
+
self.intervals[k] = range(i.start+1, i.stop)
|
142
|
+
return
|
143
|
+
# check for upper boundary
|
144
|
+
elif v==i.stop-1:
|
145
|
+
self.intervals[k] = range(i.start, i.stop-1)
|
146
|
+
return
|
147
|
+
# split existing range
|
148
|
+
elif v<i.stop-1:
|
149
|
+
x = range(i.start, v)
|
150
|
+
self.intervals[k] = range(v + 1, i.stop)
|
151
|
+
self.intervals.insert(k, x)
|
152
|
+
return
|
153
|
+
k += 1
|
154
|
+
|
155
|
+
|
156
|
+
def toString(self, literalNames:list, symbolicNames:list):
|
157
|
+
if self.intervals is None:
|
158
|
+
return "{}"
|
159
|
+
with StringIO() as buf:
|
160
|
+
if len(self)>1:
|
161
|
+
buf.write("{")
|
162
|
+
first = True
|
163
|
+
for i in self.intervals:
|
164
|
+
for j in i:
|
165
|
+
if not first:
|
166
|
+
buf.write(", ")
|
167
|
+
buf.write(self.elementName(literalNames, symbolicNames, j))
|
168
|
+
first = False
|
169
|
+
if len(self)>1:
|
170
|
+
buf.write("}")
|
171
|
+
return buf.getvalue()
|
172
|
+
|
173
|
+
def elementName(self, literalNames:list, symbolicNames:list, a:int):
|
174
|
+
if a==Token.EOF:
|
175
|
+
return "<EOF>"
|
176
|
+
elif a==Token.EPSILON:
|
177
|
+
return "<EPSILON>"
|
178
|
+
else:
|
179
|
+
if a<len(literalNames) and literalNames[a] != "<INVALID>":
|
180
|
+
return literalNames[a]
|
181
|
+
if a<len(symbolicNames):
|
182
|
+
return symbolicNames[a]
|
183
|
+
return "<UNKNOWN>"
|
@@ -0,0 +1,176 @@
|
|
1
|
+
# type: ignore
|
2
|
+
# ruff: noqa
|
3
|
+
# flake8: noqa
|
4
|
+
#
|
5
|
+
# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
6
|
+
# Use of this file is governed by the BSD 3-clause license that
|
7
|
+
# can be found in the LICENSE.txt file in the project root.
|
8
|
+
#/
|
9
|
+
from .IntervalSet import IntervalSet
|
10
|
+
from .Token import Token
|
11
|
+
from .PredictionContext import PredictionContext, SingletonPredictionContext, PredictionContextFromRuleContext
|
12
|
+
from .RuleContext import RuleContext
|
13
|
+
from .atn.ATN import ATN
|
14
|
+
from .atn.ATNConfig import ATNConfig
|
15
|
+
from .atn.ATNState import ATNState, RuleStopState
|
16
|
+
from .atn.Transition import WildcardTransition, NotSetTransition, AbstractPredicateTransition, RuleTransition
|
17
|
+
|
18
|
+
|
19
|
+
class LL1Analyzer (object):
|
20
|
+
__slots__ = 'atn'
|
21
|
+
|
22
|
+
#* Special value added to the lookahead sets to indicate that we hit
|
23
|
+
# a predicate during analysis if {@code seeThruPreds==false}.
|
24
|
+
#/
|
25
|
+
HIT_PRED = Token.INVALID_TYPE
|
26
|
+
|
27
|
+
def __init__(self, atn:ATN):
|
28
|
+
self.atn = atn
|
29
|
+
|
30
|
+
#*
|
31
|
+
# Calculates the SLL(1) expected lookahead set for each outgoing transition
|
32
|
+
# of an {@link ATNState}. The returned array has one element for each
|
33
|
+
# outgoing transition in {@code s}. If the closure from transition
|
34
|
+
# <em>i</em> leads to a semantic predicate before matching a symbol, the
|
35
|
+
# element at index <em>i</em> of the result will be {@code null}.
|
36
|
+
#
|
37
|
+
# @param s the ATN state
|
38
|
+
# @return the expected symbols for each outgoing transition of {@code s}.
|
39
|
+
#/
|
40
|
+
def getDecisionLookahead(self, s:ATNState):
|
41
|
+
if s is None:
|
42
|
+
return None
|
43
|
+
|
44
|
+
count = len(s.transitions)
|
45
|
+
look = [] * count
|
46
|
+
for alt in range(0, count):
|
47
|
+
look[alt] = set()
|
48
|
+
lookBusy = set()
|
49
|
+
seeThruPreds = False # fail to get lookahead upon pred
|
50
|
+
self._LOOK(s.transition(alt).target, None, PredictionContext.EMPTY,
|
51
|
+
look[alt], lookBusy, set(), seeThruPreds, False)
|
52
|
+
# Wipe out lookahead for this alternative if we found nothing
|
53
|
+
# or we had a predicate when we !seeThruPreds
|
54
|
+
if len(look[alt])==0 or self.HIT_PRED in look[alt]:
|
55
|
+
look[alt] = None
|
56
|
+
return look
|
57
|
+
|
58
|
+
#*
|
59
|
+
# Compute set of tokens that can follow {@code s} in the ATN in the
|
60
|
+
# specified {@code ctx}.
|
61
|
+
#
|
62
|
+
# <p>If {@code ctx} is {@code null} and the end of the rule containing
|
63
|
+
# {@code s} is reached, {@link Token#EPSILON} is added to the result set.
|
64
|
+
# If {@code ctx} is not {@code null} and the end of the outermost rule is
|
65
|
+
# reached, {@link Token#EOF} is added to the result set.</p>
|
66
|
+
#
|
67
|
+
# @param s the ATN state
|
68
|
+
# @param stopState the ATN state to stop at. This can be a
|
69
|
+
# {@link BlockEndState} to detect epsilon paths through a closure.
|
70
|
+
# @param ctx the complete parser context, or {@code null} if the context
|
71
|
+
# should be ignored
|
72
|
+
#
|
73
|
+
# @return The set of tokens that can follow {@code s} in the ATN in the
|
74
|
+
# specified {@code ctx}.
|
75
|
+
#/
|
76
|
+
def LOOK(self, s:ATNState, stopState:ATNState=None, ctx:RuleContext=None):
|
77
|
+
r = IntervalSet()
|
78
|
+
seeThruPreds = True # ignore preds; get all lookahead
|
79
|
+
lookContext = PredictionContextFromRuleContext(s.atn, ctx) if ctx is not None else None
|
80
|
+
self._LOOK(s, stopState, lookContext, r, set(), set(), seeThruPreds, True)
|
81
|
+
return r
|
82
|
+
|
83
|
+
#*
|
84
|
+
# Compute set of tokens that can follow {@code s} in the ATN in the
|
85
|
+
# specified {@code ctx}.
|
86
|
+
#
|
87
|
+
# <p>If {@code ctx} is {@code null} and {@code stopState} or the end of the
|
88
|
+
# rule containing {@code s} is reached, {@link Token#EPSILON} is added to
|
89
|
+
# the result set. If {@code ctx} is not {@code null} and {@code addEOF} is
|
90
|
+
# {@code true} and {@code stopState} or the end of the outermost rule is
|
91
|
+
# reached, {@link Token#EOF} is added to the result set.</p>
|
92
|
+
#
|
93
|
+
# @param s the ATN state.
|
94
|
+
# @param stopState the ATN state to stop at. This can be a
|
95
|
+
# {@link BlockEndState} to detect epsilon paths through a closure.
|
96
|
+
# @param ctx The outer context, or {@code null} if the outer context should
|
97
|
+
# not be used.
|
98
|
+
# @param look The result lookahead set.
|
99
|
+
# @param lookBusy A set used for preventing epsilon closures in the ATN
|
100
|
+
# from causing a stack overflow. Outside code should pass
|
101
|
+
# {@code new HashSet<ATNConfig>} for this argument.
|
102
|
+
# @param calledRuleStack A set used for preventing left recursion in the
|
103
|
+
# ATN from causing a stack overflow. Outside code should pass
|
104
|
+
# {@code new BitSet()} for this argument.
|
105
|
+
# @param seeThruPreds {@code true} to true semantic predicates as
|
106
|
+
# implicitly {@code true} and "see through them", otherwise {@code false}
|
107
|
+
# to treat semantic predicates as opaque and add {@link #HIT_PRED} to the
|
108
|
+
# result if one is encountered.
|
109
|
+
# @param addEOF Add {@link Token#EOF} to the result if the end of the
|
110
|
+
# outermost context is reached. This parameter has no effect if {@code ctx}
|
111
|
+
# is {@code null}.
|
112
|
+
#/
|
113
|
+
def _LOOK(self, s:ATNState, stopState:ATNState , ctx:PredictionContext, look:IntervalSet, lookBusy:set,
|
114
|
+
calledRuleStack:set, seeThruPreds:bool, addEOF:bool):
|
115
|
+
c = ATNConfig(s, 0, ctx)
|
116
|
+
|
117
|
+
if c in lookBusy:
|
118
|
+
return
|
119
|
+
lookBusy.add(c)
|
120
|
+
|
121
|
+
if s == stopState:
|
122
|
+
if ctx is None:
|
123
|
+
look.addOne(Token.EPSILON)
|
124
|
+
return
|
125
|
+
elif ctx.isEmpty() and addEOF:
|
126
|
+
look.addOne(Token.EOF)
|
127
|
+
return
|
128
|
+
|
129
|
+
if isinstance(s, RuleStopState ):
|
130
|
+
if ctx is None:
|
131
|
+
look.addOne(Token.EPSILON)
|
132
|
+
return
|
133
|
+
elif ctx.isEmpty() and addEOF:
|
134
|
+
look.addOne(Token.EOF)
|
135
|
+
return
|
136
|
+
|
137
|
+
if ctx != PredictionContext.EMPTY:
|
138
|
+
removed = s.ruleIndex in calledRuleStack
|
139
|
+
try:
|
140
|
+
calledRuleStack.discard(s.ruleIndex)
|
141
|
+
# run thru all possible stack tops in ctx
|
142
|
+
for i in range(0, len(ctx)):
|
143
|
+
returnState = self.atn.states[ctx.getReturnState(i)]
|
144
|
+
self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
145
|
+
finally:
|
146
|
+
if removed:
|
147
|
+
calledRuleStack.add(s.ruleIndex)
|
148
|
+
return
|
149
|
+
|
150
|
+
for t in s.transitions:
|
151
|
+
if type(t) == RuleTransition:
|
152
|
+
if t.target.ruleIndex in calledRuleStack:
|
153
|
+
continue
|
154
|
+
|
155
|
+
newContext = SingletonPredictionContext.create(ctx, t.followState.stateNumber)
|
156
|
+
|
157
|
+
try:
|
158
|
+
calledRuleStack.add(t.target.ruleIndex)
|
159
|
+
self._LOOK(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
160
|
+
finally:
|
161
|
+
calledRuleStack.remove(t.target.ruleIndex)
|
162
|
+
elif isinstance(t, AbstractPredicateTransition ):
|
163
|
+
if seeThruPreds:
|
164
|
+
self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
165
|
+
else:
|
166
|
+
look.addOne(self.HIT_PRED)
|
167
|
+
elif t.isEpsilon:
|
168
|
+
self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
|
169
|
+
elif type(t) == WildcardTransition:
|
170
|
+
look.addRange( range(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType + 1) )
|
171
|
+
else:
|
172
|
+
set_ = t.label
|
173
|
+
if set_ is not None:
|
174
|
+
if isinstance(t, NotSetTransition):
|
175
|
+
set_ = set_.complement(Token.MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
|
176
|
+
look.addSet(set_)
|
@@ -0,0 +1,332 @@
|
|
1
|
+
# type: ignore
|
2
|
+
# ruff: noqa
|
3
|
+
# flake8: noqa
|
4
|
+
# Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
|
5
|
+
# Use of this file is governed by the BSD 3-clause license that
|
6
|
+
# can be found in the LICENSE.txt file in the project root.
|
7
|
+
#/
|
8
|
+
|
9
|
+
# A lexer is recognizer that draws input symbols from a character stream.
|
10
|
+
# lexer grammars result in a subclass of self object. A Lexer object
|
11
|
+
# uses simplified match() and error recovery mechanisms in the interest
|
12
|
+
# of speed.
|
13
|
+
#/
|
14
|
+
from io import StringIO
|
15
|
+
|
16
|
+
import sys
|
17
|
+
if sys.version_info[1] > 5:
|
18
|
+
from typing import TextIO
|
19
|
+
else:
|
20
|
+
from typing.io import TextIO
|
21
|
+
from .CommonTokenFactory import CommonTokenFactory
|
22
|
+
from .atn.LexerATNSimulator import LexerATNSimulator
|
23
|
+
from .InputStream import InputStream
|
24
|
+
from .Recognizer import Recognizer
|
25
|
+
from .Token import Token
|
26
|
+
from .error.Errors import IllegalStateException, LexerNoViableAltException, RecognitionException
|
27
|
+
|
28
|
+
class TokenSource(object):
|
29
|
+
|
30
|
+
pass
|
31
|
+
|
32
|
+
|
33
|
+
class Lexer(Recognizer, TokenSource):
|
34
|
+
__slots__ = (
|
35
|
+
'_input', '_output', '_factory', '_tokenFactorySourcePair', '_token',
|
36
|
+
'_tokenStartCharIndex', '_tokenStartLine', '_tokenStartColumn',
|
37
|
+
'_hitEOF', '_channel', '_type', '_modeStack', '_mode', '_text'
|
38
|
+
)
|
39
|
+
|
40
|
+
DEFAULT_MODE = 0
|
41
|
+
MORE = -2
|
42
|
+
SKIP = -3
|
43
|
+
|
44
|
+
DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL
|
45
|
+
HIDDEN = Token.HIDDEN_CHANNEL
|
46
|
+
MIN_CHAR_VALUE = 0x0000
|
47
|
+
MAX_CHAR_VALUE = 0x10FFFF
|
48
|
+
|
49
|
+
def __init__(self, input:InputStream, output:TextIO = sys.stdout):
|
50
|
+
super().__init__()
|
51
|
+
self._input = input
|
52
|
+
self._output = output
|
53
|
+
self._factory = CommonTokenFactory.DEFAULT
|
54
|
+
self._tokenFactorySourcePair = (self, input)
|
55
|
+
|
56
|
+
self._interp = None # child classes must populate this
|
57
|
+
|
58
|
+
# The goal of all lexer rules/methods is to create a token object.
|
59
|
+
# self is an instance variable as multiple rules may collaborate to
|
60
|
+
# create a single token. nextToken will return self object after
|
61
|
+
# matching lexer rule(s). If you subclass to allow multiple token
|
62
|
+
# emissions, then set self to the last token to be matched or
|
63
|
+
# something nonnull so that the auto token emit mechanism will not
|
64
|
+
# emit another token.
|
65
|
+
self._token = None
|
66
|
+
|
67
|
+
# What character index in the stream did the current token start at?
|
68
|
+
# Needed, for example, to get the text for current token. Set at
|
69
|
+
# the start of nextToken.
|
70
|
+
self._tokenStartCharIndex = -1
|
71
|
+
|
72
|
+
# The line on which the first character of the token resides#/
|
73
|
+
self._tokenStartLine = -1
|
74
|
+
|
75
|
+
# The character position of first character within the line#/
|
76
|
+
self._tokenStartColumn = -1
|
77
|
+
|
78
|
+
# Once we see EOF on char stream, next token will be EOF.
|
79
|
+
# If you have DONE : EOF ; then you see DONE EOF.
|
80
|
+
self._hitEOF = False
|
81
|
+
|
82
|
+
# The channel number for the current token#/
|
83
|
+
self._channel = Token.DEFAULT_CHANNEL
|
84
|
+
|
85
|
+
# The token type for the current token#/
|
86
|
+
self._type = Token.INVALID_TYPE
|
87
|
+
|
88
|
+
self._modeStack = []
|
89
|
+
self._mode = self.DEFAULT_MODE
|
90
|
+
|
91
|
+
# You can set the text for the current token to override what is in
|
92
|
+
# the input char buffer. Use setText() or can set self instance var.
|
93
|
+
#/
|
94
|
+
self._text = None
|
95
|
+
|
96
|
+
|
97
|
+
def reset(self):
|
98
|
+
# wack Lexer state variables
|
99
|
+
if self._input is not None:
|
100
|
+
self._input.seek(0) # rewind the input
|
101
|
+
self._token = None
|
102
|
+
self._type = Token.INVALID_TYPE
|
103
|
+
self._channel = Token.DEFAULT_CHANNEL
|
104
|
+
self._tokenStartCharIndex = -1
|
105
|
+
self._tokenStartColumn = -1
|
106
|
+
self._tokenStartLine = -1
|
107
|
+
self._text = None
|
108
|
+
|
109
|
+
self._hitEOF = False
|
110
|
+
self._mode = Lexer.DEFAULT_MODE
|
111
|
+
self._modeStack = []
|
112
|
+
|
113
|
+
self._interp.reset()
|
114
|
+
|
115
|
+
# Return a token from self source; i.e., match a token on the char
|
116
|
+
# stream.
|
117
|
+
def nextToken(self):
|
118
|
+
if self._input is None:
|
119
|
+
raise IllegalStateException("nextToken requires a non-null input stream.")
|
120
|
+
|
121
|
+
# Mark start location in char stream so unbuffered streams are
|
122
|
+
# guaranteed at least have text of current token
|
123
|
+
tokenStartMarker = self._input.mark()
|
124
|
+
try:
|
125
|
+
while True:
|
126
|
+
if self._hitEOF:
|
127
|
+
self.emitEOF()
|
128
|
+
return self._token
|
129
|
+
self._token = None
|
130
|
+
self._channel = Token.DEFAULT_CHANNEL
|
131
|
+
self._tokenStartCharIndex = self._input.index
|
132
|
+
self._tokenStartColumn = self._interp.column
|
133
|
+
self._tokenStartLine = self._interp.line
|
134
|
+
self._text = None
|
135
|
+
continueOuter = False
|
136
|
+
while True:
|
137
|
+
self._type = Token.INVALID_TYPE
|
138
|
+
ttype = self.SKIP
|
139
|
+
try:
|
140
|
+
ttype = self._interp.match(self._input, self._mode)
|
141
|
+
except LexerNoViableAltException as e:
|
142
|
+
self.notifyListeners(e) # report error
|
143
|
+
self.recover(e)
|
144
|
+
if self._input.LA(1)==Token.EOF:
|
145
|
+
self._hitEOF = True
|
146
|
+
if self._type == Token.INVALID_TYPE:
|
147
|
+
self._type = ttype
|
148
|
+
if self._type == self.SKIP:
|
149
|
+
continueOuter = True
|
150
|
+
break
|
151
|
+
if self._type!=self.MORE:
|
152
|
+
break
|
153
|
+
if continueOuter:
|
154
|
+
continue
|
155
|
+
if self._token is None:
|
156
|
+
self.emit()
|
157
|
+
return self._token
|
158
|
+
finally:
|
159
|
+
# make sure we release marker after match or
|
160
|
+
# unbuffered char stream will keep buffering
|
161
|
+
self._input.release(tokenStartMarker)
|
162
|
+
|
163
|
+
# Instruct the lexer to skip creating a token for current lexer rule
|
164
|
+
# and look for another token. nextToken() knows to keep looking when
|
165
|
+
# a lexer rule finishes with token set to SKIP_TOKEN. Recall that
|
166
|
+
# if token==null at end of any token rule, it creates one for you
|
167
|
+
# and emits it.
|
168
|
+
#/
|
169
|
+
def skip(self):
|
170
|
+
self._type = self.SKIP
|
171
|
+
|
172
|
+
def more(self):
|
173
|
+
self._type = self.MORE
|
174
|
+
|
175
|
+
def mode(self, m:int):
|
176
|
+
self._mode = m
|
177
|
+
|
178
|
+
def pushMode(self, m:int):
|
179
|
+
if self._interp.debug:
|
180
|
+
print("pushMode " + str(m), file=self._output)
|
181
|
+
self._modeStack.append(self._mode)
|
182
|
+
self.mode(m)
|
183
|
+
|
184
|
+
def popMode(self):
|
185
|
+
if len(self._modeStack)==0:
|
186
|
+
raise Exception("Empty Stack")
|
187
|
+
if self._interp.debug:
|
188
|
+
print("popMode back to "+ self._modeStack[:-1], file=self._output)
|
189
|
+
self.mode( self._modeStack.pop() )
|
190
|
+
return self._mode
|
191
|
+
|
192
|
+
# Set the char stream and reset the lexer#/
|
193
|
+
@property
|
194
|
+
def inputStream(self):
|
195
|
+
return self._input
|
196
|
+
|
197
|
+
@inputStream.setter
|
198
|
+
def inputStream(self, input:InputStream):
|
199
|
+
self._input = None
|
200
|
+
self._tokenFactorySourcePair = (self, self._input)
|
201
|
+
self.reset()
|
202
|
+
self._input = input
|
203
|
+
self._tokenFactorySourcePair = (self, self._input)
|
204
|
+
|
205
|
+
@property
|
206
|
+
def sourceName(self):
|
207
|
+
return self._input.sourceName
|
208
|
+
|
209
|
+
# By default does not support multiple emits per nextToken invocation
|
210
|
+
# for efficiency reasons. Subclass and override self method, nextToken,
|
211
|
+
# and getToken (to push tokens into a list and pull from that list
|
212
|
+
# rather than a single variable as self implementation does).
|
213
|
+
#/
|
214
|
+
def emitToken(self, token:Token):
|
215
|
+
self._token = token
|
216
|
+
|
217
|
+
# The standard method called to automatically emit a token at the
|
218
|
+
# outermost lexical rule. The token object should point into the
|
219
|
+
# char buffer start..stop. If there is a text override in 'text',
|
220
|
+
# use that to set the token's text. Override self method to emit
|
221
|
+
# custom Token objects or provide a new factory.
|
222
|
+
#/
|
223
|
+
def emit(self):
|
224
|
+
t = self._factory.create(self._tokenFactorySourcePair, self._type, self._text, self._channel, self._tokenStartCharIndex,
|
225
|
+
self.getCharIndex()-1, self._tokenStartLine, self._tokenStartColumn)
|
226
|
+
self.emitToken(t)
|
227
|
+
return t
|
228
|
+
|
229
|
+
def emitEOF(self):
|
230
|
+
cpos = self.column
|
231
|
+
lpos = self.line
|
232
|
+
eof = self._factory.create(self._tokenFactorySourcePair, Token.EOF, None, Token.DEFAULT_CHANNEL, self._input.index,
|
233
|
+
self._input.index-1, lpos, cpos)
|
234
|
+
self.emitToken(eof)
|
235
|
+
return eof
|
236
|
+
|
237
|
+
@property
|
238
|
+
def type(self):
|
239
|
+
return self._type
|
240
|
+
|
241
|
+
@type.setter
|
242
|
+
def type(self, type:int):
|
243
|
+
self._type = type
|
244
|
+
|
245
|
+
@property
|
246
|
+
def line(self):
|
247
|
+
return self._interp.line
|
248
|
+
|
249
|
+
@line.setter
|
250
|
+
def line(self, line:int):
|
251
|
+
self._interp.line = line
|
252
|
+
|
253
|
+
@property
|
254
|
+
def column(self):
|
255
|
+
return self._interp.column
|
256
|
+
|
257
|
+
@column.setter
|
258
|
+
def column(self, column:int):
|
259
|
+
self._interp.column = column
|
260
|
+
|
261
|
+
# What is the index of the current character of lookahead?#/
|
262
|
+
def getCharIndex(self):
|
263
|
+
return self._input.index
|
264
|
+
|
265
|
+
# Return the text matched so far for the current token or any
|
266
|
+
# text override.
|
267
|
+
@property
|
268
|
+
def text(self):
|
269
|
+
if self._text is not None:
|
270
|
+
return self._text
|
271
|
+
else:
|
272
|
+
return self._interp.getText(self._input)
|
273
|
+
|
274
|
+
# Set the complete text of self token; it wipes any previous
|
275
|
+
# changes to the text.
|
276
|
+
@text.setter
|
277
|
+
def text(self, txt:str):
|
278
|
+
self._text = txt
|
279
|
+
|
280
|
+
# Return a list of all Token objects in input char stream.
|
281
|
+
# Forces load of all tokens. Does not include EOF token.
|
282
|
+
#/
|
283
|
+
def getAllTokens(self):
|
284
|
+
tokens = []
|
285
|
+
t = self.nextToken()
|
286
|
+
while t.type!=Token.EOF:
|
287
|
+
tokens.append(t)
|
288
|
+
t = self.nextToken()
|
289
|
+
return tokens
|
290
|
+
|
291
|
+
def notifyListeners(self, e:LexerNoViableAltException):
|
292
|
+
start = self._tokenStartCharIndex
|
293
|
+
stop = self._input.index
|
294
|
+
text = self._input.getText(start, stop)
|
295
|
+
msg = "token recognition error at: '" + self.getErrorDisplay(text) + "'"
|
296
|
+
listener = self.getErrorListenerDispatch()
|
297
|
+
listener.syntaxError(self, None, self._tokenStartLine, self._tokenStartColumn, msg, e)
|
298
|
+
|
299
|
+
def getErrorDisplay(self, s:str):
|
300
|
+
with StringIO() as buf:
|
301
|
+
for c in s:
|
302
|
+
buf.write(self.getErrorDisplayForChar(c))
|
303
|
+
return buf.getvalue()
|
304
|
+
|
305
|
+
def getErrorDisplayForChar(self, c:str):
|
306
|
+
if ord(c[0])==Token.EOF:
|
307
|
+
return "<EOF>"
|
308
|
+
elif c=='\n':
|
309
|
+
return "\\n"
|
310
|
+
elif c=='\t':
|
311
|
+
return "\\t"
|
312
|
+
elif c=='\r':
|
313
|
+
return "\\r"
|
314
|
+
else:
|
315
|
+
return c
|
316
|
+
|
317
|
+
def getCharErrorDisplay(self, c:str):
|
318
|
+
return "'" + self.getErrorDisplayForChar(c) + "'"
|
319
|
+
|
320
|
+
# Lexers can normally match any char in it's vocabulary after matching
|
321
|
+
# a token, so do the easy thing and just kill a character and hope
|
322
|
+
# it all works out. You can instead use the rule invocation stack
|
323
|
+
# to do sophisticated error recovery if you are in a fragment rule.
|
324
|
+
#/
|
325
|
+
def recover(self, re:RecognitionException):
|
326
|
+
if self._input.LA(1) != Token.EOF:
|
327
|
+
if isinstance(re, LexerNoViableAltException):
|
328
|
+
# skip a char and try again
|
329
|
+
self._interp.consume(self._input)
|
330
|
+
else:
|
331
|
+
# TODO: Do we lose character or line position information?
|
332
|
+
self._input.consume()
|