owl-basic 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- owl_basic/__init__.py +3 -0
- owl_basic/algorithms.py +29 -0
- owl_basic/ast_utils.py +204 -0
- owl_basic/basic_visitor.py +55 -0
- owl_basic/cfg_vertex.py +65 -0
- owl_basic/codegen/__init__.py +0 -0
- owl_basic/codegen/clr/__init__.py +0 -0
- owl_basic/codegen/clr/cil_visitor.py +1296 -0
- owl_basic/codegen/clr/cts.py +56 -0
- owl_basic/codegen/clr/emitters.py +94 -0
- owl_basic/codegen/clr/generate.py +539 -0
- owl_basic/correlation_visitor.py +119 -0
- owl_basic/data_visitor.py +62 -0
- owl_basic/decoder.py +339 -0
- owl_basic/errors.py +22 -0
- owl_basic/flow/__init__.py +17 -0
- owl_basic/flow/basic_block.py +34 -0
- owl_basic/flow/basic_block_identifier.py +66 -0
- owl_basic/flow/basic_block_orderer.py +29 -0
- owl_basic/flow/connectors.py +19 -0
- owl_basic/flow/convert_sub_visitor.py +28 -0
- owl_basic/flow/entry_point_locator.py +55 -0
- owl_basic/flow/entry_point_visitor.py +48 -0
- owl_basic/flow/flow_analysis.py +56 -0
- owl_basic/flow/flow_graph_creator.py +14 -0
- owl_basic/flow/flowgraph_visitor.py +178 -0
- owl_basic/flow/longjump_converter.py +20 -0
- owl_basic/flow/longjump_visitor.py +53 -0
- owl_basic/flow/subroutine_converter.py +38 -0
- owl_basic/flow/traversal.py +110 -0
- owl_basic/gml_visitor.py +151 -0
- owl_basic/line_mapper.py +43 -0
- owl_basic/line_number_visitor.py +65 -0
- owl_basic/main.py +381 -0
- owl_basic/node.py +21 -0
- owl_basic/options.py +22 -0
- owl_basic/owltyping/__init__.py +1 -0
- owl_basic/owltyping/function_type_inferer.py +50 -0
- owl_basic/owltyping/hindley_milner.py +524 -0
- owl_basic/owltyping/set_function_type_visitor.py +25 -0
- owl_basic/owltyping/type_system.py +220 -0
- owl_basic/owltyping/typecheck.py +60 -0
- owl_basic/owltyping/typecheck_visitor.py +471 -0
- owl_basic/parent_visitor.py +37 -0
- owl_basic/process.py +36 -0
- owl_basic/separation_visitor.py +98 -0
- owl_basic/sigil.py +30 -0
- owl_basic/simplify_visitor.py +204 -0
- owl_basic/singleton.py +127 -0
- owl_basic/source_debugging.py +124 -0
- owl_basic/symbol_table_visitor.py +220 -0
- owl_basic/symbol_tables.py +195 -0
- owl_basic/syntax/__init__.py +0 -0
- owl_basic/syntax/ast.py +1081 -0
- owl_basic/syntax/ast_meta.py +228 -0
- owl_basic/syntax/grammar.py +1972 -0
- owl_basic/syntax/lexer.py +943 -0
- owl_basic/syntax/parser.py +77 -0
- owl_basic/utility.py +26 -0
- owl_basic/visitor.py +43 -0
- owl_basic/xml_blocks.py +137 -0
- owl_basic/xml_visitor.py +101 -0
- owl_basic-0.6.0.dist-info/METADATA +37 -0
- owl_basic-0.6.0.dist-info/RECORD +69 -0
- owl_basic-0.6.0.dist-info/WHEEL +5 -0
- owl_basic-0.6.0.dist-info/entry_points.txt +2 -0
- owl_basic-0.6.0.dist-info/licenses/LICENSE +21 -0
- owl_basic-0.6.0.dist-info/licenses/THIRD-PARTY-NOTICES.md +57 -0
- owl_basic-0.6.0.dist-info/top_level.txt +1 -0
owl_basic/gml_visitor.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# A visitor implementation that creates an XML representation of the control flow graph
|
|
2
|
+
|
|
3
|
+
from owl_basic.visitor import Visitor
|
|
4
|
+
from owl_basic.syntax.ast import If, OnGoto
|
|
5
|
+
|
|
6
|
+
class GmlVisitor(Visitor):
|
|
7
|
+
"""
|
|
8
|
+
AST visitor for converting the CFG into an XML representation in GraphML.
|
|
9
|
+
We traverse the AST rather than the CFG to output the graph. We can more reliably
|
|
10
|
+
visit all nodes, since we know the AST is a single connected component.
|
|
11
|
+
"""
|
|
12
|
+
def __init__(self, filename, out_edges=True, in_edges=False, back_edges=True):
|
|
13
|
+
# .NET Framework
|
|
14
|
+
import clr
|
|
15
|
+
clr.AddReference('System.Xml')
|
|
16
|
+
from System.Xml import XmlTextWriter, Formatting
|
|
17
|
+
|
|
18
|
+
self.out_edges = out_edges
|
|
19
|
+
self.in_edges = in_edges
|
|
20
|
+
self.back_edges = back_edges
|
|
21
|
+
|
|
22
|
+
self.writer = XmlTextWriter(filename, None)
|
|
23
|
+
|
|
24
|
+
self.writer.Formatting = Formatting.Indented
|
|
25
|
+
self.writer.WriteComment("GraphML representation of the Control Flow Graph")
|
|
26
|
+
|
|
27
|
+
self.writer.WriteStartElement("graphml")
|
|
28
|
+
self.writer.WriteAttributeString("xmlns", "http://graphml.graphdrawing.org/xmlns")
|
|
29
|
+
self.writer.WriteAttributeString("xmlns", "xsi", None, "http://www.w3.org/2001/XMLSchema-instance")
|
|
30
|
+
self.writer.WriteAttributeString("xmlns", "y", None, "http://www.yworks.com/xml/graphml")
|
|
31
|
+
self.writer.WriteAttributeString("xsi", "schemaLocation", "http://graphml.graphdrawing.org/xmlns/graphml", "http://www.yworks.com/xml/schema/graphml/1.0/ygraphml.xsd")
|
|
32
|
+
|
|
33
|
+
self.writer.WriteStartElement("key")
|
|
34
|
+
self.writer.WriteAttributeString("id", "d0")
|
|
35
|
+
self.writer.WriteAttributeString("for", "node")
|
|
36
|
+
self.writer.WriteAttributeString("yfiles.type", "nodegraphics")
|
|
37
|
+
self.writer.WriteEndElement() # key
|
|
38
|
+
|
|
39
|
+
self.writer.WriteStartElement("key")
|
|
40
|
+
self.writer.WriteAttributeString("id", "d1")
|
|
41
|
+
self.writer.WriteAttributeString("for", "node")
|
|
42
|
+
self.writer.WriteAttributeString("attr.name", "description")
|
|
43
|
+
self.writer.WriteAttributeString("attr.type", "string")
|
|
44
|
+
self.writer.WriteEndElement() # key
|
|
45
|
+
|
|
46
|
+
self.writer.WriteStartElement("key")
|
|
47
|
+
self.writer.WriteAttributeString("id", "d3")
|
|
48
|
+
self.writer.WriteAttributeString("for", "edge")
|
|
49
|
+
self.writer.WriteAttributeString("yfiles.type", "edgegraphics")
|
|
50
|
+
self.writer.WriteEndElement() # key
|
|
51
|
+
|
|
52
|
+
self.writer.WriteStartElement("key")
|
|
53
|
+
self.writer.WriteAttributeString("id", "d4")
|
|
54
|
+
self.writer.WriteAttributeString("for", "graphml")
|
|
55
|
+
self.writer.WriteAttributeString("yfiles.type", "resources")
|
|
56
|
+
self.writer.WriteEndElement() # key
|
|
57
|
+
|
|
58
|
+
self.writer.WriteStartElement("graph")
|
|
59
|
+
self.writer.WriteAttributeString("id", "CFG")
|
|
60
|
+
self.writer.WriteAttributeString("edgedefault", "directed")
|
|
61
|
+
|
|
62
|
+
def close(self):
|
|
63
|
+
self.writer.WriteEndElement() # graph
|
|
64
|
+
self.writer.WriteEndElement() # graphml
|
|
65
|
+
self.writer.Flush()
|
|
66
|
+
self.writer.Close()
|
|
67
|
+
|
|
68
|
+
def visitAstNode(self, node):
|
|
69
|
+
node.forEachChild(self.visit)
|
|
70
|
+
|
|
71
|
+
def visitAstStatement(self, node):
|
|
72
|
+
self.writer.WriteStartElement("node")
|
|
73
|
+
self.writer.WriteAttributeString("id", str(node.id))
|
|
74
|
+
|
|
75
|
+
self.writer.WriteStartElement("data")
|
|
76
|
+
self.writer.WriteAttributeString("key", "d0") # Shape and label
|
|
77
|
+
self.writer.WriteStartElement("y:ShapeNode")
|
|
78
|
+
self.writer.WriteStartElement("y:NodeLabel")
|
|
79
|
+
self.writer.WriteString(str(node.lineNum) + ": "+ str(node.description) + ": " + ';'.join(node.entryPoints))
|
|
80
|
+
self.writer.WriteEndElement() # y:NodeLabel
|
|
81
|
+
self.writer.WriteStartElement("y:Shape")
|
|
82
|
+
if hasattr(node, "entryPoint"):
|
|
83
|
+
self.writer.WriteAttributeString("type", "hexagon")
|
|
84
|
+
elif isinstance(node, If) or isinstance(node, OnGoto):
|
|
85
|
+
self.writer.WriteAttributeString("type", "diamond")
|
|
86
|
+
else:
|
|
87
|
+
self.writer.WriteAttributeString("type", "roundrectangle")
|
|
88
|
+
self.writer.WriteEndElement() # y:Shape
|
|
89
|
+
self.writer.WriteEndElement() # y:ShapeNode
|
|
90
|
+
self.writer.WriteEndElement()
|
|
91
|
+
|
|
92
|
+
self.writer.WriteStartElement("data")
|
|
93
|
+
self.writer.WriteAttributeString("key", "d1") # description
|
|
94
|
+
self.writer.WriteString(node.description)
|
|
95
|
+
self.writer.WriteEndElement()
|
|
96
|
+
|
|
97
|
+
self.writer.WriteEndElement() # node
|
|
98
|
+
|
|
99
|
+
if self.out_edges:
|
|
100
|
+
for target in node.outEdges:
|
|
101
|
+
self.writer.WriteStartElement("edge")
|
|
102
|
+
self.writer.WriteAttributeString("source", str(node.id))
|
|
103
|
+
self.writer.WriteAttributeString("target", str(target.id))
|
|
104
|
+
self.writer.WriteStartElement("data")
|
|
105
|
+
self.writer.WriteAttributeString("key", "d3")
|
|
106
|
+
self.writer.WriteStartElement("y:PolyLineEdge")
|
|
107
|
+
self.writer.WriteStartElement("y:Arrows")
|
|
108
|
+
self.writer.WriteAttributeString("source", "none")
|
|
109
|
+
self.writer.WriteAttributeString("target", "standard")
|
|
110
|
+
self.writer.WriteEndElement() # y:Arrows
|
|
111
|
+
self.writer.WriteEndElement() # y:PolyLineEdge
|
|
112
|
+
self.writer.WriteEndElement() # data
|
|
113
|
+
self.writer.WriteEndElement() # edge
|
|
114
|
+
|
|
115
|
+
if self.in_edges:
|
|
116
|
+
for source in node.inEdges:
|
|
117
|
+
self.writer.WriteStartElement("edge")
|
|
118
|
+
self.writer.WriteAttributeString("source", str(source.id))
|
|
119
|
+
self.writer.WriteAttributeString("target", str(node.id))
|
|
120
|
+
self.writer.WriteStartElement("data")
|
|
121
|
+
self.writer.WriteAttributeString("key", "d3")
|
|
122
|
+
self.writer.WriteStartElement("y:PolyLineEdge")
|
|
123
|
+
self.writer.WriteStartElement("y:Arrows")
|
|
124
|
+
self.writer.WriteAttributeString("source", "none")
|
|
125
|
+
self.writer.WriteAttributeString("target", "standard")
|
|
126
|
+
self.writer.WriteEndElement() # y:Arrows
|
|
127
|
+
self.writer.WriteEndElement() # y:PolyLineEdge
|
|
128
|
+
self.writer.WriteEndElement() # data
|
|
129
|
+
self.writer.WriteEndElement() # edge
|
|
130
|
+
|
|
131
|
+
if self.back_edges:
|
|
132
|
+
for target in node.loopBackEdges:
|
|
133
|
+
self.writer.WriteStartElement("edge")
|
|
134
|
+
self.writer.WriteAttributeString("source", str(node.id))
|
|
135
|
+
self.writer.WriteAttributeString("target", str(target.id))
|
|
136
|
+
self.writer.WriteStartElement("data")
|
|
137
|
+
self.writer.WriteAttributeString("key", "d3")
|
|
138
|
+
self.writer.WriteStartElement("y:PolyLineEdge")
|
|
139
|
+
self.writer.WriteStartElement("y:Arrows")
|
|
140
|
+
self.writer.WriteAttributeString("source", "none")
|
|
141
|
+
self.writer.WriteAttributeString("target", "standard")
|
|
142
|
+
self.writer.WriteEndElement() # y:Arrows
|
|
143
|
+
self.writer.WriteEndElement() # y:PolyLineEdge
|
|
144
|
+
self.writer.WriteEndElement() # data
|
|
145
|
+
self.writer.WriteEndElement() # edge
|
|
146
|
+
|
|
147
|
+
node.forEachChild(self.visit)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
|
owl_basic/line_mapper.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
class LineMapper(object):
|
|
2
|
+
def __init__(self, physical_to_logical_map, line_to_stmt_map):
|
|
3
|
+
self.physical_to_logical_map = physical_to_logical_map
|
|
4
|
+
self.line_to_stmt_map = line_to_stmt_map
|
|
5
|
+
|
|
6
|
+
def physicalToLogical(self, physical_line_number):
|
|
7
|
+
if self.physical_to_logical_map is not None and physical_line_number is not None:
|
|
8
|
+
return self.physical_to_logical_map[physical_line_number]
|
|
9
|
+
else:
|
|
10
|
+
return physical_line_number
|
|
11
|
+
|
|
12
|
+
def logicalToPhysical(self, logical_line_number):
|
|
13
|
+
if self.physical_to_logical_map is not None:
|
|
14
|
+
|
|
15
|
+
return self.physical_to_logical_map.index(logical_line_number)
|
|
16
|
+
else:
|
|
17
|
+
return logical_line_number
|
|
18
|
+
|
|
19
|
+
def logicalStatement(self, logical_line_number):
|
|
20
|
+
physical_line_number = self.logicalToPhysical(logical_line_number)
|
|
21
|
+
#print "physical_line_number = %s" % physical_line_number
|
|
22
|
+
if physical_line_number in self.line_to_stmt_map:
|
|
23
|
+
return self.line_to_stmt_map[physical_line_number]
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
def firstStatement(self):
|
|
27
|
+
statement_lines = sorted([line for line in self.line_to_stmt_map.keys() if line is not None])
|
|
28
|
+
#for s in statement_lines:
|
|
29
|
+
# print "%s : %s" % (s, lnv.line_to_stmt[s])
|
|
30
|
+
first_statement_line = statement_lines[0]
|
|
31
|
+
#print "first_statement_line = %s" % first_statement_line
|
|
32
|
+
first_statement = self.line_to_stmt_map[first_statement_line]
|
|
33
|
+
return first_statement
|
|
34
|
+
|
|
35
|
+
def statementOnLine(self, integer_node):
|
|
36
|
+
'''
|
|
37
|
+
:param integer_node: A LiteralInteger node containing a logical line number
|
|
38
|
+
:returns: The first AstStatement node on that logical source code line
|
|
39
|
+
'''
|
|
40
|
+
logical_line_number = integer_node.value
|
|
41
|
+
statement = self.logicalStatement(logical_line_number)
|
|
42
|
+
#print "logical_line_number = %d, statement = %s" % (logical_line_number, statement)
|
|
43
|
+
return statement
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from owl_basic.visitor import Visitor
|
|
2
|
+
from owl_basic.ast_utils import *
|
|
3
|
+
|
|
4
|
+
class LineNumberVisitor(Visitor):
|
|
5
|
+
"""
|
|
6
|
+
This visitor builds a map from logical line numbers to the first statement
|
|
7
|
+
to appear on that line. It is used for resolving the targets of GOTO, GOSUB, etc.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
# TODO: This could be made into a much more useful in-order traversal visitor
|
|
11
|
+
|
|
12
|
+
def __init__(self):
|
|
13
|
+
# TODO: This should locate its own root, finding it on demand
|
|
14
|
+
self.line_to_stmt = {}
|
|
15
|
+
|
|
16
|
+
def registerStatement(self, statement):
|
|
17
|
+
if statement.lineNum is not None:
|
|
18
|
+
line_num = statement.lineNum
|
|
19
|
+
#print "line_num = %s" % line_num
|
|
20
|
+
if line_num not in self.line_to_stmt:
|
|
21
|
+
#print line_num, " ===> ", statement
|
|
22
|
+
self.line_to_stmt[line_num] = statement
|
|
23
|
+
|
|
24
|
+
def firstStatementOnLine(self, line_number):
|
|
25
|
+
return self.line_to_stmt[line_number]
|
|
26
|
+
|
|
27
|
+
def visitAstNode(self, node):
|
|
28
|
+
"Visit all children in order"
|
|
29
|
+
node.forEachChild(self.visit)
|
|
30
|
+
|
|
31
|
+
def visitAstStatement(self, statement):
|
|
32
|
+
self.registerStatement(statement)
|
|
33
|
+
|
|
34
|
+
def visitIf(self, iff):
|
|
35
|
+
"""
|
|
36
|
+
Process an IF statement. The true clause always precedes
|
|
37
|
+
the false clause so we process that first
|
|
38
|
+
"""
|
|
39
|
+
self.registerStatement(iff)
|
|
40
|
+
|
|
41
|
+
if iff.trueClause is not None:
|
|
42
|
+
if isinstance(iff.trueClause, list):
|
|
43
|
+
for node in iff.trueClause:
|
|
44
|
+
self.visit(node)
|
|
45
|
+
else:
|
|
46
|
+
self.visit(iff.trueClause)
|
|
47
|
+
|
|
48
|
+
if iff.falseClause is not None:
|
|
49
|
+
if isinstance(iff.falseClause, list):
|
|
50
|
+
for node in iff.falseClause:
|
|
51
|
+
self.visit(node)
|
|
52
|
+
else:
|
|
53
|
+
self.visit(iff.falseClause)
|
|
54
|
+
|
|
55
|
+
# TODO: Need BASIC IV and BASIC V statements here.
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
owl_basic/main.py
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
#!ipy
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
logging._srcfile = None
|
|
5
|
+
logging.basicConfig(level=logging.DEBUG,
|
|
6
|
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
|
7
|
+
|
|
8
|
+
# Python Standard Library
|
|
9
|
+
import sys
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import atexit
|
|
13
|
+
import StringIO
|
|
14
|
+
from optparse import OptionParser
|
|
15
|
+
|
|
16
|
+
from owl_basic.decoder import decode
|
|
17
|
+
from owl_basic.syntax import parser as syntax_parser
|
|
18
|
+
from owl_basic import xml_visitor
|
|
19
|
+
from owl_basic.source_debugging import SourceDebuggingVisitor
|
|
20
|
+
from owl_basic import parent_visitor
|
|
21
|
+
from owl_basic import separation_visitor
|
|
22
|
+
from owl_basic import simplify_visitor
|
|
23
|
+
from owl_basic import line_number_visitor
|
|
24
|
+
from owl_basic.flow import locateEntryPoints
|
|
25
|
+
from owl_basic.flow import createForwardControlFlowGraph
|
|
26
|
+
from owl_basic.flow import convertLongjumpsToExceptions
|
|
27
|
+
from owl_basic.flow import convertSubroutinesToProcedures
|
|
28
|
+
from owl_basic.flow import identifyBasicBlocks
|
|
29
|
+
from owl_basic.flow import orderBasicBlocks
|
|
30
|
+
from owl_basic.owltyping.typecheck import typecheck
|
|
31
|
+
from owl_basic import data_visitor
|
|
32
|
+
from owl_basic import gml_visitor
|
|
33
|
+
from owl_basic.xml_blocks import dumpXmlBlocks
|
|
34
|
+
from owl_basic.line_mapper import LineMapper
|
|
35
|
+
from owl_basic import symbol_table_visitor
|
|
36
|
+
from owl_basic.symbol_tables import SymbolTable
|
|
37
|
+
from owl_basic import correlation_visitor
|
|
38
|
+
from owl_basic.algorithms import all_indices
|
|
39
|
+
from owl_basic import process
|
|
40
|
+
|
|
41
|
+
def readFile(filename):
|
|
42
|
+
logging.debug("readFile")
|
|
43
|
+
# Read the file - processing it for line numbers if necessary
|
|
44
|
+
f = open(filename, 'rb')
|
|
45
|
+
data = f.read()
|
|
46
|
+
f.close()
|
|
47
|
+
return data
|
|
48
|
+
|
|
49
|
+
def detokenize(data, options):
|
|
50
|
+
logging.debug("detokenize")
|
|
51
|
+
if options.verbose:
|
|
52
|
+
sys.stderr.write("Detokenizing...")
|
|
53
|
+
|
|
54
|
+
detokenized = StringIO.StringIO()
|
|
55
|
+
decode(data, detokenized)
|
|
56
|
+
if options.verbose:
|
|
57
|
+
sys.stderr.write("done\n")
|
|
58
|
+
|
|
59
|
+
detokenHandle = StringIO.StringIO(detokenized.getvalue())
|
|
60
|
+
return detokenHandle
|
|
61
|
+
|
|
62
|
+
def indexLineNumbers(detokenHandle, options):
|
|
63
|
+
logging.debug("indexLineNumbers")
|
|
64
|
+
if options.verbose:
|
|
65
|
+
sys.stderr.write("Mapping physical to logical line numbers... ")
|
|
66
|
+
|
|
67
|
+
line_number_regex = re.compile(r'(\s*\d+\s*)(.*)') # TODO: Factor this out of here and decoder
|
|
68
|
+
physical_line = 0
|
|
69
|
+
logical_line = 0
|
|
70
|
+
physical_to_logical_map = []
|
|
71
|
+
line_bodies = []
|
|
72
|
+
line_number_prefixes = []
|
|
73
|
+
#line_offsets = [] # Offsets to the start of the line
|
|
74
|
+
while True:
|
|
75
|
+
#line_offsets.append(detokenHandle.tell())
|
|
76
|
+
line = detokenHandle.readline()
|
|
77
|
+
if not line:
|
|
78
|
+
break
|
|
79
|
+
m = line_number_regex.match(line)
|
|
80
|
+
if not m:
|
|
81
|
+
raise CompileException("Missing line number at physical line %d (after logical line %d)" % (physical_line, logical_line))
|
|
82
|
+
|
|
83
|
+
logical_line_string = m.group(1)
|
|
84
|
+
logical_line = int(logical_line_string)
|
|
85
|
+
line_number_prefix_length = len(logical_line_string)
|
|
86
|
+
line_number_prefixes.append(line_number_prefix_length)
|
|
87
|
+
physical_to_logical_map.append(logical_line)
|
|
88
|
+
line_bodies.append(m.group(2))
|
|
89
|
+
physical_line += 1
|
|
90
|
+
|
|
91
|
+
data = '\n'.join(line_bodies)
|
|
92
|
+
detokenHandle.close()
|
|
93
|
+
cr_indices = all_indices(data, '\n')
|
|
94
|
+
line_indices = [index + 1 for index in cr_indices]
|
|
95
|
+
line_offsets = [0]
|
|
96
|
+
line_offsets.extend(line_indices)
|
|
97
|
+
#for i, offset in enumerate(line_offsets):
|
|
98
|
+
# print "%d ==> %s" % (i, data[offset:offset + 5])
|
|
99
|
+
#print data
|
|
100
|
+
return data, physical_to_logical_map, line_offsets , line_number_prefixes
|
|
101
|
+
|
|
102
|
+
def warnOnMissingNewline(data):
|
|
103
|
+
logging.debug("warnOnMissingNewline")
|
|
104
|
+
if not data.endswith('\n'):
|
|
105
|
+
logging.warning("Missing newline at end of file")
|
|
106
|
+
data += '\n'
|
|
107
|
+
return data
|
|
108
|
+
|
|
109
|
+
def setSourceDebugging(data, line_offsets, line_number_prefixes, parse_tree):
|
|
110
|
+
logging.debug("Set source debugging")
|
|
111
|
+
# Read through the data and set character column information
|
|
112
|
+
sdv = SourceDebuggingVisitor(data, line_offsets, line_number_prefixes)
|
|
113
|
+
parse_tree.accept(sdv)
|
|
114
|
+
|
|
115
|
+
def setParents(parse_tree, options):
|
|
116
|
+
logging.debug("setParents")
|
|
117
|
+
if options.verbose:
|
|
118
|
+
sys.stderr.write("Setting parents... ")
|
|
119
|
+
|
|
120
|
+
parse_tree.accept(parent_visitor.ParentVisitor())
|
|
121
|
+
if options.verbose:
|
|
122
|
+
sys.stderr.write("done\n")
|
|
123
|
+
|
|
124
|
+
def splitComplexNodes(parse_tree, options):
|
|
125
|
+
logging.debug("splitComplexNodes")
|
|
126
|
+
if options.verbose:
|
|
127
|
+
sys.stderr.write("Separating complex Abstract Syntax Tree nodes... ")
|
|
128
|
+
|
|
129
|
+
parse_tree.accept(separation_visitor.SeparationVisitor())
|
|
130
|
+
if options.verbose:
|
|
131
|
+
sys.stderr.write("done\n")
|
|
132
|
+
|
|
133
|
+
def simplifyAst(parse_tree, options):
|
|
134
|
+
logging.debug("simplifyAst")
|
|
135
|
+
if options.verbose:
|
|
136
|
+
sys.stderr.write("Simplifying Abstract Syntax Tree... ")
|
|
137
|
+
|
|
138
|
+
parse_tree.accept(simplify_visitor.SimplificationVisitor())
|
|
139
|
+
if options.verbose:
|
|
140
|
+
sys.stderr.write("done\n")
|
|
141
|
+
|
|
142
|
+
def createLineMapper(parse_tree, physical_to_logical_map):
|
|
143
|
+
logging.debug("createLineMapper")
|
|
144
|
+
lnv = line_number_visitor.LineNumberVisitor()
|
|
145
|
+
parse_tree.accept(lnv)
|
|
146
|
+
line_mapper = LineMapper(physical_to_logical_map, lnv.line_to_stmt)
|
|
147
|
+
return line_mapper
|
|
148
|
+
|
|
149
|
+
def dumpXmlAst(parse_tree, output_filename, options):
|
|
150
|
+
logging.debug("dumpXmlAst")
|
|
151
|
+
if options.use_clr:
|
|
152
|
+
if options.verbose:
|
|
153
|
+
sys.stderr.write("Creating XML AST... ")
|
|
154
|
+
|
|
155
|
+
xmlAst(parse_tree, output_filename)
|
|
156
|
+
if options.verbose:
|
|
157
|
+
sys.stderr.write("done\n")
|
|
158
|
+
|
|
159
|
+
def correlateLoops(entry_points, options):
|
|
160
|
+
logging.debug("correlateLoops")
|
|
161
|
+
if options.verbose:
|
|
162
|
+
sys.stderr.write("Correlate opening and closing loop constructs")
|
|
163
|
+
|
|
164
|
+
for entry_point in entry_points.values():
|
|
165
|
+
# Depth first search from this entry point through the CFG
|
|
166
|
+
# maintaining a stack of loops as we go. Mark nodes that we
|
|
167
|
+
cv = correlation_visitor.CorrelationVisitor()
|
|
168
|
+
cv.start(entry_point)
|
|
169
|
+
|
|
170
|
+
def buildSymbolTables(entry_points, options):
|
|
171
|
+
logging.debug("buildSymbolTables")
|
|
172
|
+
# Attach symbol tables to each statement
|
|
173
|
+
if options.verbose:
|
|
174
|
+
sys.stderr.write("Building symbol tables")
|
|
175
|
+
|
|
176
|
+
stv = symbol_table_visitor.SymbolTableVisitor()
|
|
177
|
+
|
|
178
|
+
# Set the global symbol table for the main program entry point, if there is one
|
|
179
|
+
if '__owl__main' in entry_points:
|
|
180
|
+
entry_points['__owl__main'].symbolTable = stv.globalSymbols
|
|
181
|
+
|
|
182
|
+
for entry_point in entry_points.values():
|
|
183
|
+
entry_point.accept(stv)
|
|
184
|
+
|
|
185
|
+
for table in SymbolTable.symbol_tables:
|
|
186
|
+
title = "Symbol table '%s'" % table.name
|
|
187
|
+
if table.parent is not None:
|
|
188
|
+
parent_title = " with parent '%s'" % table.parent.name
|
|
189
|
+
else:
|
|
190
|
+
parent_title = "is the root symbol table"
|
|
191
|
+
width = max(len(title), len(parent_title))
|
|
192
|
+
print("-" * width)
|
|
193
|
+
print(title)
|
|
194
|
+
print(parent_title)
|
|
195
|
+
print("-" * width)
|
|
196
|
+
|
|
197
|
+
symbols = table.symbols.keys()
|
|
198
|
+
symbols.sort()
|
|
199
|
+
for symbol in symbols:
|
|
200
|
+
print("%-10s %-10s %s" % (symbol, table.symbols[symbol].type.__doc__, table.symbols[symbol].modifier))
|
|
201
|
+
print("-" * width)
|
|
202
|
+
print()
|
|
203
|
+
return stv
|
|
204
|
+
|
|
205
|
+
def extractData(parse_tree, options):
|
|
206
|
+
"""
|
|
207
|
+
Extract all information from DATA statements
|
|
208
|
+
"""
|
|
209
|
+
# All DATA is stored as strings, and is converted at run-time by the
|
|
210
|
+
# READ statement
|
|
211
|
+
logging.debug("extracting DATA")
|
|
212
|
+
dv = data_visitor.DataVisitor()
|
|
213
|
+
parse_tree.accept(dv)
|
|
214
|
+
return dv
|
|
215
|
+
|
|
216
|
+
def dumpXmlCfg(parse_tree, filename, options):
|
|
217
|
+
logging.debug("dumpXmlCfg")
|
|
218
|
+
if options.use_clr:
|
|
219
|
+
if options.verbose:
|
|
220
|
+
sys.stderr.write("Creating XML CFG... ")
|
|
221
|
+
xmlCfg(parse_tree, filename)
|
|
222
|
+
if options.verbose:
|
|
223
|
+
sys.stderr.write("done\n")
|
|
224
|
+
|
|
225
|
+
def xmlAst(tree, filename):
|
|
226
|
+
xmlv = xml_visitor.XmlVisitor(filename)
|
|
227
|
+
tree.accept(xmlv)
|
|
228
|
+
xmlv.close()
|
|
229
|
+
|
|
230
|
+
def xmlCfg(tree, filename):
|
|
231
|
+
gmlv = gml_visitor.GmlVisitor(filename)
|
|
232
|
+
tree.accept(gmlv)
|
|
233
|
+
gmlv.close()
|
|
234
|
+
|
|
235
|
+
class Usage(Exception):
|
|
236
|
+
def __init__(self, msg):
|
|
237
|
+
self.msg = msg
|
|
238
|
+
|
|
239
|
+
class CompileException(Exception):
|
|
240
|
+
def __init__(self, msg):
|
|
241
|
+
self.msg = msg
|
|
242
|
+
|
|
243
|
+
def main(argv=None):
|
|
244
|
+
if argv is None:
|
|
245
|
+
argv = sys.argv
|
|
246
|
+
try:
|
|
247
|
+
usage = "usage: %prog [options] source-file"
|
|
248
|
+
version = "%prog 0.5"
|
|
249
|
+
parser = OptionParser(usage=usage, version=version)
|
|
250
|
+
parser.add_option("-x", "--debug-lex", action='store_true', dest='debug_lex', default=False)
|
|
251
|
+
parser.add_option("-c", "--debug-no-clr", action='store_false', dest='use_clr', default=(sys.platform == 'cli'))
|
|
252
|
+
parser.add_option("-v", "--verbose", action='store_true', dest='verbose', default=False)
|
|
253
|
+
parser.add_option("-i", "--il", action='store_true', dest='create_il', default=False)
|
|
254
|
+
parser.add_option("-p", "--peverify", action='store_true', dest='peverify', default=False)
|
|
255
|
+
|
|
256
|
+
(options, args) = parser.parse_args()
|
|
257
|
+
if len(args) != 1:
|
|
258
|
+
parser.error("No source file name supplied")
|
|
259
|
+
compile(args[0], options)
|
|
260
|
+
except Usage as err:
|
|
261
|
+
parser.err(err.msg)
|
|
262
|
+
return 2
|
|
263
|
+
except CompileException as err:
|
|
264
|
+
print(err.msg, file=sys.stderr)
|
|
265
|
+
return 1
|
|
266
|
+
|
|
267
|
+
def compile(filename, options):
|
|
268
|
+
if options.use_clr:
|
|
269
|
+
# .NET Framework
|
|
270
|
+
import clr
|
|
271
|
+
clr.AddReference('System.Xml')
|
|
272
|
+
from System.Xml import XmlTextWriter, Formatting
|
|
273
|
+
|
|
274
|
+
if not options.use_clr:
|
|
275
|
+
# TODO: Use non-recursive code for the flowgraph
|
|
276
|
+
sys.setrecursionlimit(2000)
|
|
277
|
+
|
|
278
|
+
data = readFile(filename)
|
|
279
|
+
detokenHandle = detokenize(data, options)
|
|
280
|
+
data, physical_to_logical_map, line_offsets, line_number_prefixes = indexLineNumbers(detokenHandle, options)
|
|
281
|
+
data = warnOnMissingNewline(data)
|
|
282
|
+
|
|
283
|
+
parse_tree = syntax_parser.parse(data, options)
|
|
284
|
+
setSourceDebugging(data, line_offsets, line_number_prefixes, parse_tree)
|
|
285
|
+
setParents(parse_tree, options)
|
|
286
|
+
splitComplexNodes(parse_tree, options)
|
|
287
|
+
simplifyAst(parse_tree, options)
|
|
288
|
+
line_mapper = createLineMapper(parse_tree, physical_to_logical_map)
|
|
289
|
+
dv = extractData(parse_tree, options)
|
|
290
|
+
createForwardControlFlowGraph(parse_tree, line_mapper, options)
|
|
291
|
+
entry_points = locateEntryPoints(parse_tree, line_mapper, options)
|
|
292
|
+
convertLongjumpsToExceptions(parse_tree, line_mapper, options)
|
|
293
|
+
convertSubroutinesToProcedures(parse_tree, entry_points, line_mapper, options)
|
|
294
|
+
correlateLoops(entry_points, options)
|
|
295
|
+
basic_blocks = identifyBasicBlocks(entry_points, options)
|
|
296
|
+
ordered_basic_blocks = orderBasicBlocks(basic_blocks, options)
|
|
297
|
+
typecheck(parse_tree, entry_points, options)
|
|
298
|
+
# Array visitor - rewrite array expressions
|
|
299
|
+
stv = buildSymbolTables(entry_points, options)
|
|
300
|
+
|
|
301
|
+
dumpXmlAst(parse_tree, filename + "_ast.xml", options)
|
|
302
|
+
dumpXmlCfg(parse_tree, filename + "_cfg.graphml", options)
|
|
303
|
+
dumpXmlBlocks(basic_blocks, filename + "_blocks.graphml", options)
|
|
304
|
+
|
|
305
|
+
output_name = os.path.splitext(os.path.basename(filename))[0]
|
|
306
|
+
source_path = os.path.abspath(filename)
|
|
307
|
+
|
|
308
|
+
if options.use_clr:
|
|
309
|
+
from owl_basic.codegen.clr.generate import AssemblyGenerator
|
|
310
|
+
ag = AssemblyGenerator(line_mapper)
|
|
311
|
+
exe_filename = ag.generateAssembly(source_path, output_name, stv.globalSymbols, dv, ordered_basic_blocks)
|
|
312
|
+
if options.peverify:
|
|
313
|
+
# Run PEVerify on the resulting executable
|
|
314
|
+
logging.debug("Verifying")
|
|
315
|
+
peverify_exe = r'C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\PEVerify.exe'
|
|
316
|
+
process.execute(peverify_exe, exe_filename)
|
|
317
|
+
if options.create_il:
|
|
318
|
+
# Create debuggable CIL files by disassebling and reassembling the
|
|
319
|
+
# executable
|
|
320
|
+
# Run ILDASM on the produced file
|
|
321
|
+
logging.debug("Disassembling to CIL")
|
|
322
|
+
|
|
323
|
+
ildasm_exe = r"C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\x64\ildasm.exe"
|
|
324
|
+
logging.debug("ILDAsm.exe from %s", ildasm_exe)
|
|
325
|
+
il_filename = exe_filename[:-3] + 'il'
|
|
326
|
+
process.execute(ildasm_exe, '/OUT=%s' % il_filename, exe_filename)
|
|
327
|
+
|
|
328
|
+
logging.debug("Reassembling with CIL debug info")
|
|
329
|
+
clr.AddReference("Microsoft.Build.Utilities")
|
|
330
|
+
from Microsoft.Build.Utilities import ToolLocationHelper, TargetDotNetFrameworkVersion
|
|
331
|
+
ilasm_exe = ToolLocationHelper.GetPathToDotNetFrameworkFile("ILAsm.exe", TargetDotNetFrameworkVersion.VersionLatest)
|
|
332
|
+
logging.debug("ILAsm.exe from %s", ilasm_exe)
|
|
333
|
+
process.execute(ilasm_exe, '/EXE', '/DEBUG', il_filename)
|
|
334
|
+
|
|
335
|
+
# Structural analysis
|
|
336
|
+
|
|
337
|
+
#splitBasicBlock(parse_tree)
|
|
338
|
+
|
|
339
|
+
# Type checking and casting
|
|
340
|
+
# Have we finished type-checking?
|
|
341
|
+
# == Optimisation ==
|
|
342
|
+
# Remove basic blocks containing only GOTO. Is this the same as replacing all GOTOs
|
|
343
|
+
# with edges in the CFG?
|
|
344
|
+
# constant folding
|
|
345
|
+
# constant propagation
|
|
346
|
+
# Note - during constant propagation we can
|
|
347
|
+
# eliminate some redundant casts by changing the
|
|
348
|
+
# type of the constant at compile time
|
|
349
|
+
# for example A = 5 can become A = 5.0 and
|
|
350
|
+
# A% = 6.0 can become A% = 6
|
|
351
|
+
# String Concatenation using the different forms
|
|
352
|
+
# of String.Concat
|
|
353
|
+
# eliminate locals
|
|
354
|
+
# static single assignment form
|
|
355
|
+
# TODO: Inline single-entry GOSUB
|
|
356
|
+
# Optimise by combining conditions and branches better
|
|
357
|
+
# rather than relying on Brtrue and Brfalse.
|
|
358
|
+
#
|
|
359
|
+
# Trace back from RETURN statements - if only one
|
|
360
|
+
# GOSUB is reached - move the code in the GOSUB to the call site
|
|
361
|
+
# of the GOSUB
|
|
362
|
+
# Locate nodes with no inbound edges and trace unreachable code
|
|
363
|
+
# from them. Remove unreachable code from the CFG and the AST. This
|
|
364
|
+
# will need to be traced from program and procedure entry points
|
|
365
|
+
|
|
366
|
+
# TODO: Replace Goto -> ReturnFromProcedure with ReturnFromProcedure
|
|
367
|
+
#elimiateCommonSubexpressions(parse_tree opti
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def printProfile():
|
|
372
|
+
import clr
|
|
373
|
+
for p in sorted(clr.GetProfilerData(), key=lambda p: p.ExclusiveTime):
|
|
374
|
+
print('%s\t%d\t%d\t%d' % (p.Name, p.InclusiveTime, p.ExclusiveTime, p.Calls))
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
if __name__ == "__main__":
|
|
378
|
+
import clr
|
|
379
|
+
clr.EnableProfiler(False)
|
|
380
|
+
atexit.register(printProfile)
|
|
381
|
+
sys.exit(main())
|
owl_basic/node.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
class Node(object):
|
|
2
|
+
def __init__(self, nodeType=None, formalType=None, description="A parameter"):
|
|
3
|
+
self._node_type = nodeType
|
|
4
|
+
self._formal_type = formalType
|
|
5
|
+
self._description = description
|
|
6
|
+
|
|
7
|
+
def _getNodeType(self):
|
|
8
|
+
return self._node_type
|
|
9
|
+
|
|
10
|
+
nodeType = property(_getNodeType)
|
|
11
|
+
|
|
12
|
+
def _getFormalType(self):
|
|
13
|
+
return self._formal_type
|
|
14
|
+
|
|
15
|
+
formalType = property(_getFormalType)
|
|
16
|
+
|
|
17
|
+
def _getDescription(self):
|
|
18
|
+
return self._description
|
|
19
|
+
|
|
20
|
+
description = property(_getDescription)
|
|
21
|
+
|
owl_basic/options.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class Option(object):
|
|
2
|
+
pass
|
|
3
|
+
|
|
4
|
+
class BoolOption(Option):
|
|
5
|
+
def __init__(self, default=None):
|
|
6
|
+
self.value = default
|
|
7
|
+
|
|
8
|
+
class IntegerOption(Option):
|
|
9
|
+
def __init__(self, default=None):
|
|
10
|
+
self.value = default
|
|
11
|
+
|
|
12
|
+
class FloatOption(Option):
|
|
13
|
+
def __init__(self, default=None):
|
|
14
|
+
self.value = default
|
|
15
|
+
|
|
16
|
+
class StringOption(Option):
|
|
17
|
+
def __init__(self, default=None):
|
|
18
|
+
self.value = default
|
|
19
|
+
|
|
20
|
+
class TypeOption(Option):
|
|
21
|
+
def __init__(self, default=None):
|
|
22
|
+
self.value = default
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|