codeanalyzer-python 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

codeanalyzer/dataflow/scalpel/cfg/model.py

@@ -0,0 +1,331 @@
+"""
+Control flow graph for Python programs.
+"""
+
+import ast
+import sys 
+import re
+import token
+import tokenize 
+import astor
+
+try:  # PATCH (codeanalyzer): graphviz is only used by build_visual(), which
+    import graphviz as gv  # codeanalyzer never calls. Keep the module importable
+except ImportError:  # without the graphviz dependency.
+    gv = None
+
+__all__ = ["Block", "Link", "CFG"]
+
+class Block(object):
+    """
+    Basic block in a control flow graph.
+
+    Contains a list of statements executed in a program without any control
+    jumps. A block of statements is exited through one of its exits. Exits are
+    a list of Links that represent control flow jumps.
+    """
+
+    __slots__ = ["id", "statements", "func_calls", "predecessors", "exits"]
+
+    def __init__(self, id):
+        # Id of the block.
+        self.id = id
+        # Statements in the block.
+        self.statements = []
+        # Calls to functions inside the block (represents context switches to
+        # some functions' CFGs).
+        self.func_calls = []
+        # Links to predecessors in a control flow graph.
+        self.predecessors = []
+        # Links to the next blocks in a control flow graph.
+        self.exits = []
+
+    def __del__(self):
+        self.statements.clear()
+        self.func_calls.clear()
+        self.predecessors.clear()
+        self.exits.clear()
+
+    def __str__(self):
+        if self.statements:
+            return "block:{}@{}".format(self.id, self.at())
+        return "empty block:{}".format(self.id)
+
+    def __repr__(self):
+        txt = "{} with {} exits".format(str(self), len(self.exits))
+        if self.statements:
+            txt += ", body=["
+            txt += ", ".join([ast.dump(node) for node in self.statements])
+            txt += "]"
+        return txt
+
+    def at(self):
+        """
+        Get the line number of the first statement of the block in the program.
+        """
+        if self.statements and self.statements[0].lineno >= 0:
+            return self.statements[0].lineno
+        return None
+
+    def is_empty(self):
+        """
+        Check if the block is empty.
+        Returns:
+            A boolean indicating if the block is empty (True) or not (False).
+        """
+        return len(self.statements) == 0
+    '''
+    def strip_comment(self, src):
+        clean_src = ""
+
+        prev_toktype = token.INDENT
+        first_line = None
+        last_lineno = -1
+        last_col = 0
+
+        tokgen = tokenize.generate_tokens(src)
+        for toktype, ttext, (slineno, scol), (elineno, ecol), ltext in tokgen:
+            if 0:   # Change to if 1 to see the tokens fly by.
+                print("%10s %-14s %-20r %r" % (
+                    tokenize.tok_name.get(toktype, toktype),
+                    "%d.%d-%d.%d" % (slineno, scol, elineno, ecol),
+                    ttext, ltext
+                    ))
+            if slineno > last_lineno:
+                last_col = 0
+            if scol > last_col:
+                mod.write(" " * (scol - last_col))
+            if toktype == token.STRING and prev_toktype == token.INDENT:
+                # Docstring
+                mod.write("#--")
+            elif toktype == tokenize.COMMENT:
+                # Comment
+                mod.write("##\n")
+            else:
+                mod.write(ttext)
+            prev_toktype = toktype
+            last_col = ecol
+            last_lineno = elineno
+        '''
+    def get_source(self):
+        """
+        Get a string containing the Python source code corresponding to the
+        statements in the block.
+        Returns:
+            A string containing the source code of the statements.
+        """
+        src = "#" + str(self.id)+'\n'
+        for statement in self.statements:
+            if type(statement) in [ast.If, ast.For, ast.While, ast.With]:
+                src += (astor.to_source(statement)).split('\n')[0] + "\n"
+            elif type(statement) == ast.Try:
+                src += (astor.to_source(statement)).split('\n')[0] + "\n"
+            #elif type(statement) == ast.If:
+            #    src += (astor.to_source(statement)).split('\n')[0] + "\n"
+            elif type(statement) in [ast.FunctionDef,ast.AsyncFunctionDef,
+                    ast.ClassDef]:
+                src += (astor.to_source(statement)).split('\n')[0] + "...\n"
+            elif type(statement) == ast.ClassDef:
+                src += (astor.to_source(statement)).split('\n')[0] + "...\n"
+            else:
+                src += astor.to_source(statement)
+        return src
+
+    def get_calls(self):
+        """
+        Get a string containing the calls to other functions inside the block.
+
+        Returns:
+            A string containing the names of the functions called inside the
+            block.
+        """
+        txt = ""
+        for func_call_entry in self.func_calls:
+            txt += func_call_entry['name'] + '\n'
+        return txt
+
+
+class Link(object):
+    """
+    Link between blocks in a control flow graph.
+
+    Represents a control flow jump between two blocks. Contains an exitcase in
+    the form of an expression, representing the case in which the associated
+    control jump is made.
+    """
+
+    __slots__ = ["source", "target", "exitcase"]
+
+    def __init__(self, source, target, exitcase=None):
+        assert type(source) == Block, "Source of a link must be a block"
+        assert type(target) == Block, "Target of a link must be a block"
+        # Block from which the control flow jump was made.
+        self.source = source
+        # Target block of the control flow jump.
+        self.target = target
+        # 'Case' leading to a control flow jump through this link.
+        self.exitcase = exitcase
+
+    def __str__(self):
+        return "link from {} to {}".format(str(self.source), str(self.target))
+
+    def __repr__(self):
+        if self.exitcase is not None:
+            return "{}, with exitcase {}".format(str(self),
+                                                 ast.dump(self.exitcase))
+        return str(self)
+
+    def get_exitcase(self):
+        """
+        Get a string containing the Python source code corresponding to the
+        exitcase of the Link.
+
+        Returns:
+            A string containing the source code.
+        """
+        if self.exitcase:
+            return astor.to_source(self.exitcase)
+        return ""
+    def __del__(self):
+        self.source = None
+        # Target block of the control flow jump.
+        self.target = None
+        # 'Case' leading to a control flow jump through this link.
+        self.exitcase = None
+
+
+
+class CFG(object):
+    """
+    Control flow graph (CFG).
+
+    A control flow graph is composed of basic blocks and links between them
+    representing control flow jumps. It has a unique entry block and several
+    possible 'final' blocks (blocks with no exits representing the end of the
+    CFG).
+    """
+    def __init__(self, name, asynchr=False):
+        """
+        The constructor of CFG class. Only name of this graph is required. 
+        """
+        assert type(name) == str, "Name of a CFG must be a string"
+        assert type(asynchr) == bool, "Async must be a boolean value"
+        # Name of the function or module being represented.
+        self.name = name
+        # Type of function represented by the CFG (sync or async). A Python
+        # program is considered as a synchronous function (main).
+        self.asynchr = asynchr
+        # Entry block of the CFG.
+        self.entryblock = None
+        # Final blocks of the CFG.
+        self.finalblocks = []
+        # Sub-CFGs for functions defined inside the current CFG.
+        self.functioncfgs = {}
+        self.class_cfgs = {}
+
+        self.function_args = {}
+        self.class_args = {}
+
+    def __del__(self):
+        pass 
+    
+    def __str__(self):
+        return "CFG for {}".format(self.name)
+
+    def remove_comments(self, src):
+        pass
+
+    
+    
+    def get_all_blocks(self):
+        """
+        Get a list of code blocks in this CFG; This is generated by BFS order.
+
+        Returns:
+            A list of code blocks. 
+        """
+        import queue
+        all_blocks = []
+        
+        visited = set()
+        working_queue = queue.Queue() 
+        working_queue.put(self.entryblock)
+
+        while not working_queue.empty():
+            block = working_queue.get()
+            # this block has been visited 
+            if block.id in visited:
+                continue
+            all_blocks.append(block)
+            visited.add(block.id)
+            for suc_link in block.exits:
+                if suc_link.target.id not in visited:
+                    working_queue.put(suc_link.target)
+        return all_blocks
+        #def dfs(start_block):
+        #    # non-recurisve implementation of DFS search
+   
+    def __iter__(self):
+        """
+        Generator that yields all the blocks in the current graph, then
+        recursively yields from any sub graphs
+        """
+        visited = set()
+        to_visit = [self.entryblock]
+
+        while to_visit:
+            block = to_visit.pop(0)
+            visited.add(block)
+            for exit_ in block.exits:
+                if exit_.target in visited or exit_.target in to_visit:
+                    continue
+                to_visit.append(exit_.target)
+            yield block
+
+        for subcfg in self.functioncfgs.values():
+            yield from subcfg
+    
+    def _visit_blocks(self, graph, block, visited=[], calls=True):
+        # Don't visit blocks twice.
+        if block.id in visited:
+            return
+
+        nodelabel = block.get_source()
+        graph.node(str(block.id), label=nodelabel)
+
+        visited.append(block.id)
+        # Show the block's function calls in a node.
+        if calls and block.func_calls:
+            calls_node = str(block.id)+"_calls"
+            calls_label = block.get_calls().strip()
+            graph.node(calls_node, label=calls_label,
+                       _attributes={'shape': 'box'})
+            graph.edge(str(block.id), calls_node, label="calls",
+                       _attributes={'style': 'dashed'})
+        # Recursively visit all the blocks of the CFG.
+        for exit in block.exits:
+            self._visit_blocks(graph, exit.target, visited, calls=calls)
+            edgelabel = exit.get_exitcase().strip()
+            graph.edge(str(block.id), str(exit.target.id), label=edgelabel)
+
+    def _build_visual(self, format='pdf', calls=True):
+        graph = gv.Digraph(name='cluster'+self.name, format=format,
+                           graph_attr={'label': self.name})
+        self._visit_blocks(graph, self.entryblock, visited=[], calls=False)
+        return graph
+
+    def build_visual(self, format, calls=True, show=True):
+        """
+        Build a visualisation of the CFG with graphviz and output it in a DOT
+        file.
+
+        Args:
+            filename: The name of the output file in which the visualisation
+                      must be saved.
+            format: The format to use for the output file (PDF, ...).
+            show: A boolean indicating whether to automatically open the output
+                  file after building the visualisation.
+        """
+        graph = self._build_visual(format, calls)
+        return graph
+

codeanalyzer/dataflow/scalpel/core/func_call_visitor.py

@@ -0,0 +1,234 @@
+import sys
+import ast
+from collections import deque
+from ast import NodeVisitor
+from copy import deepcopy
+
+
+def is_py38_or_higher():
+    if sys.version_info.major == 3 and sys.version_info.minor >= 8:
+        return True
+    return False
+
+
+NAMECONSTANT_TYPE = ast.Constant if is_py38_or_higher() else ast.NameConstant
+
+
+class CallTransformer(ast.NodeTransformer):
+    def __init__(self):
+        self.call_names = []
+
+    def visit_Attribute(self, node):
+        #        self.generic_visit(node.value)
+        return node
+
+    def param2str(self, param):
+
+        def get_func(node):
+            if type(node) == ast.Name:
+                return node.id
+            elif type(node) == ast.Constant:
+                # ingore such as  "this is a constant".join()
+                return ""
+            elif type(node) == ast.BinOp:
+                # ingore such as  (a+b+c).fun()
+                return ""
+            elif type(node) == ast.Str:
+                # ingore such as  "xxx".fun()
+                return ""
+            elif type(node) == ast.JoinedStr:
+                # ingore such as  "xxx".fun()
+                return ""
+            elif type(node) == ast.Bytes:
+                # ingore such as  "xxx".fun()
+                return ""
+            elif type(node) == ast.Compare:
+                # example "(x.matrix_exp() == torch.eye(20, 20, dtype=dtype, device=device)).all().item()"
+                # tests/test-cases/cfg-tests/pytorch-test-test_linalg.py
+                # ignore for now
+                return ""
+            elif type(node) == ast.Subscript:
+                # currently, we will ignore the slices because we cannot track the type of the value.
+                # for instance,  a[something].fun() ->  a.fun()
+                # this sacrifice
+                return get_func(node.value)  
+            #elif type(node) == ast.JoinedStr:
+            #    return ""
+            elif type(node) == ast.Attribute:
+                if type(node.value) in [ast.JoinedStr, ast.Constant]:
+                    return node.attr
+                else:
+                    return get_func(node.value) + "." + node.attr 
+            elif type(node) == ast.Call:
+                return get_func(node.func)
+            elif type(node) == ast.IfExp:
+                return ""
+            elif type(node) == ast.Compare:
+                return ""
+            elif type(node) == ast.UnaryOp:
+                return ""
+            #ast.UnaryOp
+            else:
+                #import astor
+                #print(astor.to_source(node))
+                raise Exception(str(type(node)))
+    
+        if isinstance(param, ast.Subscript):
+            return self.param2str(param.value)
+        if isinstance(param, ast.Call):
+            return get_func(param)
+        elif isinstance(param, ast.Name):
+            return param.id
+        elif isinstance(param, ast.Num):
+            # python 3.6  
+            return param.n
+            #return param.value
+        elif isinstance(param, ast.List):
+            return "List"
+        elif isinstance(param, ast.ListComp):
+            return "List"
+        elif isinstance(param, ast.Tuple):
+            return "Tuple"
+        elif isinstance(param, (ast.Dict, ast.DictComp)):
+            return "Dict"
+        elif isinstance(param, (ast.Set, ast.SetComp)):
+            return "Set"
+        elif isinstance(param, ast.Str):
+            return param.s
+        elif isinstance(param, ast.NameConstant):
+            return param.value
+        elif isinstance(param, ast.Constant):
+            return param.value
+        elif isinstance(param, ast.Expr):
+            return "Expr"
+        else:
+            return "unknown"
+
+    def visit_Call(self, node):
+
+        tmp_fun_node = deepcopy(node)
+        tmp_fun_node.args = []
+        tmp_fun_node.keywords = []
+
+        callvisitor = FuncCallVisitor()
+        callvisitor.visit(tmp_fun_node)
+
+        call_info = {"name": callvisitor.name,
+                     "lineno": tmp_fun_node.lineno,
+                     "col_offset": tmp_fun_node.col_offset,
+                     "params": []
+                     }
+        self.call_names += [call_info]
+        for arg in node.args:
+            call_info["params"] += [self.param2str(arg)]
+            self.generic_visit(arg)
+
+        for kw in node.keywords:
+            call_info["params"] += [self.param2str(kw.value)]
+            self.generic_visit(kw)
+        self.generic_visit(tmp_fun_node)
+
+        return node
+
+
+class FuncCallVisitor(ast.NodeVisitor):
+    def __init__(self):
+        self._name = deque()
+        self.call_names = []
+
+    def clear(self):
+        self._name = deque()
+        self.call_names = []
+
+    @property
+    def name(self):
+        return '.'.join(self._name)
+
+    @name.deleter
+    def name(self):
+        self._name.clear()
+
+    def visit_Name(self, node):
+        self._name.appendleft(node.id)
+
+    def visit_Attribute(self, node):
+
+        try:
+            self._name.appendleft(node.attr)
+            self._name.appendleft(node.value.id)
+        except AttributeError as e:
+            self.generic_visit(node)
+
+    def visit_Call(self, node):
+        node.args = []
+        node.keywords = []
+        self.generic_visit(node)
+        return node
+
+
+    def visit_Subscript(self, node):
+        # ingore subscription slice 
+        self.visit(node.value)
+        return node
+
+def get_args(node):
+    arg_type = []
+    for arg in node.args:
+        if isinstance(arg, ast.Name):
+            arg_type.append(arg.id)
+        elif isinstance(arg, ast.Num):
+            arg_type.append("Num")
+        elif isinstance(arg, ast.List):
+            arg_type.append("List")
+        elif isinstance(arg, ast.ListComp):
+            arg_type.append("List")
+        elif isinstance(arg, ast.Tuple):
+            arg_type.append("Tuple")
+        elif isinstance(arg, ast.Dict):
+            arg_type.append("Dict")
+        elif isinstance(arg, ast.DictComp):
+            arg_type.append("Dict")
+        elif isinstance(arg, ast.Set):
+            arg_type.append("Set")
+        elif isinstance(arg, ast.SetComp):
+            arg_type.append("Set")
+        elif isinstance(arg, ast.Str):
+            arg_type.append("Str")
+        elif isinstance(arg, ast.NameConstant):
+            arg_type.append("NameConstant")
+        elif isinstance(arg, ast.Constant):
+            arg_type.append("Constant")
+        elif isinstance(arg, ast.Call):
+            arg_type.append(("Call", get_func_calls(arg)[0]))
+        else:
+            arg_type.append("Other")
+    return arg_type
+
+
+def get_call_type(tree):
+    # how to remove 
+    func_calls = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Call):
+            callvisitor = FuncCallVisitor()
+            callvisitor.visit(node.func)
+            func_calls += [(callvisitor.name, get_args(node))]
+    return func_calls
+
+
+def get_call_type(tree):
+    # how to remove 
+    func_calls = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Call):
+            callvisitor = FuncCallVisitor()
+            callvisitor.visit(node.func)
+            func_calls += [(callvisitor.name, get_args(node))]
+    return func_calls
+
+
+def get_func_calls(tree):
+    node = deepcopy(tree)
+    transformer = CallTransformer()
+    transformer.visit(node)
+    return transformer.call_names