data-prep-toolkit-transforms 0.2.2__1-py3-none-any.whl

UAST.py

@@ -0,0 +1,270 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import json
+import networkx
+import matplotlib.pyplot as plt
+
+class UASTNode:
+    """
+    Represents a node in the Universal Abstract Syntax Tree (UAST).
+
+    Attributes:
+        id (int): The unique identifier of the node.
+        code_snippet (str): The line(s) of code associated with the node.
+        node_type (str): The type of the node.
+        parents (list): The list of parent nodes.
+        children (list): The list of child nodes.
+        metadata (dict): The associated information/metadata of the node
+        start_point (tuple(int, int)): The start line number and byte of the line of the node.
+        end_point (tuple(int, int)): The end line number and byte of the node.
+    """
+
+    def __init__(self, 
+                 id: int = 0,
+                 code_snippet: str = None,
+                 node_type: str = None,
+                 parents: list = list(),
+                 children: list = list(), 
+                 metadata : dict = dict(),
+                 start_point : tuple[int,int] = (None, None),
+                 end_point : tuple[int,int] = (None, None)) -> None: 
+        
+        self.id = id
+        self.code_snippet = code_snippet
+        self.node_type = node_type
+        self.parents = parents
+        self.children = children
+        self.metadata = metadata
+        self.start_point = start_point
+        self.end_point = end_point
+
+    def __str__(self) -> str:
+        return f"ID: {self.id}, Type: {self.node_type}, Snippet: {repr(self.code_snippet)}, Parents: {self.parents}, Children: {self.children}, Metadata = {self.metadata}"
+    
+    def __repr__(self) -> str:
+        return f"ID: {self.id}, Type: {self.node_type}, Snippet: {repr(self.code_snippet)}, Parents: {self.parents}, Children: {self.children}, Metadata = {self.metadata}"
+    
+    def __eq__(self, other) -> bool:
+        return self.id == other.id and self.code_snippet == other.code_snippet and self.node_type == other.node_type and self.parents == other.parents and self.children == other.children and self.metadata == other.metadata and self.start_point == other.start_point and self.end_point == other.end_point
+    
+class UASTEdge:
+    """
+    Represents an edge in the UAST (Universal Abstract Syntax Tree).
+
+    Attributes:
+        start_id (int): The ID of the starting node of the edge.
+        end_id (int): The ID of the ending node of the edge.
+        directed_relation (str): The directed relation between the nodes.
+        metadata (dict): The metadata information associated with the edge.
+    """
+
+    def __init__(self, 
+                 start_id: int = None, 
+                 end_id: int = None, 
+                 directed_relation: str = None,
+                 metadata : dict = dict()):
+
+        self.start_id = start_id
+        self.end_id = end_id
+        self.directed_relation = directed_relation
+        self.metadata = metadata
+
+    def __str__(self) -> str:
+        return f"Start: {self.start_id}, End: {self.end_id}, Relation: {self.directed_relation}, Metadata = {self.metadata}, Metadata: {self.metadata}"
+    
+    def __repr__(self) -> str:
+        return f"Start: {self.start_id}, End: {self.end_id}, Relation: {self.directed_relation}, Metadata = {self.metadata}, Metadata: {self.metadata}"
+    
+    def __eq__(self, other) -> bool:
+        return self.start_id == other.start_id and self.end_id == other.end_id and self.directed_relation == other.directed_relation and self.metadata == other.metadata
+    
+    def __hash__(self) -> int:
+        return hash((self.start_id, self.end_id, self.directed_relation, self.metadata))
+    
+class UAST:
+    """
+    Represents a graph of a Universal Abstract Syntax Tree (UAST).
+
+    Attributes:
+        nodes (dict[int, UASTNode]): A dictionary mapping node IDs to UASTNode objects.
+        edges (list[UASTEdge]): A list of UASTEdge objects representing the edges between nodes.
+        assigned_id (int): The ID to be assigned to the next node added to the UAST.
+
+    Methods:
+        __init__(): Initializes an empty UAST object.
+        __len__(): Returns the number of nodes in the UAST.
+        __str__(): Returns a string representation of the UAST.
+        __repr__(): Returns a string representation of the UAST.
+        __eq__(other): Checks if the UAST is equal to another UAST.
+        add_node(node): Adds a node to the UAST.
+        _create_root(): Creates a root node for the UAST.
+        create_node(node_type, code_snippet, start_point, end_point): Creates a new node and adds it to the UAST, also returns the node object.
+        add_edge(node1, node2, directed_relation, metadata): Adds an edge between two nodes in the UAST.
+        get_node(id): Retrieves a node from the UAST based on its ID.
+        get_nodes_of_type(node_type): Retrieves the ID of all nodes of the input type
+        get_children(node): Retrieves the children of a node in the UAST.
+        get_parents(node): Retrieves the parent of a node in the UAST.
+        print_graph(id): Prints the UAST starting from the specified node ID.
+        save_to_file(file_path): Saves the UAST to a file in JSON format.
+        load_from_file(file_path): Loads the UAST from a file in JSON format.
+        visualize(): Visualizes the graph using NetworkX
+    """
+    def __init__(self):
+        self.nodes : dict[int,UASTNode] = dict()
+        self.edges : list[UASTEdge] = list()
+        self.assigned_id : int = 0
+        self.nodes_of_type : dict = dict()
+        self.root = self._create_root()
+
+    def __len__(self) -> int:
+        return len(self.nodes)
+
+    def __str__(self) -> str:
+        return f"Nodes: {self.nodes} \nEdges: {self.edges}"
+    
+    def __repr__(self) -> str:
+        return f"Nodes: {self.nodes} \nEdges: {self.edges}"
+    
+    def __eq__(self, other) -> bool:
+        return self.nodes == other.nodes and self.edges == other.edges
+
+    def add_node(self, node : UASTNode) -> None:
+        self.nodes[self.assigned_id] = node
+        self.assigned_id += 1
+        if node.node_type not in self.nodes_of_type :
+            self.nodes_of_type[node.node_type] = list()
+        self.nodes_of_type[node.node_type].append(node.id)
+        return
+
+    def _create_root(self) -> UASTNode:
+        return self.create_node(node_type = "uast_root", code_snippet = "root", metadata= {"info" : "links to all"}, start_point = (-1,0), end_point = (-1,3))
+
+    def create_node(self, 
+                    node_type : str = None, 
+                    code_snippet : str = None,
+                    metadata : dict = dict(),
+                    start_point : tuple[int,int] = (None, None),
+                    end_point : tuple[int,int] = (None, None)) -> UASTNode: 
+            
+        node = UASTNode(id = self.assigned_id, node_type = node_type, code_snippet = code_snippet, metadata = metadata, start_point = start_point, end_point = end_point, children= list(), parents = list())
+        self.add_node(node)
+        return node
+    
+    def add_edge(self, node1 : UASTNode = None, node2 : UASTNode = None, directed_relation : str = None, metadata : dict = dict())-> UASTEdge:
+        edge = UASTEdge(start_id = node1.id, end_id = node2.id, directed_relation = directed_relation, metadata = metadata)
+        node2.parents.append(node1.id)
+        node1.children.append(node2.id)
+        self.edges.append(edge)
+        return edge
+    
+    def get_node(self, id : int) -> UASTNode:
+        return self.nodes[id]
+    
+    def get_nodes_of_type(self, node_type : str) -> list[int]:
+        return self.nodes_of_type[node_type]
+    
+    def get_children(self, node : UASTNode) -> list[int]:
+        return node.children
+    
+    def get_parents(self, node : UASTNode) -> int: 
+        return node.parents
+
+    def print_graph(self, id):
+        if id not in self.nodes:
+            return
+        visited = set()
+        
+        def dfs(id, visited):
+            visited.add(id)
+            print(self.nodes[id])
+            for child in self.nodes[id].children:
+                if child not in visited:
+                    dfs(child, visited)
+        
+        dfs(id, visited)
+        del visited
+    
+
+    def save_to_file(self, file_path):
+        # convert children list to list for serialization
+        copy_nodes = self.nodes.copy()
+        for k, v in self.nodes.items():
+            v.children = list(v.children)
+            v.parents = list(v.parents)
+            copy_nodes[k] = v
+
+
+        data = {
+            "nodes": {str(k): v.__dict__ for k, v in self.nodes.items()},
+            "edges": [edge.__dict__ for edge in self.edges]
+        }
+
+        with open(file_path, 'w') as f:
+            json.dump(data, f, indent= 4)
+
+        return
+    
+    def get_json(self):
+
+        copy_nodes = self.nodes.copy()
+        for k, v in self.nodes.items():
+            v.children = list(v.children)
+            v.parents = list(v.parents)
+            copy_nodes[k] = v
+
+        data = {
+            "nodes": {str(k): v.__dict__ for k, v in self.nodes.items()},
+            "edges": [edge.__dict__ for edge in self.edges]
+        }
+        
+        return data
+    
+    def load_from_json_string(self, obj: str):
+        data = json.loads(obj)
+        self.nodes = {int(k): UASTNode(**v) for k, v in data["nodes"].items()}
+        self.edges = [UASTEdge(**edge) for edge in data["edges"]]
+        self.assigned_id = max(self.nodes.keys()) + 1
+        for node in self.nodes.values():
+            node.start_point = tuple(node.start_point)
+            node.end_point = tuple(node.end_point)
+        return 
+
+    def load_from_file(self, file_path):
+        with open(file_path, 'r') as f:
+            data = json.load(f)
+            self.nodes = {int(k): UASTNode(**v) for k, v in data["nodes"].items()}
+            self.edges = [UASTEdge(**edge) for edge in data["edges"]]
+            self.assigned_id = max(self.nodes.keys()) + 1
+            for node in self.nodes.values():
+                node.start_point = tuple(node.start_point)
+                node.end_point = tuple(node.end_point)
+        return 
+
+    def visualize(self):
+        edges_viz = []
+        labeldict = {}
+        for edge in self.edges:
+            edges_viz.append([edge.start_id, edge.end_id])
+            labeldict[edge.start_id] = self.nodes[edge.start_id].node_type
+            labeldict[edge.end_id] = self.nodes[edge.end_id].node_type
+        print(labeldict)
+        plt.figure(figsize=(10,10))
+        plt.rcParams["font.size"] = 20
+        G = networkx.Graph() 
+        G.add_edges_from(edges_viz) 
+        pos = networkx.spring_layout(G)
+        networkx.draw_networkx_labels(G, pos, labels= labeldict, font_size= 12, )
+        networkx.draw_networkx_nodes(G, pos, nodelist= self.nodes.keys(), node_size= 300)
+        networkx.draw_networkx_edges(G, pos, edgelist= edges_viz)
+        plt.show() 
+        return

UAST_parser.py

@@ -0,0 +1,271 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from UAST import UAST
+import json
+from tree_sitter import Tree
+import os
+import sys
+sys.setrecursionlimit(10000)
+
+"""
+Initialize the parser with a path for rules and grammar.
+"""
+class UASTParser():
+    def __init__(self):
+        self.language : str = None
+        self.uast : UAST = None
+        self.rules : dict = None
+        self.cached_rules = dict()
+
+        # Compute the absolute path to the tree-sitter-bindings directory
+        grammar_dir = os.path.dirname(os.path.abspath(__file__))
+        self.grammar_path = os.path.join(grammar_dir, '..', '..', 'python', 'src', 'grammar', 'UAST_Grammar.json')
+
+        if not os.path.exists(self.grammar_path):
+            print("Current working directory:", os.getcwd())
+            raise FileNotFoundError(f"UAST Grammar file not found at {self.grammar_path}. Please ensure it exists.")
+
+        with open(self.grammar_path, "r") as grammar_file:
+            self.grammar = json.load(grammar_file)
+
+        # Compute the absolute path to the ruleset directory based on the script's location
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        self.rule_directory = os.path.join(script_dir, 'ruleset/')
+        
+        if not os.path.isdir(self.rule_directory):
+            print("Script directory:", script_dir)
+            raise FileNotFoundError(f"Ruleset directory not found at {self.rule_directory}. Please ensure it exists.")
+
+        '''
+        # Rule directory and file
+        self.rule_directory = "../../python/src/ruleset/"
+        if not os.path.isdir(self.rule_directory):
+            print("Current working directory:", os.getcwd())
+            raise FileNotFoundError(f"Ruleset directory not found at {self.rule_directory}. Please ensure it exists.")
+        '''
+        self.rule_file_name: str = "UAST_rules_"
+
+        self.AST : Tree = None
+        # self.offset : int = None
+        # self.prev_line : int = -1
+        self.extracted : str = None
+        self.function_info = dict()
+        self.class_info = dict()
+        self.user_defined_entity = {"uast_function": "self.function_info[snippet] = id",
+                         "uast_class": "self.class_info[snippet] = id"}
+
+
+    def set_rule_dir_path(self, path: str):
+        self.rule_directory = path
+
+    def set_grammar_path(self, path : str):
+        self.grammar_path = path
+        self.grammar = json.load(open(self.grammar_path, "r"))
+
+    # set language for the parser
+    def set_language(self, language : str):
+        self.language = language
+
+        if (language not in self.cached_rules):
+            rules_cache = json.load(open(self.rule_directory + self.rule_file_name + self.language + '.json', "r"))
+            self.cached_rules[language] = rules_cache
+        
+        self.rules = self.cached_rules[language]
+
+    # initialise a DFS traversal on the AST and an empty UAST.
+    def parse(self, AST, code_snippet) :
+        if(self.language == None) :
+            print("Language not loaded")
+            return
+        self.AST = AST
+        self.uast = UAST()
+        self.uast.root.metadata["language"] = self.language
+        self.uast.root.metadata["loc_snippet"] = self.count_loc(code_snippet, self.language)
+        self._dfs(AST_node = self.AST.root_node, parent = self.uast.root)
+        '''
+        # commenting this block temporarily
+        # Call the new modularized function to calculate the code-to-comment ratio
+        code_to_comment_ratio = self.calculate_code_to_comment_ratio(self.uast.root)
+        # Add the code_to_comment_ratio to the root node's metadata
+        self.uast.root.metadata["code_to_comment_ratio"] = code_to_comment_ratio
+        '''
+        return self.uast
+
+    def calculate_code_to_comment_ratio(self, root_node):
+        # Get the loc_snippet from the root node's metadata
+        loc_snippet = root_node.metadata.get("loc_snippet", 0)
+
+        # Sum all loc_original_code for uast_comment nodes
+        total_comment_loc = 0
+
+        # Recursive function to sum comment LOC
+        def sum_comment_loc(node):
+            nonlocal total_comment_loc
+
+            # Check if the node is a comment node
+            if node.node_type == "uast_comment":
+                total_comment_loc += node.metadata.get("loc_original_code", 0)
+
+            # Traverse the children, ensuring we get the actual node objects
+            for child_id in node.children:
+                child_node = self.uast.get_node(child_id)  # Fetch the actual child node using self.uast
+                sum_comment_loc(child_node)  # Recursively sum for the child node
+
+        # Start summing loc_original_code from the root node
+        sum_comment_loc(root_node)
+
+        # Calculate the code-to-comment ratio (handling division by zero)
+        if total_comment_loc > 0:
+            return loc_snippet / total_comment_loc
+        else:
+            return None  # Handle no comments
+        
+    def count_lo_comments(self, code_snippet):
+        lines = code_snippet.split('\n')
+        loc_count = 0
+        for line in lines:
+            stripped_line = line.strip()
+            # Count all lines except blank ones
+            if stripped_line:
+                loc_count += 1
+        return loc_count
+
+    def count_loc(self, code_snippet, language):
+        # Define the comment markers for each language
+        language_comment_markers = {
+            "c": ('//', '/*', '*/'),
+            "java": ('//', '/*', '*/'),
+            "C#": ('//', '/*', '*/'),
+            "c_sharp": ('//', '/*', '*/'),
+            "cpp": ('//', '/*', '*/'),
+            "objc": ('//', '/*', '*/'),
+            "rust": ('//', '/*', '*/'),
+            "go": ('//', '/*', '*/'),
+            "kotlin": ('//', '/*', '*/'),
+            "VHDL": ('--', None, None),
+            "py": ('#', '"""', '"""'),
+            "js": ('//', '/*', '*/'),
+            "dart": ('//', '/*', '*/'),
+            "QML": ('//', None, None),
+            "typescript": ('//', '/*', '*/'),
+            "perl": ('#', None, None),
+            "haskell": ('--', '{-', '-}'),
+            "elm": ('--', '{-', '-}'),
+            "agda": ('--', '{-', '-}'),
+            "d": ('//', '/*', '*/'),
+            "nim": ('#', '##', None),
+            "ocaml": ('(*', '(*', '*)'),
+            "scala": ('//', '/*', '*/')
+        }
+        
+        single_line_comment, multi_line_comment_start, multi_line_comment_end = language_comment_markers.get(language, (None, None, None))
+        
+        if not single_line_comment:
+            raise ValueError(f"Unsupported language: {language}")
+        
+        lines = code_snippet.split('\n')
+        loc_count = 0
+        inside_multiline_comment = False
+
+        for line in lines:
+            stripped_line = line.strip()
+            
+            # Skip empty lines
+            if not stripped_line:
+                continue
+            
+            # Handle multi-line comments
+            if multi_line_comment_start and multi_line_comment_end:
+                if inside_multiline_comment:
+                    # Check if the line contains the end of a multi-line comment
+                    if multi_line_comment_end in stripped_line:
+                        inside_multiline_comment = False
+                    continue
+                elif multi_line_comment_start in stripped_line:
+                    # If the line starts a multi-line comment
+                    inside_multiline_comment = True
+                    continue
+            
+            # Skip single-line comments
+            if stripped_line.startswith(single_line_comment):
+                continue
+            
+            # If the line is neither a comment nor blank, count it as LOC
+            loc_count += 1
+
+        return loc_count
+
+    def _add_user_defined(self, node):
+        id = node.id
+        type = node.node_type
+        
+        if node.code_snippet is not None:
+            snippet = node.code_snippet.replace(type, '').strip()
+            # Add further processing with the snippet
+        else:
+            # Handle the case where code_snippet is None
+            snippet = ""
+            # You can log a warning or take other appropriate action
+            print(f"Warning: node.code_snippet is None for node type: {type}")
+
+        if (type in self.user_defined_entity):
+            exec(self.user_defined_entity[type])
+            node.metadata["user_defined"] = True
+
+        del id
+        del type
+        del snippet
+        return
+
+    # Traversing through the AST to create nodes recursively.
+    def _dfs(self, AST_node, parent) :
+        if (AST_node.type in self.rules) :
+            ast_snippet = AST_node.text.decode("utf8")
+            node_type = self.rules[AST_node.type]["uast_node_type"]
+            exec_string = self.rules[AST_node.type]["extractor"]
+            uast_snippet = self._extract(ast_snippet = ast_snippet, node_type = node_type, exec_string = exec_string)
+
+            if node_type == "uast_comment":
+                loc_original_code = self.count_lo_comments(ast_snippet)
+            else:
+                loc_original_code = self.count_loc(ast_snippet, self.language)
+
+            node = self.uast.create_node(
+                node_type = node_type,
+                code_snippet = uast_snippet,
+                # choose to enable or disbale the storage of original code by removing the following line.            
+                metadata = {
+                    "original_code" : ast_snippet,
+                    "loc_original_code": loc_original_code
+                },
+            )
+            self._add_user_defined(node)
+            self.uast.add_edge(node1 = parent, node2 = node, directed_relation = "parent_node")
+            parent = node
+
+        for child in AST_node.children:
+            try:
+                self._dfs(AST_node= child, parent = parent)
+            except RecursionError as e:
+                print(f"RecursionError caught: {str(e)}")
+
+    def _extract(self, ast_snippet, node_type, exec_string):
+        code_snippet = ast_snippet
+        try:
+            exec(exec_string)
+        except Exception as e:
+            print(e)
+        try:
+            return self.grammar[node_type]["keyword"] + " " + self.extracted        
+        except Exception as e:
+            print(e)

base_tokenizer.py

@@ -0,0 +1,36 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import string
+
+
+"""
+This implements the most simplistic tokenizer based on the white spaces
+that can be overwritten by a different a different one. This method is
+build in the library and can be overwritten using approach described at 
+https://stackoverflow.com/questions/37553545/how-do-i-override-a-function-of-a-python-library
+
+import base_tokenizer
+base_tokenizer.tokenize = my_local_tokenize
+
+"""
+
+
+def tokenize(text: str) -> list[str]:
+    """
+    Tokenize string
+    :param text: source text
+    :return: list of tokens (words)
+    """
+    # start from normalizing string
+    normal = text.strip().lower().translate(str.maketrans("", "", string.punctuation))
+    return normal.split()