testgenie-py 0.1.0__tar.gz

testgenie_py-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.3
+Name: testgenie-py
+Version: 0.1.0
+Summary: 
+Author: cjseitz
+Author-email: charlesjseitz@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: ConfigArgParse (==1.7)
+Requires-Dist: astor (==0.8.1)
+Requires-Dist: atheris (==2.3.0)
+Requires-Dist: coverage (==7.6.4)
+Requires-Dist: klara (==0.6.3)
+Requires-Dist: pytest (>=8.3.5,<9.0.0)
+Requires-Dist: staticfg (>=0.9.5,<0.10.0)
+Requires-Dist: typed-ast (==1.5.5)
+Requires-Dist: z3-solver (==4.13.3.0)
+Description-Content-Type: text/markdown
+
+

testgenie_py-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "testgenie-py"
+version = "0.1.0"
+description = ""
+authors = ["cjseitz <charlesjseitz@gmail.com>"]
+readme = "README.md"
+
+[[tool.poetry.packages]]
+include = "testgen" 
+
+[tool.poetry.scripts]
+testgenie = "main:main"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+astor = "0.8.1"
+atheris = "2.3.0"
+ConfigArgParse = "1.7"
+coverage = "7.6.4"
+klara = "0.6.3"
+typed-ast = "1.5.5"
+z3-solver = "4.13.3.0"
+staticfg = "^0.9.5"
+pytest = "^8.3.5"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

testgenie_py-0.1.0/testgen/analyzer/ast_analyzer.py

@@ -0,0 +1,149 @@
+import ast
+from typing import List
+
+import testgen.util.file_utils
+from testgen.models.test_case import TestCase
+from testgen.analyzer.test_case_analyzer import TestCaseAnalyzerStrategy
+from abc import ABC
+from testgen.models.function_metadata import FunctionMetadata
+
+class ASTAnalyzer(TestCaseAnalyzerStrategy, ABC):
+    def collect_test_cases(self, function_metadata: FunctionMetadata) -> List[TestCase]:
+        """Collect test cases by analyzing AST conditions and return statements"""
+
+        if function_metadata and function_metadata.params:
+            param_names = list(function_metadata.params.keys())
+        else:
+            param_names = [arg.arg for arg in function_metadata.func_def.args.args
+                           if arg.arg != 'self']
+
+        test_cases: List[TestCase] = self.get_conditions_recursively(
+            function_metadata,
+            function_metadata.function_name,
+            function_metadata.func_def.body,
+            param_names)
+        return test_cases
+
+    def get_conditions_recursively(self, function_metadata: FunctionMetadata, func_name: str, func_node_body: list, param_names, test_cases=None, conditions=None) -> List[TestCase]:
+        if conditions is None:
+            conditions = []
+
+        if test_cases is None:
+            test_cases = []
+
+        for node in func_node_body:
+            if isinstance(node, ast.If):
+                condition_str = self.parse_condition(node.test)
+                print(f"Condition found in function: {condition_str}")
+                self.get_conditions_recursively(function_metadata, func_name, node.body, param_names, test_cases, conditions + [condition_str])
+                if node.orelse:
+                    self.get_conditions_recursively(function_metadata, func_name, node.orelse, param_names, test_cases, conditions)
+
+            elif isinstance(node, ast.Return):
+                inputs = self.generate_inputs_from_conditions(conditions, param_names)
+                
+                input_exists = False
+                for existing_test in test_cases:
+                    if existing_test.func_name == func_name and existing_test.inputs == inputs:
+                        input_exists = True
+                        break
+                    
+                if not input_exists:
+                    try:
+                        generated_file = function_metadata.filename
+                        module = testgen.util.file_utils.load_module(generated_file)
+                        
+                        # Check if we're dealing with a class
+                        if function_metadata.class_name:
+                            cls = getattr(module, function_metadata.class_name)
+                            instance = cls()
+                            func = getattr(instance, func_name)
+                            output = func(*inputs)
+                        else:
+                            func = getattr(module, func_name)
+                            output = func(*inputs)
+                        
+                    except Exception as e:
+                        print(f"Error executing function: {e}")
+                        output = self._try_to_determine_output(node)
+                    
+                    test_case = TestCase(func_name, inputs, output)
+                    test_cases.append(test_case)
+
+        return test_cases
+
+    def _try_to_determine_output(self, return_node):
+        """Try different strategies to determine the output value"""
+        try:
+            # Try direct evaluation
+            return ast.literal_eval(return_node.value)
+        except (ValueError, SyntaxError):
+            # Check for constants
+            if isinstance(return_node.value, ast.Constant):
+                return return_node.value.value
+            # Check for name values
+            elif isinstance(return_node.value, ast.Name):
+                if return_node.value.id == 'True':
+                    return True
+                elif return_node.value.id == 'False':
+                    return False
+            # Default fallback
+            return None
+
+    @staticmethod
+    def parse_condition(condition_node):
+        """Parse an AST condition node into a string representation."""
+        try:
+            if isinstance(condition_node, ast.Compare):
+                left = condition_node.left.id if isinstance(condition_node.left, ast.Name) else str(condition_node.left)
+                op = type(condition_node.ops[0]).__name__
+                right = condition_node.comparators[0].value if isinstance(condition_node.comparators[0], ast.Constant) else str(condition_node.comparators[0])
+                return f"{left} {op} {right}"
+            elif isinstance(condition_node, ast.BoolOp):
+                # Use a simple representation for boolean operations
+                return f"BoolOp({type(condition_node.op).__name__})"
+            elif isinstance(condition_node, ast.UnaryOp):
+                # Use a simple representation for unary operations
+                return f"UnaryOp({type(condition_node.op).__name__})"
+            elif isinstance(condition_node, ast.Name):
+                return condition_node.id
+            elif isinstance(condition_node, ast.Constant):
+                return str(condition_node.value)
+            
+            return ast.dump(condition_node)
+        except Exception as e:
+            print(f"Error parsing condition: {e}")
+            return "UnknownCondition"
+
+    @staticmethod
+    def generate_inputs_from_conditions(conditions, param_names):
+        inputs = []
+
+        # Create parameter values with more variety based on condition path
+        param_values = {}
+
+        # If no meaningful conditions were found, create varied inputs
+        if not any(param in cond for cond in conditions for param in param_names):
+            for i, param in enumerate(param_names):
+                # Alternate True/False to create diverse test cases
+                param_values[param] = (i % 2 == 0)
+        else:
+            # Start with all False
+            param_values = {param: False for param in param_names}
+
+            # Process each condition to extract parameter values
+            for cond in conditions:
+                for param in param_names:
+                    if param in cond:
+                        if "Eq True" in cond or "== True" in cond:
+                            param_values[param] = True
+                        elif "Eq False" in cond or "== False" in cond:
+                            param_values[param] = False
+                        elif f"Not({param})" in cond:
+                            param_values[param] = False
+
+        # Build the input tuple
+        for param in param_names:
+            inputs.append(param_values[param])
+
+        return tuple(inputs)

testgenie_py-0.1.0/testgen/analyzer/contracts/contract.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+
+class Contract(ABC):
+    @abstractmethod
+    def check_preconditions(self, args) -> bool:
+        """Check preconditions before the method executes."""
+        pass
+
+    @abstractmethod
+    def check_postconditions(self, args, output, exception) -> bool:
+        """Check postconditions before the method executes."""
+        pass
+

testgenie_py-0.1.0/testgen/analyzer/contracts/no_exception_contract.py

@@ -0,0 +1,16 @@
+from testgen.analyzer.contracts.contract import Contract
+from abc import ABC
+
+class NoExceptionContract(Contract, ABC):
+    """Ensures function does not raise an exception"""
+
+    def check_preconditions(self, args) -> bool:
+        """Returns False if there is a violation"""
+        return True
+
+    """Returns False if there is an exception"""
+    def check_postconditions(self, args, output, exception) -> bool:
+        if exception is not None:
+            return True
+        else:
+            return False

testgenie_py-0.1.0/testgen/analyzer/contracts/nonnull_contract.py

@@ -0,0 +1,15 @@
+from abc import ABC
+
+from testgen.analyzer.contracts.contract import Contract
+
+class NonNullContract(Contract, ABC):
+    """Ensures the function does not contain null or None input values."""
+    def check_preconditions(self, args) -> bool:
+        return all(arg is not None for arg in args)
+
+    """Ensures the return value is not None"""
+    def check_postconditions(self, args, output, exception) -> bool:
+        if exception:
+            return False
+        else:
+            return output is not None

testgenie_py-0.1.0/testgen/analyzer/fuzz_analyzer.py

@@ -0,0 +1,106 @@
+from abc import ABC
+import os
+import traceback
+from typing import List
+
+from atheris.native import FuzzedDataProvider
+
+import coverage
+
+from testgen.models.test_case import TestCase
+from testgen.analyzer.test_case_analyzer import TestCaseAnalyzerStrategy
+
+from testgen.models.function_metadata import FunctionMetadata
+
+class FuzzAnalyzer(TestCaseAnalyzerStrategy, ABC):
+
+    def __init__(self, analysis_context=None):
+        super().__init__(analysis_context)
+        self.coverage_tracker = coverage.Coverage(branch=True)
+        self.executed_branches = set()
+
+    # TODO: Use getattr() to get function in FunctionMetadata
+    def collect_test_cases(self, function_metadata: FunctionMetadata) -> List[TestCase]:
+        """Collect test cases using fuzzing techniques"""
+        
+        # Use module from function_metadata if available
+        if function_metadata and function_metadata.module:
+            module = self.analysis_context.module
+        else:
+            raise ValueError("Module not set in function metadata. Cannot perform fuzzing without a module.")
+            
+        class_name = function_metadata.class_name if function_metadata.class_name else None
+        try:
+            if not class_name is None:
+                cls = getattr(module, class_name, None)
+                func = getattr(cls(), function_metadata.function_name, None) if cls else None
+            else:
+                func = getattr(module, function_metadata.function_name, None)
+            if func:
+                return self.run_fuzzing(func, function_metadata.function_name, function_metadata.params, module, 10)
+        except Exception as e:
+            print(f"[FUZZ ANALYZER ERROR]: {e}")
+            traceback.print_exc()
+            return []
+
+    def run_fuzzing(self, func, func_name, param_types, module, iterations=10) -> List[TestCase]:
+        """Run the function with fuzzed inputs and collect failing test cases."""
+        print(f"Running fuzzing {func_name}")
+        test_cases = []
+
+        for _ in range(iterations):
+            fdp = FuzzedDataProvider(os.urandom(1024))
+            inputs = self.generate_inputs_from_fuzz_data(fdp, param_types)
+
+            self.coverage_tracker.erase()
+            self.coverage_tracker.start()
+
+            try:
+                output = func(*inputs)
+                self.coverage_tracker.stop()
+                covered_branches = self.get_branch_coverage(module)
+                covered_branches_tuple = tuple(covered_branches)
+
+                print(f"[COVERED_BRANCHES]: {covered_branches}")
+                print(f"[EXECUTED BRANCHES]: {self.executed_branches}")
+
+                for branch in covered_branches:
+                    if branch[1] < 0:
+                        if covered_branches_tuple not in self.executed_branches:
+                            self.executed_branches.add(covered_branches_tuple)
+                            test_cases.append(TestCase(func_name, inputs, output))
+
+            except Exception as e:
+                self.coverage_tracker.stop()
+                test_cases.append(TestCase(func_name, inputs, (type(e), str(e) + " EXCEPTION")))
+        self.executed_branches.clear()
+        return test_cases
+
+
+    # TODO: Look into FuzzDataProvider.instrumentall()
+    @staticmethod
+    def generate_inputs_from_fuzz_data(fdp: FuzzedDataProvider, param_types):
+        """Generate fuzzed inputs based on parameter types."""
+        inputs = []
+        for param_type in param_types.values():
+            if param_type == "int":
+                inputs.append(fdp.ConsumeInt(4))
+            elif param_type == "bool":
+                inputs.append(fdp.ConsumeBool())
+            elif param_type == "float":
+                inputs.append(fdp.ConsumeFloat())
+            elif param_type == "str":
+                inputs.append(fdp.ConsumeString(10))
+            elif param_type == "bytes":
+                inputs.append(fdp.ConsumeBytes(10))
+            else:
+                inputs.append(None)
+        return tuple(inputs)
+    
+    def get_branch_coverage(self, module):
+        data = self.coverage_tracker.get_data()
+        try:
+            return data.arcs(module.__file__)
+        except Exception as e:
+            print(f"[ERROR: FUZZ ANALYZER] Couldn't get arcs Exception: {e}")
+            return set()

testgenie_py-0.1.0/testgen/analyzer/random_feedback_analyzer.py

@@ -0,0 +1,291 @@
+import ast
+import importlib
+import random
+import time
+import traceback
+from typing import List, Dict, Set
+
+import testgen.util.randomizer
+import testgen.util.utils as utils
+import testgen.util.coverage_utils as coverage_utils
+from testgen.analyzer.contracts.contract import Contract
+from testgen.analyzer.contracts.no_exception_contract import NoExceptionContract
+from testgen.analyzer.contracts.nonnull_contract import NonNullContract
+from testgen.models.test_case import TestCase
+from testgen.analyzer.test_case_analyzer import TestCaseAnalyzerStrategy
+from abc import ABC
+
+from testgen.models.function_metadata import FunctionMetadata
+
+
+# Citation in which this method and algorithm were taken from:
+# C. Pacheco, S. K. Lahiri, M. D. Ernst and T. Ball, "Feedback-Directed Random Test Generation," 29th International
+# Conference on Software Engineering (ICSE'07), Minneapolis, MN, USA, 2007, pp. 75-84, doi: 10.1109/ICSE.2007.37.
+# keywords: {System testing;Contracts;Object oriented modeling;Law;Legal factors;Open source software;Software
+# testing;Feedback;Filters;Error correction codes},
+
+class RandomFeedbackAnalyzer(TestCaseAnalyzerStrategy, ABC):
+    def __init__(self, analysis_context=None):
+        super().__init__(analysis_context)
+        self.test_cases = []
+        self.covered_lines: Dict[str, Set[int]] = {}
+
+    # Algorithm described in above article
+    # Classes is the classes for which we want to generate sequences
+    # Contracts express invariant properties that hold both at entry and exit from a call
+        # Contract takes as input the current state of the system (runtime values created in the sequence so far, and any exception thrown by the last call), and returns satisfied or violated
+        # Output is the runtime values and boolean flag violated
+    # Filters determine which values of a sequence are extensible and should be used as inputs
+    def generate_sequences(self, function_metadata: List[FunctionMetadata], classes=None, contracts: List[Contract] = None, filters=None, time_limit=20):
+        contracts = [NonNullContract(), NoExceptionContract()]
+        error_seqs = [] # execution violates a contract
+        non_error_seqs = [] # execution does not violate a contract
+
+        functions = self._analysis_context.function_data
+        start_time = time.time()
+        while(time.time() - start_time) >= time_limit:
+            # Get random function
+            func = random.choice(functions)
+            param_types: dict = func.params
+            vals: dict = self.random_seqs_and_vals(param_types)
+            new_seq = (func.function_name, vals)
+            if new_seq in error_seqs or new_seq in non_error_seqs:
+                continue
+            outs_violated: tuple = self.execute_sequence(new_seq, contracts)
+            violated: bool = outs_violated[1]
+            # Create tuple of sequence ((func name, args), output)
+            new_seq_out = (new_seq, outs_violated[0])
+            if violated:
+                error_seqs.append(new_seq_out)
+            else:
+                # Question: Should I use the failed contract to be the assertion in unit test??
+                non_error_seqs.append(new_seq_out)
+        return error_seqs, non_error_seqs
+
+    def generate_sequences_new(self, contracts: List[Contract] = None, filters=None, time_limit=20):
+        contracts = [NonNullContract(), NoExceptionContract()]
+        error_seqs = []  # execution violates a contract
+        non_error_seqs = []  # execution does not violate a contract
+
+        functions = self._analysis_context.function_data.copy()
+        start_time = time.time()
+
+        while (time.time() - start_time) < time_limit:
+            # Get random function
+            func = random.choice(functions)
+            param_types: dict = func.params
+            vals: dict = self.random_seqs_and_vals(param_types)
+            new_seq = (func.function_name, vals)
+
+            if new_seq in [seq[0] for seq in error_seqs] or new_seq in [seq[0] for seq in non_error_seqs]:
+                continue
+
+            outs_violated: tuple = self.execute_sequence(new_seq, contracts)
+            violated: bool = outs_violated[1]
+
+            # Create tuple of sequence ((func name, args), output)
+            new_seq_out = (new_seq, outs_violated[0])
+
+            if violated:
+                error_seqs.append(new_seq_out)
+
+            else:
+                non_error_seqs.append(new_seq_out)
+
+            test_case = TestCase(new_seq_out[0][0], tuple(new_seq_out[0][1].values()), new_seq_out[1])
+            self.test_cases.append(test_case)
+            fully_covered = self.covered(func)
+            if fully_covered:
+                print(f"Function {func.function_name} is fully covered")
+                functions.remove(func)
+
+            if not functions:
+                self.test_cases.sort(key=lambda tc: tc.func_name)
+                print("All functions covered")
+                break
+
+        self.test_cases.sort(key=lambda tc: tc.func_name)
+        return error_seqs, non_error_seqs
+
+
+    def covered(self, func: FunctionMetadata) -> bool:
+        if func.function_name not in self.covered_lines:
+            self.covered_lines[func.function_name] = set()
+
+        for test_case in [tc for tc in self.test_cases if tc.func_name == func.function_name]:
+            analysis = coverage_utils.get_coverage_analysis(self._analysis_context.filepath,
+                                                                         func.function_name, test_case.inputs)
+            covered = coverage_utils.get_list_of_covered_statements(analysis)
+            self.covered_lines[func.function_name].update(covered)
+
+        executable_statements = self.get_all_executable_statements(func)
+
+        return self.covered_lines[func.function_name] == executable_statements
+
+    def execute_sequence(self, sequence, contracts: List[Contract]):
+        """Execute a sequence and check contract violations"""
+        func_name, args_dict = sequence
+        args = tuple(args_dict.values())  # Convert dict values to tuple
+
+        try:
+            # Use module from analysis context if available
+            module = self.analysis_context.module
+
+            if self._analysis_context.class_name:
+                cls = getattr(module, self._analysis_context.class_name, None)
+                if cls is None:
+                    raise AttributeError(f"Class '{self._analysis_context.class_name}' not found")
+                obj = cls()  # Instantiate the class
+                func = getattr(obj, func_name, None)
+
+                import inspect
+                sig = inspect.signature(func)
+                param_names = [p.name for p in sig.parameters.values() if p.name != 'self']
+            else:
+                func = getattr(module, func_name, None)
+
+                import inspect
+                sig = inspect.signature(func)
+                param_names = [p.name for p in sig.parameters.values()]
+
+            # Create ordered arguments based on function signature
+            ordered_args = []
+            for name in param_names:
+                if name in args_dict:
+                    ordered_args.append(args_dict[name])
+
+            # Check preconditions
+            for contract in contracts:
+                if not contract.check_preconditions(tuple(ordered_args)):
+                    print(f"Preconditions failed for {func_name} with {tuple(ordered_args)}")
+                    return None, True
+
+            # Execute function with properly ordered arguments
+            output = func(*ordered_args)
+            exception = None
+
+        except Exception as e:
+            print(f"EXCEPTION IN RANDOM FEEDBACK: {e}")
+            print(traceback.format_exc())
+            output = None
+            exception = e
+
+        # Check postconditions
+        for contract in contracts:
+            if not contract.check_postconditions(tuple(ordered_args), output, exception):
+                print(f"Postcondition failed for {func_name} with {tuple(ordered_args)}")
+                return output, True
+
+        return output, False
+    
+
+    # TODO: Currently only getting random vals of primitives, extend to sequences
+    def random_seqs_and_vals(self, param_types, non_error_seqs=None):
+        return self.generate_random_inputs(param_types)
+
+    @staticmethod
+    def extract_parameter_types(func_node):
+        """Extract parameter types from a function node."""
+        param_types = {}
+        for arg in func_node.args.args:
+            param_name = arg.arg
+            if arg.annotation:
+                param_type = ast.unparse(arg.annotation)
+                param_types[param_name] = param_type
+            else:
+                if param_name != 'self':
+                    param_types[param_name] = None
+        return param_types
+
+    @staticmethod
+    def generate_random_inputs(param_types):
+        """Generate inputs for fuzzing based on parameter types."""
+        inputs = {}
+        for param, param_type in param_types.items():
+            if param_type == "int":
+                random_integer = random.randint(1, 100)
+                inputs[param] = random_integer
+            if param_type == "bool":
+                random_choice = random.choice([True, False])
+                inputs[param] = random_choice
+            if param_type == "float":
+                random_float = random.random()
+                inputs[param] = random_float
+            # TODO: Random String and Random bytes; Random objects?
+            if param_type == "str":
+                inputs[param] = "abc"
+            #elif param_type == "bytes":
+            #    inputs[param] = fdp.ConsumeBytes(10)
+            #else:
+            #    inputs[param] = None
+        return inputs
+
+    def collect_test_cases(self, function_metadata: FunctionMetadata) -> List[TestCase]:
+        """Collect test cases using random feedback technique"""
+        error_seqs, non_error_seqs = self.generate_sequences_new()
+        test_cases = []
+
+        # Process error sequences
+        if error_seqs:
+            for error_seq in error_seqs:
+                print("ERROR SEQ OUTPUT:", error_seq[1])
+                test_cases.append(TestCase(error_seq[0][0], tuple(error_seq[0][1].values()), error_seq[1]))
+
+        # Process non-error sequences
+        if non_error_seqs:
+            for non_error_seq in non_error_seqs:
+                print("NON ERROR SEQ OUTPUT:", non_error_seq[1])
+                test_cases.append(TestCase(non_error_seq[0][0], tuple(non_error_seq[0][1].values()), non_error_seq[1]))
+
+        return self.test_cases
+
+    def get_all_executable_statements(self, func: FunctionMetadata):
+        """Get all executable statements including else branches"""
+        import ast
+
+        test_cases = [tc for tc in self.test_cases if tc.func_name == func.function_name]
+
+        if not test_cases:
+            print("Warning: No test cases available to determine executable statements")
+            from testgen.util.randomizer import new_random_test_case
+            temp_case = new_random_test_case(self._analysis_context.filepath, func.func_def)
+            analysis = coverage_utils.get_coverage_analysis(self._analysis_context.filepath, func.function_name,
+                                                                         temp_case.inputs)
+        else:
+            analysis = coverage_utils.get_coverage_analysis(self._analysis_context.filepath, func.function_name, test_cases[0].inputs)
+
+        # Get standard executable lines from coverage.py
+        executable_lines = list(analysis[1])
+
+        # Parse the source file to find else branches
+        with open(self._analysis_context.filepath, 'r') as f:
+            source = f.read()
+
+        # Parse the code
+        tree = ast.parse(source)
+
+        # Find our specific function
+        for node in ast.walk(tree):
+            if isinstance(node, ast.FunctionDef) and node.name == func.func_def.name:
+                # Find all if statements in this function
+                for if_node in ast.walk(node):
+                    if isinstance(if_node, ast.If) and if_node.orelse:
+                        # There's an else branch
+                        if isinstance(if_node.orelse[0], ast.If):
+                            # This is an elif - already counted
+                            continue
+
+                        # Get the line number of the first statement in the else block
+                        # and subtract 1 to get the 'else:' line
+                        else_line = if_node.orelse[0].lineno - 1
+
+                        # Check if this is actually an else line (not a nested if)
+                        with open(self._analysis_context.filepath, 'r') as f:
+                            lines = f.readlines()
+                            if else_line <= len(lines):
+                                line_content = lines[else_line - 1].strip()
+                                if line_content == "else:":
+                                    if else_line not in executable_lines:
+                                        executable_lines.append(else_line)
+
+        return sorted(executable_lines)