py-adtools 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

adtools/__init__.py

@@ -1 +1,2 @@
 from .py_code import PyScript, PyFunction, PyClass, PyProgram
+from .evaluator import PyEvaluator

adtools/evaluator.py

@@ -0,0 +1,178 @@
+import multiprocessing
+import os
+import sys
+import time
+from abc import ABC, abstractmethod
+from queue import Empty
+from typing import Any, Literal, Dict, Callable, List
+import psutil
+
+from .py_code import PyProgram
+
+
+class PyEvaluator(ABC):
+    """Python programs evaluator."""
+
+    def __init__(self, debug_mode: bool = False, *, exec_code: bool = True):
+        """Evaluator interface for evaluating the python algorithm program.
+        Args:
+            debug_mode         : Debug mode.
+            exec_code          : Using 'exec()' to compile the code and provide the callable function.
+        """
+        self._debug_mode = debug_mode
+        self._exec_code = exec_code
+        self._JOIN_TIMEOUT_SECONDS = 5
+
+    @abstractmethod
+    def evaluate_program(
+            self,
+            program_str: str,
+            callable_functions_dict: Dict[str, Callable] | None,
+            callable_functions_list: List[Callable] | None,
+            callable_classes_dict: Dict[str, Callable] | None,
+            callable_classes_list: List[Callable] | None,
+            **kwargs
+    ) -> Any | None:
+        """Evaluate a given program.
+        Args:
+            program_str            : The raw program text.
+            callable_functions_dict: A dict maps function name to callable function.
+            callable_functions_list: A list of callable functions.
+            callable_classes_dict  : A dict maps class name to callable class.
+            callable_classes_list  : A list of callable classes.
+        Return:
+            Returns the evaluation result.
+        """
+        raise NotImplementedError('Must provide an evaluator for a python program. '
+                                  'Override this method in a subclass.')
+
+    def _kill_process_and_its_children(self, process: multiprocessing.Process):
+        # Find all children processes
+        try:
+            parent = psutil.Process(process.pid)
+            children_processes = parent.children(recursive=True)
+        except psutil.NoSuchProcess:
+            children_processes = []
+        # Terminate parent process
+        process.terminate()
+        process.join(timeout=self._JOIN_TIMEOUT_SECONDS)
+        if process.is_alive():
+            process.kill()
+            process.join()
+        # Kill all children processes
+        for child in children_processes:
+            if self._debug_mode:
+                print(f"Killing process {process.pid}'s children process {child.pid}")
+            child.terminate()
+
+    def evaluate(self, program_str: str, **kwargs):
+        try:
+            # Parse to program instance
+            program = PyProgram.from_text(program_str)
+            function_names = [f.name for f in program.functions]
+            class_names = [c.name for c in program.classes]
+            if self._exec_code:
+                # Compile the program, and maps the global func/var/class name to its address
+                all_globals_namespace = {}
+                # Execute the program, map func/var/class to global namespace
+                exec(program_str, all_globals_namespace)
+                # Get callable functions
+                callable_functions_list = [all_globals_namespace[f_name] for f_name in function_names]
+                callable_functions_dict = dict(zip(function_names, callable_functions_list))
+                # Get callable classes
+                callable_classes_list = [all_globals_namespace[c_name] for c_name in class_names]
+                callable_classes_dict = dict(zip(class_names, callable_classes_list))
+            else:
+                callable_functions_list = None
+                callable_functions_dict = None
+                callable_classes_list = None
+                callable_classes_dict = None
+
+            # Get evaluate result
+            res = self.evaluate_program(
+                program_str,
+                callable_functions_dict,
+                callable_functions_list,
+                callable_classes_dict,
+                callable_classes_list,
+                **kwargs
+            )
+            return res
+        except Exception as e:
+            if self._debug_mode:
+                print(e)
+            return None
+
+    def _evaluate_in_safe_process(
+            self,
+            program_str: str,
+            result_queue: multiprocessing.Queue,
+            redirect_to_devnull: bool,
+            **kwargs
+    ):
+        if redirect_to_devnull:
+            with open('/dev/null', 'w') as devnull:
+                os.dup2(devnull.fileno(), sys.stdout.fileno())
+                os.dup2(devnull.fileno(), sys.stderr.fileno())
+        res = self.evaluate(program_str, **kwargs)
+        result_queue.put(res)
+
+    def secure_evaluate(
+            self,
+            program: str | PyProgram,
+            timeout_seconds: int | float = None,
+            redirect_to_devnull: bool = True,
+            multiprocessing_start_method=Literal['auto', 'fork', 'spawn'],
+            **kwargs
+    ):
+        """
+        Args:
+            program: the program to be evaluated.
+            timeout_seconds: return 'None' if the execution time exceeds 'timeout_seconds'.
+            redirect_to_devnull: redirect any output to '/dev/null'.
+            multiprocessing_start_method: start a process using 'fork' or 'spawn'.
+        """
+        if multiprocessing_start_method == 'auto':
+            # Force MacOS and Linux use 'fork' to generate new process
+            if sys.platform.startswith('darwin') or sys.platform.startswith('linux'):
+                multiprocessing.set_start_method('fork', force=True)
+        elif multiprocessing_start_method == 'fork':
+            multiprocessing.set_start_method('fork', force=True)
+        else:
+            multiprocessing.set_start_method('spawn', force=True)
+
+        try:
+            # Start evaluation process
+            result_queue = multiprocessing.Queue()
+            process = multiprocessing.Process(
+                target=self._evaluate_in_safe_process,
+                args=(str(program), result_queue, redirect_to_devnull),
+                kwargs=kwargs,
+            )
+            process.start()
+
+            if timeout_seconds is not None:
+                try:
+                    # Get the result in timeout seconds
+                    result = result_queue.get(timeout=timeout_seconds)
+                    # After getting the result, terminate/kill the process
+                    self._kill_process_and_its_children(process)
+                except Empty:
+                    # Timeout
+                    if self._debug_mode:
+                        print(f'DEBUG: the evaluation time exceeds {timeout_seconds}s.')
+                    self._kill_process_and_its_children(process)
+                    result = None
+                except Exception as e:
+                    if self._debug_mode:
+                        print(f'DEBUG: evaluation failed with exception:\n{e}')
+                    self._kill_process_and_its_children(process)
+                    result = None
+            else:
+                result = result_queue.get()
+                self._kill_process_and_its_children(process)
+            return result
+        except Exception as e:
+            if self._debug_mode:
+                print(e)
+            return None

adtools/py_code.py

@@ -44,7 +44,7 @@ class PyFunction:
             # Here, we assume the indentation is always four spaces.
             new_line = '\n' if self.body else ''
             function += f'    """{self.docstring}"""{new_line}'
-        # self.body is already indented.
+        # The self.body is already indented.
         function += self.body + '\n\n'
         return function
 
@@ -109,7 +109,7 @@ class PyClass:
         # Ensure there aren't leading & trailing new lines in `body`
         if name == 'body':
             value = value.strip('\n')
-        # ensure there aren't leading & trailing quotes in `docstring`
+        # Ensure there aren't leading & trailing quotes in `docstring`
         if name == 'docstring' and value is not None:
             if '"""' in value:
                 value = value.strip()
@@ -189,7 +189,7 @@ class _ProgramVisitor(ast.NodeVisitor):
             if has_decorators:
                 # Find the minimum line number and retain the code above
                 decorator_start_line = min(decorator.lineno for decorator in node.decorator_list)
-                decorator = '\n'.join(self._codelines[decorator_start_line - 1: node.lineno - 1])
+                decorator = '\n'.join(self._codelines[decorator_start_line - 1: node.lineno - 1]).strip()
                 # Update script end line
                 script_end_line = decorator_start_line - 1
             else:
@@ -262,10 +262,10 @@ class _ProgramVisitor(ast.NodeVisitor):
                     if has_decorators:
                         # Find the minimum line number and retain the code above
                         decorator_start_line = min(decorator.lineno for decorator in item.decorator_list)
-                        # Dedent decorator code for 4 spaces
+                        # Dedent decorator code
                         decorator = []
                         for line in range(decorator_start_line - 1, item.lineno - 1):
-                            dedented_decorator = self._codelines[line][4:]
+                            dedented_decorator = self._codelines[line].strip()
                             decorator.append(dedented_decorator)
                         decorator = '\n'.join(decorator)
                     else:

py_adtools-0.1.1.dist-info/METADATA

@@ -0,0 +1,211 @@
+Metadata-Version: 2.4
+Name: py-adtools
+Version: 0.1.1
+Summary: Useful tools for parsing Python programs for algorithm design.
+Home-page: https://github.com/RayZhhh/py-adtools
+Author: Rui Zhang
+Author-email: rzhang.cs@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+Dynamic: summary
+
+# Useful tools for parsing and evaluating Python programs for algorithm design
+
+------
+
+> This repo aims to help develop more powerful [Large Language Models for Algorithm Design (LLM4AD)](https://github.com/Optima-CityU/llm4ad) applications. 
+>
+> More tools will be provided soon.
+
+------
+
+The figure demonstrates how a Python program is parsed into `PyScript`, `PyFunction`, `PyClass,` and `PyProgram` via `adtools`.
+
+![pycode](./assets/pycode.png)
+
+------
+
+## Installation
+
+> [!TIP]
+>
+> It is recommended to use Python >= 3.10.
+
+Run the following instructions to install adtools.
+
+```shell
+pip install git+https://github.com/RayZhhh/adtool.git
+```
+
+Or install via pip:
+
+```shell
+pip install py-adtools
+```
+
+## Usage
+
+### Parser for a Python program
+
+Parse your code (in string) into Python code instances, so that you can check each component and modify it.
+
+```python
+from adtools import PyProgram
+
+code = r'''
+import ast, numba                 # This part will be parsed into PyScript
+import numpy as np
+
+@numba.jit()                      # This part will be parsed into PyFunction
+def function(arg1, arg2=True):     
+    if arg2:
+    	return arg1 * 2
+    else:
+    	return arg1 * 4
+
+@some.decorators()                # This part will be parsed into PyClass
+class PythonClass(BaseClass):
+    class_var1 = 1                # This part will be parsed into PyScript
+    class_varb = 2                # and placed in PyClass.class_vars_and_code
+
+    def __init__(self, x):        # This part will be parsed into PyFunction
+        self.x = x                # and placed in PyClass.functions
+
+    def method1(self):
+        return self.x * 10
+
+    @some.decorators()
+    def method2(self, x, y):
+    	return x + y + self.method1(x)
+
+    class InnerClass:             # This part will be parsed into PyScript
+    	def __init__(self):       # and placed in PyClass.class_vars_and_code
+    		...
+
+if __name__ == '__main__':        # This part will be parsed into PyScript
+	res = function(1)
+	print(res)
+	res = PythonClass().method2(1, 2)
+'''
+
+p = PyProgram.from_text(code)
+print(p)
+print(f'-------------------------------------')
+print(p.classes[0].functions[2].decorator)
+print(f'-------------------------------------')
+print(p.functions[0].name)
+```
+
+### Evaluate Python programs
+
+Evaluate Python programs in a secure process to avoid the abortation of the main process. Two steps:
+
+- Extend the `PyEvaluator` class and override the `evaluate_program` method.
+- Evaluate the program (in str) by calling the `evaluate` (directly evaluate without executing in a sandbox process) or the `secure_evaluate` (evaluate in a sandbox process) methods. 
+
+```python
+import time
+from typing import Dict, Callable, List, Any
+
+from adtools import PyEvaluator
+
+
+class SortAlgorithmEvaluator(PyEvaluator):
+    def evaluate_program(
+            self,
+            program_str: str,
+            callable_functions_dict: Dict[str, Callable] | None,
+            callable_functions_list: List[Callable] | None,
+            callable_classes_dict: Dict[str, Callable] | None,
+            callable_classes_list: List[Callable] | None,
+            **kwargs
+    ) -> Any | None:
+        """Evaluate a given sort algorithm program.
+        Args:
+            program_str            : The raw program text.
+            callable_functions_dict: A dict maps function name to callable function.
+            callable_functions_list: A list of callable functions.
+            callable_classes_dict  : A dict maps class name to callable class.
+            callable_classes_list  : A list of callable classes.
+        Return:
+            Returns the evaluation result.
+        """
+        # Get the sort algorithm
+        sort_algo: Callable = callable_functions_dict['merge_sort']
+        # Test data
+        input = [10, 2, 4, 76, 19, 29, 3, 5, 1]
+        # Compute execution time
+        start = time.time()
+        res = sort_algo(input)
+        duration = time.time() - start
+        if res == sorted(input):  # If the result is correct
+            return duration  # Return the execution time as the score of the algorithm
+        else:
+            return None  # Return None as the algorithm is incorrect
+
+
+code_generated_by_llm = '''
+def merge_sort(arr):
+    if len(arr) <= 1:
+        return arr
+
+    mid = len(arr) // 2              
+    left = merge_sort(arr[:mid])     
+    right = merge_sort(arr[mid:])   
+
+    return merge(left, right)
+
+def merge(left, right):
+    result = []
+    i = j = 0
+
+    while i < len(left) and j < len(right):
+        if left[i] < right[j]:
+            result.append(left[i])
+            i += 1
+        else:
+            result.append(right[j])
+            j += 1
+
+    result.extend(left[i:])
+    result.extend(right[j:])
+
+    return result
+'''
+
+harmful_code_generated_by_llm = '''
+def merge_sort(arr):
+    while True:
+        pass
+'''
+
+if __name__ == '__main__':
+    evaluator = SortAlgorithmEvaluator()
+
+    # Evaluate
+    score = evaluator.evaluate(code_generated_by_llm)
+    print(f'Score: {score}')
+
+    # Secure evaluate (the evaluation is executed in a sandbox process)
+    score = evaluator.secure_evaluate(code_generated_by_llm, timeout_seconds=10)
+    print(f'Score: {score}')
+
+    # Evaluate a harmful code, the evaluation will be terminated within 10 seconds
+    # We will obtain a score of `None` due to the violation of time restriction
+    score = evaluator.secure_evaluate(harmful_code_generated_by_llm, timeout_seconds=10)
+    print(f'Score: {score}')
+```
+

py_adtools-0.1.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+adtools/__init__.py,sha256=99EPY13OhXs2Zt7IOsa53uQ0Uv-g9dSkHby8mEjDdQM,96
+adtools/evaluator.py,sha256=KyYUqVhUpek1vS0CdiMJaNooVtIZVugIvzVH6bGi81Y,7255
+adtools/py_code.py,sha256=1XNc5ckX_ivePSGYEEcKHXat-v4zjbxKUcYiTIEjqCw,13649
+py_adtools-0.1.1.dist-info/licenses/LICENSE,sha256=E5GGyecx3y5h2gcEGQloF-rDY9wbaef5IHjRsvtFbt8,1065
+py_adtools-0.1.1.dist-info/METADATA,sha256=FrDmTPVetLES_vx0q0n5v1nf8OXfFA2vvG3nChka2Vo,6168
+py_adtools-0.1.1.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
+py_adtools-0.1.1.dist-info/top_level.txt,sha256=X2kKzmJFDAKR2FWCij5pfMG9pVVjVUomyl4e-1VLXIk,8
+py_adtools-0.1.1.dist-info/RECORD,,

py_adtools-0.1.0.dist-info/METADATA

@@ -1,89 +0,0 @@
-Metadata-Version: 2.4
-Name: py-adtools
-Version: 0.1.0
-Summary: Useful tools for parsing Python programs for algorithm design.
-Home-page: https://github.com/RayZhhh/py-adtools
-Author: Rui Zhang
-Author-email: rzhang.cs@gmail.com
-Classifier: Programming Language :: Python :: 3
-Classifier: Operating System :: OS Independent
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Scientific/Engineering
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-python
-Dynamic: summary
-
-# Useful tools for parsing Python programs for algorithm design
-
-------
-
-> This repo aims to help develop more powerful [Large Language Models for Algorithm Design (LLM4AD)](https://github.com/Optima-CityU/llm4ad) applications. 
->
-> More tools will be provided soon.
-
-------
-
-The figure demonstrates how a Python program is parsed into `PyScript`, `PyFunction`, `PyClass,` and `PyProgram` via `adtools`.
-
-![pycode](./assets/pycode.png)
-
-------
-
-## Installation
-
-> [!TIP]
->
-> It is recommended to use Python >= 3.10.
-
-Run the following instructions to install adtools.
-
-```shell
-pip install git+https://github.com/RayZhhh/adtool.git
-```
-
-## Usage
-
-Parse your code (in string) into Python code instances.
-
-```python
-from adtools import PyProgram
-
-code = r'''
-import ast
-import numpy as np
-
-def func():
-    a = 5
-    return a + a
-
-class A(B):
-    a=1
-    
-    @yes()
-    @deco()
-    def __init__(self):
-        pass
-
-    def method(self):
-        pass
-    
-    b=2
-'''
-
-p = PyProgram.from_text(code)
-print(p)
-print(f'-------------------------------------')
-print(p.classes[0].functions[0].decorator)
-print(f'-------------------------------------')
-print(p.functions[0].name)
-```
-

py_adtools-0.1.0.dist-info/RECORD

@@ -1,7 +0,0 @@
-adtools/__init__.py,sha256=l1JmS0mViywF9Nv3Dt4wtrOewfbTwchq_5vfiTwtMSw,62
-adtools/py_code.py,sha256=4_KGvhHZaruVFtZ3LTERkJ-swXgPWyTtEwIMTm39dwQ,13646
-py_adtools-0.1.0.dist-info/licenses/LICENSE,sha256=E5GGyecx3y5h2gcEGQloF-rDY9wbaef5IHjRsvtFbt8,1065
-py_adtools-0.1.0.dist-info/METADATA,sha256=1YEeHkE81yVH6vMGfXcjp9phR8j_uEyFdyf-IsKUlrQ,1870
-py_adtools-0.1.0.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
-py_adtools-0.1.0.dist-info/top_level.txt,sha256=X2kKzmJFDAKR2FWCij5pfMG9pVVjVUomyl4e-1VLXIk,8
-py_adtools-0.1.0.dist-info/RECORD,,