py-adtools 0.1.7__py3-none-any.whl → 0.1.8__py3-none-any.whl

adtools/evaluator.py

@@ -8,9 +8,10 @@ Commercial use of this software or its derivatives requires prior written permis
 import multiprocessing
 import os
 import sys
+import time
 from abc import ABC, abstractmethod
 from queue import Empty
-from typing import Any, Literal, Dict, Callable, List
+from typing import Any, Literal, Dict, Callable, List, Tuple
 import psutil
 import traceback
 
@@ -22,6 +23,7 @@ class PyEvaluator(ABC):
     def __init__(
             self,
             exec_code: bool = True,
+            find_and_kill_children_evaluation_process: bool = False,
             debug_mode: bool = False,
             *,
             join_timeout_seconds: int = 10
@@ -33,11 +35,15 @@ class PyEvaluator(ABC):
                 which will be passed to 'self.evaluate_program()'. Set this parameter to 'False' if you are going to
                 evaluate a Python scripy. Note that if the parameter is set to 'False', the arguments 'callable_...'
                 in 'self.evaluate_program()' will no longer be affective.
+            find_and_kill_children_evaluation_process: If using 'self.secure_evaluate', kill children processes
+                when they are terminated. Note that it is suggested to set to 'False' if the evaluation process
+                does not start new processes.
             debug_mode: Debug mode.
             join_timeout_seconds: Timeout in seconds to wait for the process to finish. Kill the process if timeout.
         """
         self.debug_mode = debug_mode
         self.exec_code = exec_code
+        self.find_and_kill_children_evaluation_process = find_and_kill_children_evaluation_process
         self.join_timeout_seconds = join_timeout_seconds
 
     @abstractmethod
@@ -66,11 +72,14 @@ class PyEvaluator(ABC):
         )
 
     def _kill_process_and_its_children(self, process: multiprocessing.Process):
-        # Find all children processes
-        try:
-            parent = psutil.Process(process.pid)
-            children_processes = parent.children(recursive=True)
-        except psutil.NoSuchProcess:
+        if self.find_and_kill_children_evaluation_process:
+            # Find all children processes
+            try:
+                parent = psutil.Process(process.pid)
+                children_processes = parent.children(recursive=True)
+            except psutil.NoSuchProcess:
+                children_processes = []
+        else:
             children_processes = []
         # Terminate parent process
         process.terminate()
@@ -151,8 +160,9 @@ class PyEvaluator(ABC):
             timeout_seconds: int | float = None,
             redirect_to_devnull: bool = False,
             multiprocessing_start_method: Literal['default', 'auto', 'fork', 'spawn'] = 'auto',
+            get_evaluate_time=False,
             **kwargs
-    ):
+    ) -> Any | Tuple[Any, float]:
         """Evaluate program in a new process. This enables timeout restriction and output redirection.
         Args:
             program: the program to be evaluated.
@@ -161,7 +171,11 @@ class PyEvaluator(ABC):
             multiprocessing_start_method: start a process using 'fork' or 'spawn'. If set to 'auto',
                 the process will be started using 'fork' with Linux/macOS and 'spawn' with Windows.
                 If set to 'default', there will be no changes to system default.
+            get_evaluate_time: get evaluation time for this program.
             **kwargs: additional keyword arguments to pass to 'evaluate_program'.
+        Returns:
+            Returns the evaluation results. If the 'get_evaluate_time' is True,
+            the return value will be (Results, Time).
         """
         if multiprocessing_start_method == 'auto':
             # Force macOS and Linux use 'fork' to generate new process
@@ -180,21 +194,28 @@ class PyEvaluator(ABC):
                 args=(str(program), result_queue, redirect_to_devnull),
                 kwargs=kwargs,
             )
+            evaluate_start_time = time.time()
             process.start()
 
             if timeout_seconds is not None:
                 try:
                     # Get the result in timeout seconds
                     result = result_queue.get(timeout=timeout_seconds)
+                    # Calculate the evaluate time
+                    eval_time = time.time() - evaluate_start_time
                     # After getting the result, terminate/kill the process
                     self._kill_process_and_its_children(process)
                 except Empty:  # The queue is empty indicates a timeout
+                    # Calculate the evaluate time
+                    eval_time = time.time() - evaluate_start_time
                     if self.debug_mode:
                         print(f'DEBUG: the evaluation time exceeds {timeout_seconds}s.')
                     # Terminate/kill all processes if timeout happens
                     self._kill_process_and_its_children(process)
                     result = None
                 except Exception as e:
+                    # Calculate the evaluate time
+                    eval_time = time.time() - evaluate_start_time
                     if self.debug_mode:
                         print(f'DEBUG: evaluation failed with exception:\n{traceback.format_exc()}')
                     # Terminate/kill all processes if meet exceptions
@@ -203,9 +224,12 @@ class PyEvaluator(ABC):
             else:
                 # If there is no timeout limit, wait execution to finish
                 result = result_queue.get()
+                # Calculate the evaluate time
+                eval_time = time.time() - evaluate_start_time
                 # Terminate/kill all processes after evaluation
                 self._kill_process_and_its_children(process)
-            return result
+
+            return (result, eval_time) if get_evaluate_time else result
         except Exception as e:
             if self.debug_mode:
                 print(traceback.format_exc())

adtools/evaluator_pool.py

@@ -62,21 +62,23 @@ class EvaluatorExecutorPool:
             program: the program to be evaluated.
             timeout_seconds: return 'None' if the execution time exceeds 'timeout_seconds'.
             redirect_to_devnull: redirect any output to '/dev/null'.
-            multiprocessing_start_method: start a process using 'fork' or 'spawn'.
+            multiprocessing_start_method: start a process using 'fork' or 'spawn'. If set to 'auto',
+                the process will be started using 'fork' with Linux/macOS and 'spawn' with Windows.
+                If set to 'default', there will be no changes to system default.
+            return_time: get evaluation time for this program.
             **kwargs: additional keyword arguments to pass to 'evaluate_program'.
+        Returns:
+            Returns the evaluation results. If the 'get_evaluate_time' is True,
+            the return value will be (Results, Time).
         """
-        start_time = time.time()
         future = self.pool.submit(
             self.evaluator.secure_evaluate,
             program,
             timeout_seconds,
             redirect_to_devnull,
             multiprocessing_start_method,
+            return_time,
             **kwargs
         )
         res = future.result()
-        duration = time.time() - start_time
-        if return_time:
-            return res, duration
-        else:
-            return res
+        return res

{py_adtools-0.1.7.dist-info → py_adtools-0.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-adtools
-Version: 0.1.7
+Version: 0.1.8
 Summary: Useful tools for parsing and evaluating Python programs for LLM-based algorithm design.
 Home-page: https://github.com/RayZhhh/py-adtools
 Author: Rui Zhang

{py_adtools-0.1.7.dist-info → py_adtools-0.1.8.dist-info}/RECORD

@@ -1,13 +1,13 @@
 adtools/__init__.py,sha256=kbxntZFeCcURiIypNOdMWyeKPdlzRsWOB-K7z6HNCsc,150
-adtools/evaluator.py,sha256=weA6zR1WyUE3f5pt7wQYF1ukmkA-e2kDLaogbDmG_Ig,9154
-adtools/evaluator_pool.py,sha256=v_NZibN4VI3STVUZt6ARdyoB4Z061xAefZlH8lkWsjE,2972
+adtools/evaluator.py,sha256=A33E0HmLsIMtMbdjRmfPzck9gDodJc96gvXR0cL_SR0,10577
+adtools/evaluator_pool.py,sha256=zfQ7DgAjpByqPM5SE1tRQT_HGXU5uwNntvguzrXaPEk,3258
 adtools/py_code.py,sha256=FZfkp-IZ4zpOjrWe6svKNJsQhVANaTTkE0l0mc4aMW8,14277
 adtools/lm/__init__.py,sha256=PZf5Lraly9rAWz-cxOSLCvZ9OZ4EI8aQEluetvNX8LM,146
 adtools/lm/lm_base.py,sha256=KtO7KTrrMW7oWN-BhncoIOsbOVQsSc-0gNCYtvR6Sog,1105
 adtools/lm/openai_api.py,sha256=LcfLkNOBrJTdsp0zcUjaCelIcQK5XknpHWrlB0S67_k,2390
 adtools/lm/vllm_server.py,sha256=BPZoTS77wNJDcJ_0FO2QFyZTf6WR0isYKMuTctqKEU8,12942
-py_adtools-0.1.7.dist-info/licenses/LICENSE,sha256=E5GGyecx3y5h2gcEGQloF-rDY9wbaef5IHjRsvtFbt8,1065
-py_adtools-0.1.7.dist-info/METADATA,sha256=952zQgrdHcpBYGmnYqfwlVFfMubvvSYixg-weIf4eWU,6386
-py_adtools-0.1.7.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
-py_adtools-0.1.7.dist-info/top_level.txt,sha256=X2kKzmJFDAKR2FWCij5pfMG9pVVjVUomyl4e-1VLXIk,8
-py_adtools-0.1.7.dist-info/RECORD,,
+py_adtools-0.1.8.dist-info/licenses/LICENSE,sha256=E5GGyecx3y5h2gcEGQloF-rDY9wbaef5IHjRsvtFbt8,1065
+py_adtools-0.1.8.dist-info/METADATA,sha256=22hW8kcx1OxxlcedTDYIX1EzfB79pRYEtU3cXptveE8,6386
+py_adtools-0.1.8.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
+py_adtools-0.1.8.dist-info/top_level.txt,sha256=X2kKzmJFDAKR2FWCij5pfMG9pVVjVUomyl4e-1VLXIk,8
+py_adtools-0.1.8.dist-info/RECORD,,