py-adtools 0.3.2__py3-none-any.whl

adtools/__init__.py

@@ -0,0 +1 @@
+from adtools.py_code import PyCodeBlock, PyFunction, PyClass, PyProgram

adtools/cli.py

@@ -0,0 +1,61 @@
+import argparse
+import sys
+
+# Import the main function of the auto_server
+from adtools.evaluator.auto_server import main as auto_server_main
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="ADTools CLI for various utilities.",
+        formatter_class=argparse.RawTextHelpFormatter,  # Preserve formatting for help messages
+    )
+
+    # Define subcommands
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Serve subcommand
+    serve_parser = subparsers.add_parser(
+        "serve",
+        help="Launch the Auto-Evaluation Server. All arguments are passed directly to the server.",
+        formatter_class=argparse.RawTextHelpFormatter,  # Preserve formatting for help messages
+    )
+
+    # The 'serve' command doesn't define its own arguments here.
+    # Instead, it will capture all subsequent arguments and pass them to auto_server_main.
+    # This simplifies argument parsing, as auto_server_main already handles its own args.
+
+    # Parse arguments provided to the main 'adtools' command
+    # We only parse the initial command (e.g., 'adtools serve')
+    # and then pass the rest of the arguments to the subcommand's main function.
+
+    # If no subcommand is given, print help
+    if len(sys.argv) == 1:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
+    # Parse the main command and subcommand
+    args = parser.parse_args(sys.argv[1:2])  # Only parse 'adtools' and 'serve'
+
+    if args.command == "serve":
+        # Pass all remaining arguments (from sys.argv[2:]) to the auto_server_main function.
+        # This effectively makes 'adtools serve ARGS...' behave like 'adtools.evaluator.auto_server.main(ARGS...)'.
+        # We need to temporarily replace sys.argv for auto_server_main to parse correctly.
+        original_sys_argv = sys.argv
+        sys.argv = [original_sys_argv[0]] + original_sys_argv[
+            2:
+        ]  # Keep script name, then add server args
+
+        try:
+            auto_server_main()
+        finally:
+            sys.argv = original_sys_argv  # Restore sys.argv
+    else:
+        # This case should ideally not be reached if subparsers are configured correctly
+        # and 'dest="command"' is used.
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

adtools/evaluator/__init__.py

@@ -0,0 +1,2 @@
+from adtools.evaluator.py_evaluator import PyEvaluator
+from adtools.evaluator.py_evaluator_ray import PyEvaluatorRay

adtools/evaluator/auto_server.py

@@ -0,0 +1,258 @@
+import json
+import os
+import sys
+import argparse
+import importlib.util
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Optional, Dict
+
+import requests
+
+from adtools import PyClass
+from adtools.evaluator import PyEvaluator, PyEvaluatorRay
+
+__all__ = ["submit_code", "submit_code_async"]
+
+
+def submit_code(
+    host: str,
+    port: int | str,
+    code: str,
+    timeout: Optional[float] = None,
+    *,
+    post_timeout_seconds: float = 1800,
+) -> Dict:
+    """Submit code to the evaluation server.
+
+    Args:
+        host: Server host.
+        port: Server port.
+        code: Code to submit.
+        timeout: evaluation timeout in seconds.
+        post_timeout_seconds: Post request timeout in seconds.
+
+    Returns:
+        A dict containing the evaluation metadata.
+    """
+    url = f"http://{host}:{port}/"
+    payload = {"code": code, "timeout": timeout}
+
+    with requests.Session() as s:
+        r = s.post(url, json=payload, timeout=post_timeout_seconds)
+        r.raise_for_status()
+        return r.json()
+
+
+async def submit_code_async(
+    host: str,
+    port: int | str,
+    code: str,
+    timeout: Optional[float] = None,
+    *,
+    post_timeout_seconds: float = 1800,
+) -> Dict:
+    """Submit code to the evaluation server.
+
+    Args:
+        host: Server host.
+        port: Server port.
+        code: Code to submit.
+        timeout: evaluation timeout in seconds.
+        post_timeout_seconds: Post request timeout in seconds.
+
+    Returns:
+        A dict containing the evaluation metadata.
+    """
+    try:
+        import aiohttp
+    except ImportError:
+        raise ImportError("Please install 'aiohttp'.")
+
+    url = f"http://{host}:{port}/"
+    payload = {"code": code, "timeout": timeout}
+    timeout_cfg = aiohttp.ClientTimeout(total=post_timeout_seconds)
+
+    async with aiohttp.ClientSession(timeout=timeout_cfg) as session:
+        async with session.post(url, json=payload) as resp:
+            resp.raise_for_status()
+            return await resp.json()
+
+
+class EvaluationHandler(BaseHTTPRequestHandler):
+
+    def do_POST(self):
+        try:
+            self.connection.settimeout(10)
+            content_length = int(self.headers.get("Content-Length", 0))
+            post_data = self.rfile.read(content_length)
+
+            try:
+                req_data = json.loads(post_data.decode("utf-8"))
+            except Exception:
+                req_data = {}
+
+            code_str = req_data.get("code")
+            # Use server-level default timeout if not provided
+            timeout = req_data.get("timeout", self.server.default_timeout)
+
+            if not code_str:
+                response = {
+                    "result": None,
+                    "evaluate_time": 0.0,
+                    "error_msg": "No 'code' field in request or invalid JSON",
+                }
+            else:
+                # Use the semaphore to limit concurrent evaluations
+                # This blocks the thread handling this request until a slot is free
+                with self.server.semaphore:
+                    results = self.server.evaluator.secure_evaluate(
+                        code_str, timeout_seconds=timeout
+                    )
+                    response = dict(results)
+
+            # Ensure serialization
+            try:
+                json_response = json.dumps(response)
+            except (TypeError, OverflowError):
+                # If result is not serializable, convert it to string
+                response["result"] = str(response["result"])
+                json_response = json.dumps(response)
+
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            self.wfile.write(json_response.encode("utf-8"))
+
+        except Exception as e:
+            error_msg = str(e)
+            print(f"Server Error: {error_msg}")
+            error_response = json.dumps(
+                {
+                    "result": None,
+                    "evaluate_time": 0.0,
+                    "error_msg": f"Server internal error: {error_msg}",
+                }
+            )
+            # We still return 200 OK because the *HTTP* request succeeded,
+            # but the *application* (evaluation) had an error.
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            self.wfile.write(error_response.encode("utf-8"))
+
+    def log_message(self, format, *args):
+        pass
+
+
+class ThreadedHTTPServer(ThreadingHTTPServer):
+    # Allow passing custom attributes to the server instance
+    pass
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d", "--dir", required=True, help="Directory (file path) of the evaluator."
+    )
+    parser.add_argument("--host", default="0.0.0.0", help="Host of the server.")
+    parser.add_argument("--port", default=8000, type=int, help="Port of the server.")
+    parser.add_argument(
+        "-t", "--timeout", default=None, type=float, help="Default timeout in seconds."
+    )
+    parser.add_argument(
+        "--max-workers", default=4, type=int, help="Max concurrent evaluations."
+    )
+    args = parser.parse_args()
+
+    # Read file
+    with open(args.dir) as f:
+        program = f.read()
+
+    # Extract all classes
+    classes = PyClass.extract_all_classes_from_text(program)
+
+    # Count the number of public classes
+    count_public_classes = 0
+    public_class_name = None
+    for cls in classes:
+        if not cls.name.startswith("_"):
+            count_public_classes += 1
+            public_class_name = cls.name
+
+    if count_public_classes == 0:
+        raise Exception("No public classes found.")
+    if count_public_classes > 1:
+        raise Exception(
+            f"The file should only have one pubic class, "
+            f"but found {count_public_classes}"
+        )
+
+    # Import evaluator from directory
+    file_path = os.path.abspath(args.dir)
+    dir_name = os.path.dirname(file_path)
+    base_name = os.path.basename(file_path)
+    module_name = os.path.splitext(base_name)[0]
+
+    # Add to sys.path for current process
+    if dir_name not in sys.path:
+        sys.path.insert(0, dir_name)
+
+    # Add to PYTHONPATH for child processes (multiprocessing spawn)
+    current_pythonpath = os.environ.get("PYTHONPATH", "")
+    if dir_name not in current_pythonpath.split(os.pathsep):
+        os.environ["PYTHONPATH"] = (
+            (dir_name + os.pathsep + current_pythonpath)
+            if current_pythonpath
+            else dir_name
+        )
+
+    module = importlib.import_module(module_name)
+    EvaluatorClass = getattr(module, public_class_name)
+
+    # Assert the evaluator is either "PyEvaluator" or "PyEvaluatorRay"
+    if not issubclass(EvaluatorClass, (PyEvaluator, PyEvaluatorRay)):
+        raise TypeError(
+            f"Class {public_class_name} must inherit from PyEvaluator or PyEvaluatorRay"
+        )
+
+    # Instantiate the evaluator
+    evaluator = EvaluatorClass()
+
+    # Check whether timeout is set
+    timeout_defined_in_class = None
+    for field in ["timeout_seconds, timeout, _timeout_seconds, _timeout"]:
+        if hasattr(evaluator, field):
+            timeout_defined_in_class = getattr(evaluator, field)
+
+    # Initialize Threaded HTTP Server
+    # We use ThreadedHTTPServer to handle requests in separate threads
+    server = ThreadedHTTPServer((args.host, args.port), EvaluationHandler)
+
+    # Attach shared resources to the server instance so handlers can access them
+    server.evaluator = evaluator
+    server.default_timeout = args.timeout or timeout_defined_in_class
+    server.semaphore = threading.Semaphore(args.max_workers)
+
+    print(f"Evaluator '{public_class_name}' loaded from {args.dir}")
+    print(
+        f"HTTP Server running at http://{args.host}:{args.port} with max_workers={args.max_workers}"
+    )
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        server.server_close()
+
+
+if __name__ == "__main__":
+    import multiprocessing
+
+    try:
+        multiprocessing.set_start_method("spawn", force=True)
+    except RuntimeError:
+        pass
+
+    main()

adtools/evaluator/py_evaluator.py

@@ -0,0 +1,170 @@
+"""
+Copyright (c) 2025 Rui Zhang <rzhang.cs@gmail.com>
+
+NOTICE: This code is under MIT license. This code is intended for academic/research purposes only.
+Commercial use of this software or its derivatives requires prior written permission.
+"""
+
+import time
+import traceback
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Callable, List
+
+from adtools.py_code import PyProgram
+from adtools.sandbox import SandboxExecutor, ExecutionResults
+
+__all__ = [
+    "PyEvaluator",
+]
+
+
+class PyEvaluator(ABC):
+
+    def __init__(
+        self,
+        exec_code: bool = True,
+        find_and_kill_children_evaluation_process: bool = False,
+        debug_mode: bool = False,
+        *,
+        join_timeout_seconds: int = 10,
+    ):
+        """Evaluator interface for evaluating the Python algorithm program. Override this class and implement
+        'evaluate_program' method, then invoke 'self.evaluate()' or 'self.secure_evaluate()' for evaluation.
+
+        Args:
+            exec_code: Using 'exec()' to execute the program code and obtain the callable functions and classes,
+                which will be passed to 'self.evaluate_program()'. Set this parameter to 'False' if you are going to
+                evaluate a Python scripy. Note that if the parameter is set to 'False', the arguments 'callable_...'
+                in 'self.evaluate_program()' will no longer be affective.
+            find_and_kill_children_evaluation_process: If using 'self.secure_evaluate', kill children processes
+                when they are terminated. Note that it is suggested to set to 'False' if the evaluation process
+                does not start new processes.
+            debug_mode: Debug mode.
+            join_timeout_seconds: Timeout in seconds to wait for the process to finish. Kill the process if timeout.
+        """
+        self.debug_mode = debug_mode
+        self.exec_code = exec_code
+        self.sandbox_executor = SandboxExecutor(
+            evaluate_worker=self,
+            find_and_kill_children_evaluation_process=find_and_kill_children_evaluation_process,
+            debug_mode=debug_mode,
+            join_timeout_seconds=join_timeout_seconds,
+        )
+
+    @abstractmethod
+    def evaluate_program(
+        self,
+        program_str: str,
+        callable_functions_dict: Dict[str, Callable] | None,
+        callable_functions_list: List[Callable] | None,
+        callable_classes_dict: Dict[str, Callable] | None,
+        callable_classes_list: List[Callable] | None,
+        **kwargs,
+    ) -> Any:
+        """Evaluate a given program.
+
+        Args:
+            program_str: The raw program text.
+            callable_functions_dict: A dictionary where keys are the names of functions
+                defined in the `program_str` and values are the corresponding callable function objects.
+            callable_functions_list: A list of callable function objects
+                defined in the `program_str`, ordered as they appear in the program.
+            callable_classes_dict: A dictionary where keys are the names of classes
+                defined in the `program_str` and values are the corresponding callable class objects.
+            callable_classes_list: A list of callable class objects
+                defined in the `program_str`, ordered as they appear in the program.
+
+        Returns:
+            Returns the evaluation result.
+        """
+        raise NotImplementedError(
+            "Must provide an evaluator for a python program. "
+            "Override this method in a subclass."
+        )
+
+    def _exec_and_get_res(self, program: str | PyProgram, **kwargs):
+        """Evaluate a program.
+
+        Args:
+            program: the program to be evaluated.
+            **kwargs: additional keyword arguments to pass to 'evaluate_program'.
+        """
+        # Parse to program instance
+        if isinstance(program, str):
+            program = PyProgram.from_text(program)
+        function_names = [f.name for f in program.functions]
+        class_names = [c.name for c in program.classes]
+
+        # Execute the code and get callable instances
+        if self.exec_code:
+            all_globals_namespace = {}
+            # Execute the program, map func/var/class to global namespace
+            exec(str(program), all_globals_namespace)
+            # Get callable functions
+            callable_funcs_list = [
+                all_globals_namespace[f_name] for f_name in function_names
+            ]
+            callable_funcs_dict = dict(zip(function_names, callable_funcs_list))
+            # Get callable classes
+            callable_cls_list = [
+                all_globals_namespace[c_name] for c_name in class_names
+            ]
+            callable_cls_dict = dict(zip(class_names, callable_cls_list))
+        else:
+            (
+                callable_funcs_list,
+                callable_funcs_dict,
+                callable_cls_list,
+                callable_cls_dict,
+            ) = (None, None, None, None)
+
+        # Get evaluate result
+        res = self.evaluate_program(
+            str(program),
+            callable_funcs_dict,
+            callable_funcs_list,
+            callable_cls_dict,
+            callable_cls_list,
+            **kwargs,
+        )
+        return res
+
+    def evaluate(self, program: str | PyProgram, **kwargs) -> ExecutionResults:
+        start_time = time.time()
+        error_msg = ""
+        # noinspection PyBroadException
+        try:
+            res = self._exec_and_get_res(program, **kwargs)
+        except:
+            res = None
+            error_msg = str(traceback.format_exc())
+
+        return ExecutionResults(
+            result=res, evaluate_time=time.time() - start_time, error_msg=error_msg
+        )
+
+    def secure_evaluate(
+        self,
+        program: str | PyProgram,
+        timeout_seconds: int | float = None,
+        redirect_to_devnull: bool = False,
+        **kwargs,
+    ) -> ExecutionResults:
+        """Evaluate program in a new process. This enables timeout restriction and output redirection.
+
+        Args:
+            program: the program to be evaluated.
+            timeout_seconds: return 'None' if the execution time exceeds 'timeout_seconds'.
+            redirect_to_devnull: redirect any output to '/dev/null'.
+            **kwargs: additional keyword arguments to pass to 'evaluate_program'.
+
+        Returns:
+            Returns the evaluation results.
+        """
+        return self.sandbox_executor.secure_execute(
+            worker_execute_method_name="_exec_and_get_res",
+            method_args=[program],
+            method_kwargs=kwargs,
+            timeout_seconds=timeout_seconds,
+            redirect_to_devnull=redirect_to_devnull,
+        )

adtools/evaluator/py_evaluator_ray.py

@@ -0,0 +1,110 @@
+"""
+Copyright (c) 2025 Rui Zhang <rzhang.cs@gmail.com>
+
+NOTICE: This code is under MIT license. This code is intended for academic/research purposes only.
+Commercial use of this software or its derivatives requires prior written permission.
+"""
+
+from abc import abstractmethod
+from typing import Any, Dict, List, Callable
+
+from adtools.py_code import PyProgram
+from adtools.evaluator.py_evaluator import PyEvaluator
+from adtools.sandbox.sandbox_executor_ray import SandboxExecutorRay, ExecutionResults
+
+
+__all__ = ["PyEvaluatorRay"]
+
+
+class PyEvaluatorRay(PyEvaluator):
+
+    def __init__(
+        self,
+        init_ray: bool = True,
+        exec_code: bool = True,
+        debug_mode: bool = False,
+        *,
+        ray_rotation_max_bytes: int = 50 * 1024 * 1024,  # 50 MB
+        ray_rotation_backup_count: int = 1,
+    ):
+        """Evaluator using Ray for secure, isolated execution.
+        It supports efficient zero-copy return of large objects (e.g., Tensors).
+
+        Args:
+            init_ray: Whether to initialize the ray.
+            exec_code: Whether to execute the code using 'exec()'.
+            debug_mode: Enable debug print statements.
+        """
+        super().__init__(
+            exec_code=exec_code,
+            debug_mode=debug_mode,
+        )
+
+        self.sandbox_executor = SandboxExecutorRay(
+            evaluate_worker=self,
+            init_ray=init_ray,
+            debug_mode=debug_mode,
+            ray_rotation_max_bytes=ray_rotation_max_bytes,
+            ray_rotation_backup_count=ray_rotation_backup_count,
+        )
+
+    @abstractmethod
+    def evaluate_program(
+        self,
+        program_str: str,
+        callable_functions_dict: Dict[str, Callable] | None,
+        callable_functions_list: List[Callable] | None,
+        callable_classes_dict: Dict[str, Callable] | None,
+        callable_classes_list: List[Callable] | None,
+        **kwargs,
+    ) -> Any:
+        """Evaluate a given program.
+
+        Args:
+            program_str: The raw program text.
+            callable_functions_dict: A dictionary where keys are the names of functions
+                defined in the `program_str` and values are the corresponding callable function objects.
+            callable_functions_list: A list of callable function objects
+                defined in the `program_str`, ordered as they appear in the program.
+            callable_classes_dict: A dictionary where keys are the names of classes
+                defined in the `program_str` and values are the corresponding callable class objects.
+            callable_classes_list: A list of callable class objects
+                defined in the `program_str`, ordered as they appear in the program.
+
+        Returns:
+            Returns the evaluation result.
+        """
+        raise NotImplementedError(
+            "Must provide an evaluator for a python program. "
+            "Override this method in a subclass."
+        )
+
+    def secure_evaluate(
+        self,
+        program: str | PyProgram,
+        timeout_seconds: int | float = None,
+        redirect_to_devnull: bool = False,
+        *,
+        ray_actor_options: dict[str, Any] = None,
+        **kwargs,
+    ) -> ExecutionResults:
+        """Evaluates the program in a separate Ray Actor (process).
+
+        Args:
+            program: the program to be evaluated.
+            timeout_seconds: return 'None' if the execution time exceeds 'timeout_seconds'.
+            redirect_to_devnull: redirect any output to '/dev/null'.
+            ray_actor_options: kwargs pass to RayWorkerClass.options(...).
+            **kwargs: additional keyword arguments to pass to 'evaluate_program'.
+
+        Returns:
+            Returns the evaluation results.
+        """
+        return self.sandbox_executor.secure_execute(
+            worker_execute_method_name="_exec_and_get_res",
+            method_args=[program],
+            method_kwargs=kwargs,
+            timeout_seconds=timeout_seconds,
+            redirect_to_devnull=redirect_to_devnull,
+            ray_actor_options=ray_actor_options,
+        )

adtools/lm/__init__.py

@@ -0,0 +1,4 @@
+from adtools.lm.lm_base import LanguageModel
+from adtools.lm.openai_api import OpenAIAPI
+from adtools.lm.vllm_server import VLLMServer
+from adtools.lm.sglang_server import SGLangServer

adtools/lm/lm_base.py

@@ -0,0 +1,63 @@
+"""
+Copyright (c) 2025 Rui Zhang <rzhang.cs@gmail.com>
+
+NOTICE: This code is under MIT license. This code is intended for academic/research purposes only.
+Commercial use of this software or its derivatives requires prior written permission.
+"""
+
+from abc import abstractmethod
+from typing import List, Optional
+
+import openai.types.chat
+
+
+class LanguageModel:
+    """Base class for language model interface."""
+
+    def chat_completion(
+        self,
+        message: str | List[openai.types.chat.ChatCompletionMessageParam],
+        max_tokens: int,
+        timeout_seconds: float,
+        *args,
+        **kwargs,
+    ):
+        """Send a chat completion query with OpenAI format to the vLLM server.
+        Return the response content.
+
+        Args:
+            message: The message in str or openai format.
+            max_tokens: The maximum number of tokens to generate.
+            timeout_seconds: The timeout seconds.
+        """
+        pass
+
+    def embedding(
+        self,
+        text: str | List[str],
+        dimensions: Optional[int] = None,
+        timeout_seconds: Optional[float] = None,
+        **kwargs,
+    ) -> List[float] | List[List[float]]:
+        """Generate embeddings for the given text(s) using the model specified during initialization.
+
+        Args:
+            text: The text or a list of texts to embed.
+            dimensions: The number of dimensions for the output embeddings.
+            timeout_seconds: The timeout seconds.
+
+        Returns:
+            The embedding for the text, or a list of embeddings for the list of texts.
+        """
+        pass
+
+    def close(self):
+        """Release resources (if necessary)."""
+        pass
+
+    def reload(self):
+        """Reload the language model (if necessary)."""
+        pass
+
+    def __del__(self):
+        self.close()