evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/live_code_bench/prompts.py

@@ -0,0 +1,207 @@
+# Copyright LiveCodeBench @ 2024,
+
+import json
+
+
+class CodeGenerationPromptConstants:
+    SYSTEM_MESSAGE_GENERIC = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.'  # noqa: E501
+
+    SYSTEM_MESSAGE_GEMINI = 'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. Do NOT use system calls like `exit` in the generated program.'  # noqa: E501
+
+    SYSTEM_MESSAGE_DEEPSEEK = 'You are an AI programming assistant, utilizing the DeepSeek Coder model, developed by DeepSeek Company, and you answer questions related to computer science.'  # noqa: E501
+
+    SYSTEM_MESSAGE_MAGIC = 'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n@@ Instruction\n'  # noqa: E501
+
+    SYSTEM_MESSAGE_WIZARD = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'  # noqa: E501
+
+    SYSTEM_MESSAGE_PHIND = """You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. Put your fixed program within code delimiters, for example:
+```python
+# YOUR CODE HERE
+```"""  # noqa: E501
+
+    SYSTEM_MESSAGE_CODEQWEN = (
+        '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user'  # noqa: E501
+    )
+
+    FORMATTING_MESSAGE_WITH_STARTER_CODE = 'You will use the following starter code to write the solution to the problem and enclose your code within delimiters.'  # noqa: E501
+
+    FORMATTING_WITHOUT_STARTER_CODE = 'Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows.'  # noqa: E501
+
+    PYTHON_FORMAT = '```python\n# YOUR CODE HERE\n```\n\n'
+
+
+class TestOutputPromptConstants:
+    SYSTEM_MESSAGE_CHAT_GENERIC = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user to write a test case to help to check the correctness of the function. The user has written a input for the testcase. You will calculate the output of the testcase and write the whole assertion statement in the markdown code block with the correct output.'  # noqa: E501
+
+    SYSTEM_MESSAGE_COMPLETION_GENERIC = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user to write a test case to help to check the correctness of the function.'  # noqa: E501
+
+    SYSTEM_MESSAGE_INST_CLLAMA = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user to write a test case to help to check the correctness of the function. The user has written a input for the testcase. You will calculate the output of the testcase and write out the complete assertion statement between [PYTHON] and [/PYTHON] tags.'  # noqa: E501
+
+    SYSTEM_MESSAGE_WIZARD = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'  # noqa: E501
+
+    SYSTEM_MESSAGE_PHIND = """You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. You must put the entire fixed program within code delimiters only for once., for example:
+```python
+# YOUR CODE HERE
+```"""  # noqa: E501
+
+    FORMATTING_MESSAGE = 'You will use the following starter code to write the solution to the problem and enclose your code within delimiters.'  # noqa: E501
+
+    FORMATTING_WITHOUT_STARTER_MESSAGE = 'Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows.'  # noqa: E501
+
+
+class SelfRepairPromptConstants:
+    SYSTEM_MESSAGE_GENERIC = 'You are a helpful programming assistant and an expert Python programmer. You are helping a user write a program to solve a problem. The user has written some code, but it has some errors and is not passing the tests. You will help the user by first giving a concise (at most 2-3 sentences) textual explanation of what is wrong with the code. After you have pointed out what is wrong with the code, you will then generate a fixed version of the program. You must put the entire fixed program within code delimiters only for once.'  # noqa: E501
+
+    SYSTEM_MESSAGE_DEEPSEEK = 'You are an AI programming assistant, utilizing the DeepSeek Coder model, developed by DeepSeek Company, and you are helping a user correct a error program for code competition. The user has written some code, but it has some errors and is not passing the tests. You will help the user by first giving a concise (at most 2-3 sentences) textual explanation of what is wrong with the code. After you have pointed out what is wrong with the code, you will then generate a fixed version of the entire executable program. You must put the entire fixed executable program within code delimiters.'  # noqa: E501
+
+    SYSTEM_MESSAGE_MAGIC = 'You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.\n\n@@ Instruction\n'  # noqa: E501
+
+    SYSTEM_MESSAGE_WIZARD = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'  # noqa: E501
+
+    SYSTEM_MESSAGE_PHIND = """You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. You must put the entire fixed program within code delimiters only for once., for example:
+```python
+# YOUR CODE HERE
+```"""  # noqa: E501
+
+    FORMATTING_REPEAT = 'First reason about the code providing a textual explanation of what is wrong with the code and then generate a fixed of the program enclosed code delimiters.'  # noqa: E501
+
+    FORMATTING_MESSAGE = 'You will use the following starter code to write the solution to the problem and enclose your code within delimiters.'  # noqa: E501
+
+    FORMATTING_WITHOUT_STARTER_CODE = 'Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows.'  # noqa: E501
+
+
+def make_code_execution_prompt(code, input, cot):
+    if cot:
+        # make_cot_output_prompt
+        return f"""You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Execute the program step by step before arriving at an answer, and provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples.
+
+[PYTHON]
+def performOperation(s):
+    s = s + s
+    return "b" + s + "a"
+assert performOperation(s = "hi") == ??
+[/PYTHON]
+[THOUGHT]
+Let's execute the code step by step:
+
+1. The function performOperation is defined, which takes a single argument s.
+2. The function is called with the argument "hi", so within the function, s is initially "hi".
+3. Inside the function, s is concatenated with itself, so s becomes "hihi".
+4. The function then returns a new string that starts with "b", followed by the value of s (which is now "hihi"), and ends with "a".
+5. The return value of the function is therefore "bhihia".
+[/THOUGHT]
+[ANSWER]
+assert performOperation(s = "hi") == "bhihia"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert {input} == ??
+[/PYTHON]
+[THOUGHT]
+"""  # noqa: E501
+    else:
+        # make_direct_output_prompt
+        return f"""You are given a Python function and an assertion containing an input to the function. Complete the assertion with a literal (no unsimplified expressions, no function calls) containing the output when executing the provided code on the given input, even if the function is incorrect or incomplete. Do NOT output any extra information. Provide the full assertion with the correct output in [ANSWER] and [/ANSWER] tags, following the examples.
+
+[PYTHON]
+def repeatNumber(number : int) -> int:
+    return number
+assert repeatNumber(number = 17) == ??
+[/PYTHON]
+[ANSWER]
+assert repeatNumber(number = 17) == 17
+[/ANSWER]
+
+[PYTHON]
+def addCharacterA(string : str) -> str:
+    return string + "a"
+assert addCharacterA(string = "x9j") == ??
+[/PYTHON]
+[ANSWER]
+assert addCharacterA(string = "x9j") == "x9ja"
+[/ANSWER]
+
+[PYTHON]
+{code}
+assert {input} == ??
+[/PYTHON]
+[ANSWER]
+"""  # noqa: E501
+
+
+def get_generic_question_template_test_completion(question_content, starter_code, testcase_input: str):
+
+    def format_testcase_func_name_input(function_name, testcase):
+        """use the form of "assert func_name(input) == "."""
+        # TODO should there be a space after the == ?
+        input_str = ', '.join(testcase.split('\n'))
+        return f'assert {function_name}({input_str}) == # TODO'
+
+    def parse_function_name_from_starter_code(starter_code):
+        """
+        starter_code : str
+        """
+        import ast
+
+        tree = ast.parse(starter_code)
+        fn = None
+        for node in ast.walk(tree):
+            if isinstance(node, ast.FunctionDef):
+                assert fn is None
+                fn = node.name
+        return fn
+
+    prompt = f'Problem:\n{question_content}'
+    prompt += f'Function:\n```\n{starter_code}\n```\n'
+
+    # parse function name from starter_code
+    func_name = parse_function_name_from_starter_code(starter_code)
+    prompt += 'Please complete the following test case:\n\n'
+    prompt += (
+        f'```\n{format_testcase_func_name_input(func_name, testcase_input)}\n```\n'  # noqa: E501
+    )
+
+    return prompt
+
+
+def get_generic_question_template_answer_self_repair(question: str, code, metadata):
+
+    def get_check_prompt(metadata):
+        # def get_check_prompt(question: str, result, metadata):
+        # # assumes i/o examples are already truncated!
+        # # less pressure on storing 10 MB json because on a single large
+        # # input-output pair
+        # result_by_test_case = result
+        # assert len(metadata) == 1, f"metadata = {metadata}"
+        # metadata = metadata[0]
+        metadata = json.loads(metadata)
+        if 'error_code' not in metadata:
+            return ''
+        if metadata['error_code'] == -1:
+            # time limit exceeded
+            message = f"The above code is incorrect and got the following compilation error.\n{metadata['error']}"  # noqa: E501
+        elif metadata['error_code'] == -2:
+            # wrong answer
+            message = f"The above code is incorrect and got a wrong answer.\nInput: {metadata['inputs']}\nGenerated Output: {metadata['output']}\nExpected: {metadata['expected']}"  # noqa: E501
+        elif metadata['error_code'] == -3:
+            # time limit exceeded
+            message = f"The above code is incorrect and got time limit exceeded.\n{metadata['error']}\nInput: {metadata['inputs']}\nExpected: {metadata['expected']}"  # noqa: E501
+            pass
+        elif metadata['error_code'] == -4:
+            # runtime error
+            message = f"The above code is incorrect and got a runtime error.\nInput: {metadata['inputs']}\nExpected: {metadata['expected']}\n{metadata['error']}"  # noqa: E501
+        else:
+            raise NotImplementedError(
+                f"metadata['error_code'] = {metadata['error_code']} not implemented || {metadata=}"  # noqa: E501
+            )
+        return message
+
+    prompt = f'### Question:\n{question}\n\n'
+    prompt += f'### Answer:\n```python\n{code}\n```\n\n'
+    # prompt += get_check_prompt(question, result, metadata) + "\n"
+    prompt += get_check_prompt(metadata) + '\n'
+    prompt += f'### Format: {SelfRepairPromptConstants.FORMATTING_WITHOUT_STARTER_CODE}\n'  # noqa: E501
+    prompt += '```python\n# YOUR CODE HERE\n```\n\n'
+    prompt += '### Answer: (use the provided format with backticks)\n\n'
+    return prompt

evalscope/benchmarks/live_code_bench/sandbox_evaluate_utils.py

@@ -0,0 +1,220 @@
+import json
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.api.mixin.sandbox_mixin import SandboxMixin
+
+logger = get_logger()
+
+
+def evaluate_in_sandbox(
+    adapter: 'SandboxMixin',
+    code: str,
+    evaluation_sample: str,
+    timeout: int = 6,
+    debug: bool = False
+) -> Tuple[bool, Dict]:
+    """
+    Evaluate code in sandbox environment for Live Code Bench.
+
+    Args:
+        adapter: The adapter instance with sandbox capabilities
+        code: The code to evaluate
+        evaluation_sample: JSON string containing input/output test cases
+        timeout: Timeout for execution
+        debug: Whether to enable debug logging
+
+    Returns:
+        Tuple[bool, Dict]: (overall_pass, detailed_results)
+    """
+    try:
+        # Parse the evaluation sample
+        test_data = json.loads(evaluation_sample)
+        inputs = test_data.get('inputs', [])
+        outputs = test_data.get('outputs', [])
+        fn_name = test_data.get('fn_name')
+
+        if debug:
+            logger.info(f'Evaluating code with {len(inputs)} test cases')
+            logger.info(f'Function name: {fn_name}')
+
+        # Determine if this is call-based or stdio-based
+        if fn_name:
+            # Call-based evaluation
+            return _evaluate_call_based_in_sandbox(adapter, code, inputs, outputs, fn_name, timeout, debug)
+        else:
+            # Standard input/output evaluation
+            return _evaluate_stdio_in_sandbox(adapter, code, inputs, outputs, timeout, debug)
+
+    except Exception as e:
+        if debug:
+            logger.error(f'Sandbox evaluation error: {str(e)}')
+        return False, {'error': str(e), 'total_tests': 0, 'passed_tests': 0}
+
+
+def _evaluate_call_based_in_sandbox(
+    adapter: 'SandboxMixin', code: str, inputs: list, outputs: list, fn_name: str, timeout: int, debug: bool
+) -> Tuple[bool, Dict]:
+    """Evaluate call-based problems in sandbox."""
+    try:
+        all_passed = True
+        passed_count = 0
+        failed_cases = []
+
+        for i, (test_input, expected_output) in enumerate(zip(inputs, outputs)):
+            # Prepare individual test code for each test case
+            test_code = f"""
+import json
+import sys
+
+# User's code
+{code}
+
+# Test execution for single test case
+try:
+    test_input = {repr(test_input)}
+    expected_output = {repr(expected_output)}
+
+    if 'class Solution' in '''{code}''':
+        # LeetCode style
+        solution = Solution()
+        method = getattr(solution, '{fn_name}')
+    else:
+        # Function is directly available
+        method = {fn_name}
+
+    # Parse input if it's JSON string
+    if isinstance(test_input, str):
+        try:
+            test_input = json.loads(test_input)
+        except:
+            pass  # Keep as string if not valid JSON
+
+    # Call the method
+    if isinstance(test_input, list):
+        result = method(*test_input)
+    else:
+        result = method(test_input)
+
+    # Parse expected output if it's JSON string
+    if isinstance(expected_output, str):
+        try:
+            expected_output = json.loads(expected_output)
+        except:
+            pass  # Keep as string if not valid JSON
+
+    # Convert tuple to list for comparison
+    if isinstance(result, tuple):
+        result = list(result)
+
+    if result == expected_output:
+        print("TEST_PASSED")
+    else:
+        print(f"TEST_FAILED: expected {{expected_output}}, got {{result}}")
+
+except Exception as e:
+    print(f"EXECUTION_ERROR: {{str(e)}}")
+    import traceback
+    traceback.print_exc()
+"""
+
+            # Execute in sandbox
+            result = adapter.execute_code_in_sandbox(code=test_code, timeout=timeout, language='python')
+
+            if debug:
+                logger.info(f'Test case {i} execution result: {result}')
+
+            # Check if execution was successful and test passed
+            if result.get('status') == 'success':
+                output = result.get('output', '')
+                if 'TEST_PASSED' in output:
+                    passed_count += 1
+                elif 'TEST_FAILED:' in output:
+                    # Extract failure details from output
+                    for line in output.split('\n'):
+                        if line.startswith('TEST_FAILED:'):
+                            failed_cases.append(f"Test {i}: {line.replace('TEST_FAILED: ', '')}")
+                            break
+                    all_passed = False
+                    break
+                elif 'EXECUTION_ERROR:' in output:
+                    # Extract error details
+                    for line in output.split('\n'):
+                        if line.startswith('EXECUTION_ERROR:'):
+                            failed_cases.append(f'Test {i}: {line}')
+                            break
+                    all_passed = False
+                    break
+                else:
+                    failed_cases.append(f'Test {i}: Unknown error in output. Result: {result}')
+                    all_passed = False
+                    break
+            else:
+                failed_cases.append(f'Test {i}: Sandbox execution failed - Result: {result}')
+                all_passed = False
+                break
+
+        detailed_results = {'total_tests': len(inputs), 'passed_tests': passed_count, 'failed_cases': failed_cases}
+
+        return all_passed, detailed_results
+
+    except Exception as e:
+        if debug:
+            logger.error(f'Call-based evaluation error: {str(e)}')
+        return False, {'error': str(e), 'total_tests': len(inputs), 'passed_tests': 0}
+
+
+def _evaluate_stdio_in_sandbox(
+    adapter: 'SandboxMixin', code: str, inputs: list, outputs: list, timeout: int, debug: bool
+) -> Tuple[bool, Dict]:
+    """Evaluate stdio-based problems in sandbox."""
+    try:
+        all_passed = True
+        passed_count = 0
+        failed_cases = []
+
+        for i, (test_input, expected_output) in enumerate(zip(inputs, outputs)):
+            test_code = f"""
+import sys
+from io import StringIO
+
+# Redirect stdin
+sys.stdin = StringIO('''{test_input}''')
+
+# User's code
+{code}
+"""
+
+            # Execute in sandbox
+            result = adapter.execute_code_in_sandbox(code=test_code, timeout=timeout, language='python')
+
+            if result.get('status') != 'success':
+                if debug:
+                    logger.error(f'Test case {i} execution failed: {result}')
+                failed_cases.append(f'Test {i}: Execution error - Result: {result}')
+                all_passed = False
+                break
+
+            # Compare output
+            actual_output = result.get('output', '').strip()
+            expected_output = expected_output.strip()
+
+            if actual_output == expected_output:
+                passed_count += 1
+            else:
+                if debug:
+                    logger.info(f"Test case {i} failed: expected '{expected_output}', got '{actual_output}'")
+                failed_cases.append(f"Test {i}: Expected '{expected_output}', got '{actual_output}'")
+                all_passed = False
+                break
+
+        detailed_results = {'total_tests': len(inputs), 'passed_tests': passed_count, 'failed_cases': failed_cases}
+
+        return all_passed, detailed_results
+
+    except Exception as e:
+        if debug:
+            logger.error(f'Stdio evaluation error: {str(e)}')
+        return False, {'error': str(e), 'total_tests': len(inputs), 'passed_tests': 0}