nvidia-livecodebench 25.8__py3-none-any.whl

livecodebench/prompts/test_output_prediction.py

@@ -0,0 +1,327 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+from anthropic import HUMAN_PROMPT, AI_PROMPT
+
+from livecodebench.lm_styles import LMStyle
+from livecodebench.benchmarks import TestOutputPredictionProblem
+
+
+class PromptConstants:
+    SYSTEM_MESSAGE_CHAT_GENERIC = f"You are a helpful programming assistant and an expert Python programmer.\
+ You are helping a user to write a test case to help to check the correctness of the function.\
+ The user has written a input for the testcase.\
+ You will calculate the output of the testcase and\
+ write the whole assertion statement in the markdown code block with the correct output."
+
+    SYSTEM_MESSAGE_COMPLETION_GENERIC = f"You are a helpful programming assistant and an expert Python programmer.\
+ You are helping a user to write a test case to help to check the correctness of the function."
+
+    SYSTEM_MESSAGE_INST_CLLAMA = f"You are a helpful programming assistant and an expert Python programmer.\
+ You are helping a user to write a test case to help to check the correctness of the function.\
+ The user has written a input for the testcase.\
+ You will calculate the output of the testcase and \
+ write out the complete assertion statement between [PYTHON] and [/PYTHON] tags."
+
+    SYSTEM_MESSAGE_WIZARD = "Below is an instruction that describes a task. Write a response that appropriately completes the request."
+
+    SYSTEM_MESSAGE_PHIND = f"""You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program. You must put the entired fixed program within code delimiters only for once., for example: 
+```python 
+# YOUR CODE HERE
+```"""
+
+    FORMATTING_MESSAGE = "You will use the following starter code to write the solution to the problem and enclose your code within delimiters."
+
+    FORMATTING_WITHOUT_STARTER_MESSAGE = "Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows."
+
+
+def truncate_io(io):
+    if len(str(io)) > 1000:
+        io = str(io)[:1000] + "...."
+        print(io)
+    return io
+
+
+def format_testcase_func_name_input(function_name, testcase):
+    """
+    use the form of "assert func_name(input) == "
+    """
+    # TODO should there be a space after the == ?
+    input_str = ", ".join(testcase.split("\n"))
+    return f"assert {function_name}({input_str}) == # TODO"
+
+
+def parse_function_name_from_starter_code(starter_code):
+    """
+    starter_code : str
+    """
+    import ast
+
+    tree = ast.parse(starter_code)
+    fn = None
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef):
+            assert fn is None
+            fn = node.name
+    return fn
+
+
+def get_generic_question_template_test_completion(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    prompt = f"Problem:\n{question.question_content}"
+    prompt += f"Function:\n```\n{question.starter_code}\n```\n"
+
+    # parse function name from starter_code
+    func_name = parse_function_name_from_starter_code(question.starter_code)
+    prompt += "Please complete the following test case:\n\n"
+    prompt += (
+        f"```\n{format_testcase_func_name_input(func_name, testcase_input)}\n```\n"
+    )
+
+    return prompt
+
+
+def get_cllama_question_template_answer(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    prompt = f"### Question\n"
+    prompt += get_generic_question_template_test_completion(question, testcase_input)
+    prompt += f"### Answer\n"
+    return prompt
+
+
+def get_deepseekcode_question_template_answer(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    prompt = f"### Instruction: {PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n\n"
+    prompt += get_generic_question_template_test_completion(question, testcase_input)
+    prompt += f"### Response:\n\n"
+    return prompt
+
+
+def get_magicoder_question_template_answer(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    # prompt = f"You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.\n\n"
+    prompt = f"Question:\n"
+    prompt += get_generic_question_template_test_completion(question, testcase_input)
+    prompt += f"@@ Response \n"
+    return prompt
+
+
+def get_mixtral_question_template_answer(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    prompt = get_generic_question_template_test_completion(question, testcase_input)
+    return prompt
+
+
+def get_wizard_question_template_answer(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    prompt = f"""### Instruction: {PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n"""
+    prompt += get_generic_question_template_test_completion(question, testcase_input)
+    prompt += f"### Response:\n"
+    return prompt
+
+
+def get_phind_question_template_answer(
+    question: TestOutputPredictionProblem, testcase_input: str
+):
+    prompt = get_generic_question_template_test_completion(question, testcase_input)
+    prompt += f"\n\n### Assistant"
+    return prompt
+
+def get_qwen_question_template_answer(question: TestOutputPredictionProblem, testcase_input: str):
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        "abacusai/Dracarys-72B-Instruct", padding_side="left", use_fast=False
+    )
+
+    prompt = f"""### Instruction: {PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n"""
+    prompt += get_generic_question_template_test_completion(question, testcase_input)
+    prompt += f"### Response:\n"
+
+    messages = [
+        {"role": "user", "content": prompt},
+    ]
+
+    prompt = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+        truncation=False,
+        padding=False,
+    )
+    return prompt
+
+def format_prompt_test_output(
+    question: TestOutputPredictionProblem, LanguageModelStyle: LMStyle
+) -> str:
+    testcase_input = question.test[0].input
+    if LanguageModelStyle in (LMStyle.OpenAIChat, LMStyle.GenericOAIServer):
+        chat_messages = [
+            {
+                "role": "system",
+                "content": PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC,
+            },
+        ]
+        chat_messages += [
+            {
+                "role": "user",
+                "content": get_generic_question_template_test_completion(
+                    question, testcase_input
+                ),
+            },
+        ]
+        return chat_messages
+    if LanguageModelStyle == LMStyle.LLaMa3:
+        chat_messages = [
+            {
+                "role": "system",
+                "content": PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC,
+            },
+        ]
+        chat_messages += [
+            {
+                "role": "user",
+                "content": get_generic_question_template_test_completion(
+                    question, testcase_input
+                ),
+            },
+        ]
+        from transformers import AutoTokenizer
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            "meta-llama/Meta-Llama-3-8B-Instruct", padding_side="left", use_fast=False
+        )
+        return tokenizer.apply_chat_template(
+            chat_messages,
+            tokenize=False,
+            add_generation_prompt=True,
+            truncation=False,
+            padding=False,
+        )
+    elif LanguageModelStyle == LMStyle.Claude:
+        prompt = f"{HUMAN_PROMPT}\n{PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n\n"
+        prompt += f"{get_generic_question_template_test_completion(question, testcase_input).rstrip()}\n{AI_PROMPT}"
+        return prompt
+    elif LanguageModelStyle == LMStyle.Claude3:
+        system = PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC
+        prompt = [
+            {
+                "role": "user",
+                "content": get_generic_question_template_test_completion(
+                    question, testcase_input
+                ).rstrip(),
+            }
+        ]
+        return system, prompt
+    elif LanguageModelStyle == LMStyle.Gemini:
+        prompt = f"{PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n"
+        prompt += (
+            f"{get_generic_question_template_test_completion(question, testcase_input)}"
+        )
+        return prompt
+
+    elif LanguageModelStyle == LMStyle.StarCoderInstruct:
+        prompt = f"{PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n"
+        prompt += (
+            f"{get_generic_question_template_test_completion(question, testcase_input)}"
+        )
+        return prompt
+
+    elif LanguageModelStyle == LMStyle.DeepSeekCodeInstruct:
+        prompt = (
+            f"{get_deepseekcode_question_template_answer(question, testcase_input)}"
+        )
+        return prompt
+    elif LanguageModelStyle == LMStyle.CodeLLaMaInstruct:
+        prompt = f"[INST] <<SYS>>\n{PromptConstants.SYSTEM_MESSAGE_INST_CLLAMA}\n<</SYS>>\n\n"
+        prompt += (
+            f"{get_cllama_question_template_answer(question, testcase_input)}\n[/INST]"
+        )
+        return prompt
+    elif LanguageModelStyle == LMStyle.MagiCoder:
+        prompt = f"{PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n"
+        prompt += f"{get_magicoder_question_template_answer(question, testcase_input)}"
+        return prompt
+    elif LanguageModelStyle == LMStyle.WizardCoder:
+        prompt = f"{PromptConstants.SYSTEM_MESSAGE_WIZARD}\n\n{get_wizard_question_template_answer(question, testcase_input)}"
+        return prompt
+    elif LanguageModelStyle == LMStyle.Phind:
+        prompt = f"### System Prompt\n\n{PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n\n### User Message\n\n{get_phind_question_template_answer(question, testcase_input)}"
+        return prompt
+    elif LanguageModelStyle == LMStyle.OC:
+        prompt = f"{PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC}\n"
+        prompt += (
+            f"{get_generic_question_template_test_completion(question, testcase_input)}"
+        )
+        return prompt
+    elif LanguageModelStyle == LMStyle.MistralWeb:
+        chat_messages = [
+            {
+                "role": "system",
+                "content": PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC,
+            },
+            {
+                "role": "user",
+                "content": get_generic_question_template_test_completion(
+                    question, testcase_input
+                ),
+            },
+        ]
+        return chat_messages
+    elif (
+        LanguageModelStyle == LMStyle.DracarysQwen
+    ):
+        prompt = f"{get_qwen_question_template_answer(question, testcase_input)}"
+        return prompt
+    elif LanguageModelStyle == LMStyle.DracarysLlama:
+        chat_messages = [
+            {
+                "role": "system",
+                "content": PromptConstants.SYSTEM_MESSAGE_CHAT_GENERIC,
+            },
+        ]
+        chat_messages += [
+            {
+                "role": "user",
+                "content": get_generic_question_template_test_completion(
+                    question, testcase_input
+                ),
+            },
+        ]
+        from transformers import AutoTokenizer
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            "abacusai/Dracarys-Llama-3.1-70B-Instruct", padding_side="right", use_fast=False
+        )
+        return tokenizer.apply_chat_template(
+            chat_messages,
+            tokenize=False,
+            add_generation_prompt=True,
+            truncation=False,
+            padding=False,
+        )
+    else:
+        raise NotImplementedError(
+            f"LanguageModelStyle {LanguageModelStyle} not implemented"
+        )

livecodebench/runner/base_runner.py

@@ -0,0 +1,188 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+import os
+import json
+from abc import ABC
+
+from tqdm import tqdm
+
+from livecodebench.lm_styles import LanguageModel
+from livecodebench.utils.path_utils import get_cache_path
+from livecodebench.utils.multiprocess import run_tasks_in_parallel
+from livecodebench.runner.scenario_router import Scenario
+
+
+class BaseRunner(ABC):
+    def __init__(self, args, model: LanguageModel):
+        self.args = args
+        self.model = model
+        self.client_kwargs: dict[str | str] = {}
+
+        if self.args.use_cache:
+            self.cache_path = get_cache_path(model.model_repr, args)
+            if os.path.exists(self.cache_path):
+                with open(self.cache_path) as f:
+                    self.cache: dict = json.load(f)
+            else:
+                self.cache = {}
+        else:
+            self.cache_path = None
+            self.cache = None
+
+    def save_cache(self):
+        if self.args.use_cache:
+            with open(self.cache_path, "w") as f:
+                json.dump(self.cache, f, indent=4)
+
+    # @abstractmethod
+    def _run_single(self, prompt: str | list[dict[str, str]]) -> list[str]:
+        pass
+
+    @staticmethod
+    def run_single(combined_args) -> list[str]:
+        """
+        Run the model for a single prompt and return the output
+        Static method to be used in multiprocessing
+        Calls the _run_single method with the combined arguments
+        """
+        prompt: str | list[dict[str, str]]
+        cache: dict[str, str]
+        call_method: callable
+        prompt, cache, args, call_method, sample_idx = combined_args
+        
+        if cache is not None and sample_idx in cache:
+            if len(cache[sample_idx]) == args.n:
+                return cache[sample_idx]
+
+        result = call_method(prompt, idx=sample_idx)
+        assert len(result) == args.n
+
+        return result
+
+    def run_batch(self, prompts: list[str | list[dict[str, str]]], indices: list[int], use_progress_bar: bool = False) -> list[list[str]]:
+        outputs = []
+        arguments = [
+            (
+                prompt,
+                self.cache,  ## pass the cache as argument for cache check
+                self.args,  ## pass the args as argument for cache check
+                self._run_single,  ## pass the _run_single method as argument because of multiprocessing
+                idx,  ## pass the sample index
+            )
+            for prompt, idx in zip(prompts, indices)
+        ]
+        if self.args.multiprocess > 1:
+            parallel_outputs = run_tasks_in_parallel(
+                self.run_single,
+                arguments,
+                self.args.multiprocess,
+                use_progress_bar=use_progress_bar,
+            )
+            for output in parallel_outputs:
+                if output.is_success():
+                    outputs.append(output.result)
+                else:
+                    print("Failed to run the model for some prompts")
+                    print(output.status)
+                    print(output.exception_tb)
+                    raise RuntimeError(f"Model run failed with status='{output.status}'.")
+        else:
+            outputs = [self.run_single(argument) for argument in tqdm(arguments)]
+
+        if self.args.use_cache:
+            for output, idx in zip(outputs, indices):
+                self.cache[idx] = output  ## save the output to cache using sample_idx as key
+
+        return outputs
+
+    def prompts_to_outputs(
+        self, prompts: list[str | list[dict[str, str]]]
+    ) -> list[list[str]]:
+        if self.args.use_cache:
+            outputs = []
+            batch_size = self.args.cache_batch_size
+            for i in tqdm(range(0, len(prompts), batch_size)):
+                batch = prompts[i : i + batch_size]
+                batch_indices = list(range(i, i + len(batch)))
+                batch_outputs = self.run_batch(batch, batch_indices, use_progress_bar=False)
+                outputs.extend(batch_outputs)
+                self.save_cache()
+        else:
+            indices = list(range(len(prompts)))
+            outputs = self.run_batch(prompts, indices, use_progress_bar=True)
+        return outputs
+
+    def run_main_repair(self, benchmark: list, format_prompt: callable) -> list[list[str]]:
+        assert self.args.n == 1
+        with open(
+            f"output/{self.model.model_repr}/{Scenario.codegeneration}_{self.args.codegen_n}_{self.args.temperature}_eval_all.json"
+        ) as f:
+            check_metadata_list = json.load(f)
+
+        outputs = [
+            [None for _ in range(self.args.codegen_n)]
+            for _ in range(len(benchmark))
+        ]
+        prompts = []
+        prompt_index_to_question_idx = {}
+        prompt_index_to_code_idx = {}
+        count = 0
+
+        for problem_idx, problem in enumerate(benchmark):
+            for check_metadata_idx, check_metadata in enumerate(check_metadata_list):
+                if problem.question_id == check_metadata['question_id']:
+                    count += 1 
+                    question_content = check_metadata["question_content"]
+                    code_list = check_metadata["code_list"]
+                    output_list = check_metadata["output_list"]
+                    graded_list = check_metadata["graded_list"]
+                    metadata = check_metadata["metadata"]
+                    for code_idx in range(len(code_list)):
+                        prompt = format_prompt(
+                            question_content,
+                            self.model.model_style,
+                            code_list[code_idx],
+                            graded_list[code_idx],
+                            metadata[code_idx],
+                        )
+                        if prompt == "":
+                            outputs[problem_idx][code_idx] = output_list[code_idx]
+                            continue
+                        prompts.append(prompt)
+                        prompt_index_to_question_idx[len(prompts) - 1] = problem_idx
+                        prompt_index_to_code_idx[len(prompts) - 1] = code_idx
+        assert len(benchmark)==count, f"{len(benchmark)=}!={count=}"
+
+        prompt_outputs = self.prompts_to_outputs(prompts)
+        for prompt_idx, output in enumerate(prompt_outputs):
+            question_idx = prompt_index_to_question_idx[prompt_idx]
+            code_idx = prompt_index_to_code_idx[prompt_idx]
+            outputs[question_idx][code_idx] = output
+
+        return outputs
+
+    def run_main(self, benchmark: list, format_prompt: callable) -> list[list[str]]:
+        if self.args.scenario == Scenario.selfrepair:
+            return self.run_main_repair(benchmark, format_prompt)
+
+        prompts = [
+            format_prompt(problem, self.model.model_style) for problem in benchmark
+        ]
+        outputs = self.prompts_to_outputs(prompts)
+        return outputs

livecodebench/runner/claude3_runner.py

@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+import os
+from time import sleep
+
+try:
+    from anthropic import Anthropic
+except ImportError as e:
+    pass
+
+from livecodebench.runner.base_runner import BaseRunner
+
+
+class Claude3Runner(BaseRunner):
+    client = Anthropic(api_key=os.getenv("ANTHROPIC_KEY"))
+
+    def __init__(self, args, model):
+        super().__init__(args, model)
+        self.client_kwargs: dict[str | str] = {
+            "model": args.model,
+            "temperature": args.temperature,
+            "max_tokens": args.max_tokens,
+            "top_p": args.top_p,
+        }
+
+    def _run_single(self, prompt: tuple[str, str]) -> list[str]:
+
+        def __run_single(counter):
+            try:
+                response = self.client.messages.create(
+                    system=prompt[0],
+                    messages=prompt[1],
+                    **self.client_kwargs,
+                )
+                content = "\n".join([x.text for x in response.content])
+                return content
+            except Exception as e:
+                print("Exception: ", repr(e), "Sleeping for 20 seconds...")
+                sleep(20 * (11 - counter))
+                counter = counter - 1
+                if counter == 0:
+                    print(f"Failed to run model for {prompt}!")
+                    print("Exception: ", repr(e))
+                    raise e
+                return __run_single(counter)
+
+        outputs = []
+        try:
+            for _ in range(self.args.n):
+                outputs.append(__run_single(10))
+        except Exception as e:
+            raise e
+
+        return outputs

livecodebench/runner/claude_runner.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+import os
+from time import sleep
+
+try:
+    from anthropic import Anthropic
+except ImportError as e:
+    pass
+
+from livecodebench.runner.base_runner import BaseRunner
+
+
+class ClaudeRunner(BaseRunner):
+    client = Anthropic(api_key=os.getenv("ANTHROPIC_KEY"))
+
+    def __init__(self, args, model):
+        super().__init__(args, model)
+        self.client_kwargs: dict[str | str] = {
+            "model": args.model,
+            "temperature": args.temperature,
+            "max_tokens_to_sample": args.max_tokens,
+            "top_p": args.top_p,
+        }
+
+    def _run_single(self, prompt: str) -> list[str]:
+
+        def __run_single(counter):
+            try:
+                response = self.client.completions.create(
+                    prompt=prompt,
+                    **self.client_kwargs,
+                )                                
+                content = response.completion
+                return content
+            except Exception as e:
+                print("Exception: ", repr(e), "Sleeping for 20 seconds...")
+                sleep(20 * (11 - counter))
+                counter = counter - 1
+                if counter == 0:
+                    print(f"Failed to run model for {prompt}!")
+                    print("Exception: ", repr(e))
+                    raise e
+                return __run_single(counter)
+
+        outputs = []
+        try:
+            for _ in range(self.args.n):
+                outputs.append(__run_single(10))
+        except Exception as e:
+            raise e
+
+        return outputs