nvidia-livecodebench 25.8__py3-none-any.whl

core_evals/livecodebench/__init__.py

@@ -0,0 +1 @@
+

core_evals/livecodebench/framework.yml

@@ -0,0 +1,233 @@
+framework:
+  name: livecodebench
+  pkg_name: livecodebench
+  full_name: LiveCodeBench
+  description: Holistic and Contamination Free Evaluation of Large Language Models for Code. Paper https://arxiv.org/pdf/2403.07974
+  url: https://github.com/LiveCodeBench/LiveCodeBench
+  source: https://gitlab-master.nvidia.com/dl/JoC/competitive_evaluation/LiveCodeBench
+defaults:
+  command: >
+    {% if target.api_endpoint.api_key is not none %}
+      export API_KEY=${{target.api_endpoint.api_key}} && 
+    {% endif %}
+    livecodebench --model {{target.api_endpoint.model_id}} \
+                --scenario {{config.params.task}} \
+                --release_version {{config.params.extra.release_version}} \
+                --url {{target.api_endpoint.url}} \
+                --temperature {{config.params.temperature}} \
+                --top_p {{config.params.top_p}} \
+                --evaluate \
+                --codegen_n {{config.params.extra.n_samples}} \
+                --use_cache \
+                --cache_batch_size {{config.params.extra.cache_batch_size}} \
+                --num_process_evaluate {{config.params.extra.num_process_evaluate}} \
+                --n {{config.params.extra.n_samples}} \
+                --max_tokens {{config.params.max_new_tokens}} \
+                --out_dir {{config.output_dir}} \
+                --multiprocess {{config.params.parallelism}} \
+                --max_retries {{config.params.max_retries}} \
+                --timeout {{config.params.request_timeout}}{% if config.params.extra.start_date is not none %} --start_date {{config.params.extra.start_date}} {% endif %} {% if config.params.extra.end_date is not none %} --end_date {{config.params.extra.end_date}} {% endif %} {% if config.params.extra.support_system_role %} --support_system_role {% endif %} {% if config.params.limit_samples is not none %} --first_n {{config.params.limit_samples}}{% endif %}{% if config.params.extra.cot_code_execution == true %} --cot_code_execution {% endif %}{% if config.params.extra.args is defined %} {{ config.params.extra.args }} {% endif %}
+  
+
+  
+  config:
+    params:
+      limit_samples: null
+      max_new_tokens: 4096
+      temperature: 0.0
+      top_p: 0.00001
+      parallelism: 10
+      max_retries: 5
+      request_timeout: 60
+      extra:
+        n_samples: 10
+        num_process_evaluate: 5
+        cache_batch_size: 10
+        support_system_role: false
+        start_date: null # format: YYYY-MM-DD
+        end_date : null # format: YYYY-MM-DD
+        cot_code_execution: false
+
+  target:
+    api_endpoint: {} # required to add: url, model_id, api_key
+evaluations:
+  - name: codegeneration_release_latest
+    description: Code generation latest version
+    defaults:
+      config:
+        type: codegeneration_release_latest
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_latest
+  - name: codegeneration_release_v1
+    description: The initial release of the dataset with problems released between May 2023 and Mar 2024 containing 400 problems.
+    defaults:
+      config:
+        type: codegeneration_release_v1
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v1
+
+  - name: codegeneration_release_v2
+    description: The updated release of the dataset with problems released between May 2023 and May 2024 containing 511 problems.
+    defaults:
+      config:
+        type: codegeneration_release_v2
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v2
+
+  - name: codegeneration_release_v3
+    description: The updated release of the dataset with problems released between May 2023 and Jul 2024 containing 612 problems.
+    defaults:
+      config:
+        type: codegeneration_release_v3
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v3
+
+  - name: codegeneration_release_v4
+    description: The updated release of the dataset with problems released between May 2023 and Sep 2024 containing 713 problems.
+    defaults:
+      config:
+        type: codegeneration_release_v4
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v4
+
+  - name: codegeneration_release_v5
+    description: The updated release of the dataset with problems released between May 2023 and Jan 2025 containing 880 problems.
+    defaults:
+      config:
+        type: codegeneration_release_v5
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v5
+  - name: codegeneration_release_v6
+    description: The updated release of the dataset with problems released between May 2023 and Apr 2025 containing 1055 problems.
+    defaults:
+      config:
+        type: codegeneration_release_v6
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v6
+  - name: codegeneration_notfast
+    description: Not fast version of code generation (v2).
+    defaults:
+      config:
+        type: codegeneration_notfast
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            args: "--not_fast"
+
+  - name: testoutputprediction
+    description: Solve the natural language task on a specified input, evaluating the ability to generate testing outputs. The model is given the natural language problem description and an input, and the output should be the output for the problem.
+    defaults:
+      config:
+        type: testoutputprediction
+        supported_endpoint_types:
+          - chat
+        params:
+          task: testoutputprediction
+          extra:
+            release_version: release_latest
+
+  - name: codeexecution_v2
+    description: “Execute” a program on an input, evaluating code comprehension ability. The model is given a program and an input, and the output should be the result.
+    defaults:
+      config:
+        type: codeexecution_v2
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codeexecution
+          extra:
+            release_version: release_v2
+  - name: codeexecution_v2_cot
+    description: “CoT. Execute” a program on an input, evaluating code comprehension ability. The model is given a program and an input, and the output should be the result.
+    defaults:
+      config:
+        type: codeexecution_v2_cot
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codeexecution
+          extra:
+            release_version: release_v2
+            cot_code_execution: true
+  # NOTE(dfridman): same as `livecodebench_0724_0125`. Leaving it as legacy.
+  - name: AA_code_generation
+    description: "AA code generation evaluating code comprehension ability. The model is given a program and an input, and the output should be the result."
+    defaults:
+      config:
+        type: AA_codegeneration
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          extra:
+            release_version: release_v5
+            n_samples: 3
+            start_date: 2024-07-01
+            end_date: 2025-01-01
+
+  - name: livecodebench_0724_0125
+    description: >-
+     - Code generation evaluating code comprehension ability. The model is given a program and an input, and the output should be the result.
+     - The data period and sampling parameters used by Artificial Analaysis (https://artificialanalysis.ai/methodology/intelligence-benchmarking)
+    defaults:
+      config:
+        type: livecodebench_0724_0125
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          temperature: 0.0
+          max_new_tokens: 4096
+          extra:
+            release_version: release_v5
+            n_samples: 3
+            start_date: 2024-07-01
+            end_date: 2025-01-01
+  - name: livecodebench_0824_0225
+    description:
+      - Code generation evaluating code comprehension ability. The model is given a program and an input, and the output should be the result.
+      - The data period and sampling parameters used by NeMo Alignment team.
+    defaults:
+      config:
+        type: livecodebench_0824_0225
+        supported_endpoint_types:
+          - chat
+        params:
+          task: codegeneration
+          temperature: 0.0
+          max_new_tokens: 4096
+          extra:
+            release_version: release_v5
+            n_samples: 3
+            start_date: 2024-08-01
+            end_date: 2025-02-01

core_evals/livecodebench/framework_entrypoint.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from nvidia_eval_commons.api.run import run_eval
+
+
+def main():
+    warnings.warn(
+        "This command is deprecated and will be removed in the next release. Please use the `eval-factory` command instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    # Add this `framework_entrypoint:main` as an entrypoint to `pyproject.toml`, example
+    run_eval()

core_evals/livecodebench/output.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import pathlib
+import re
+
+from nvidia_eval_commons.api.api_dataclasses import EvaluationResult
+
+# This is the only required function
+def parse_output(output_dir: str) -> EvaluationResult:
+    result_files = list(pathlib.Path(output_dir).rglob("*eval.json"))
+    if not result_files:
+        raise FileNotFoundError("Failed to find `*eval.json` with metric.")
+    if len(result_files) > 1:
+        raise ValueError(
+            "More than 1 `*eval.json.json` files found. `output_dir` must contain a single evaluation."
+        )
+    match = re.search(r"Scenario\.([^_]+)_", str(result_files[0]))
+    task_name = match.group(1)
+
+    with open(result_files[0]) as fp:
+        results = json.load(fp)[0]
+    if 'detail' in results:
+        results.pop('detail')
+
+
+    tasks = {}
+    metrics = {}
+    for metric_name, value in results.items():
+        if metric_name.endswith("_stderr"):
+            continue
+        metrics[metric_name] = dict(
+                scores={metric_name: dict(value=value, stats={"stderr": results[f"{metric_name}_stderr"]})}
+            )
+    tasks[task_name] = dict(
+        metrics=metrics
+    )
+    return EvaluationResult(groups=tasks, tasks=tasks)

livecodebench/benchmarks/__init__.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+from livecodebench.benchmarks.code_generation import (
+    CodeGenerationProblem,
+    load_code_generation_dataset,
+    load_code_generation_dataset_not_fast,
+)
+from livecodebench.benchmarks.test_output_prediction import (
+    TestOutputPredictionProblem,
+    load_test_prediction_dataset,
+)
+from livecodebench.benchmarks.code_execution import (
+    CodeExecutionProblem,
+    load_code_execution_dataset,
+)

livecodebench/benchmarks/code_execution.py

@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+from datetime import datetime
+from dataclasses import dataclass
+
+from datasets import load_dataset
+from livecodebench.benchmarks.utils import filter_dataset_by_date
+
+
+@dataclass
+class CodeExecutionProblem:
+    question_id: str
+    contest_id: str
+    contest_date: datetime
+    difficulty: str
+    function_name: str
+    code: str
+    input: str
+    output: str
+    id: str
+    problem_id: str
+    numsteps: int
+
+    def __post_init__(self):
+        pass
+
+    def insert_output(self, output_list: list[str], pred_list: list[str]) -> dict:
+        return {
+            "question_id": self.question_id,
+            "contest_id": self.contest_id,
+            "contest_date": self.contest_date.isoformat(),
+            "difficulty": self.difficulty,
+            "function_name": self.function_name,
+            "code": self.code,
+            "input": self.input,
+            "output": self.output,
+            "id": self.id,
+            "problem_id": self.problem_id,
+            "numsteps": self.numsteps,
+            "output_list": output_list,
+            "pred_list": pred_list,
+        }
+
+    def insert_output_evaluation(
+        self, output_list: list[str], code_list: list[str], graded_list: list[bool]
+    ) -> dict:
+        output = self.insert_output(output_list, code_list)
+        output["graded_list"] = graded_list
+        output["pass@1"] = graded_list.count(True) / len(graded_list)
+        return output
+
+    def get_evaluation_sample(self) -> dict:
+        return {
+            "code": self.code,
+            "input": self.input,
+            "output": self.output,
+        }
+
+
+def load_code_execution_dataset(release_version="release_v1", start_date: str = None, end_date: str = None) -> list:
+    dataset = load_dataset("livecodebench/execution-v2", split="test")
+    dataset = filter_dataset_by_date(dataset, start_date=start_date, end_date=end_date)
+    dataset = [CodeExecutionProblem(**p) for p in dataset]  # type: ignore
+    print(f"Loaded {len(dataset)} problems")
+    return dataset
+
+
+if __name__ == "__main__":
+    dataset = load_code_execution_dataset()

livecodebench/benchmarks/code_generation.py

@@ -0,0 +1,160 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+import json
+import zlib
+import pickle
+import base64
+from enum import Enum
+from datetime import datetime
+from dataclasses import dataclass
+
+from datasets import load_dataset
+from livecodebench.benchmarks.utils import filter_dataset_by_date
+
+
+class Platform(Enum):
+    LEETCODE = "leetcode"
+    CODEFORCES = "codeforces"
+    ATCODER = "atcoder"
+
+
+class Difficulty(Enum):
+    EASY = "easy"
+    MEDIUM = "medium"
+    HARD = "hard"
+
+
+class TestType(Enum):
+    STDIN = "stdin"
+    FUNCTIONAL = "functional"
+
+
+@dataclass
+class Test:
+    input: str
+    output: str
+    testtype: TestType
+
+    def __post_init__(self):
+        self.testtype = TestType(self.testtype)
+        # if self.testtype == TestType.FUNCTIONAL:
+        #     self.input = json.loads(self.input)
+        #     self.output = json.loads(self.output)
+
+
+@dataclass
+class CodeGenerationProblem:
+    question_title: str
+    question_content: str
+    platform: Platform
+    question_id: str
+    contest_id: str
+    contest_date: datetime
+    starter_code: str
+    difficulty: Difficulty
+    public_test_cases: list[Test]
+    private_test_cases: list[Test]
+    metadata: dict
+
+    def __post_init__(self):
+        self.platform = Platform(self.platform)
+        self.difficulty = Difficulty(self.difficulty)
+        self.contest_date = datetime.fromisoformat(self.contest_date)
+
+        self.public_test_cases = json.loads(self.public_test_cases)  # type: ignore
+        self.public_test_cases = [Test(**t) for t in self.public_test_cases]
+
+        try:
+            self.private_test_cases = json.loads(self.private_test_cases)  # type: ignore
+        except:
+            self.private_test_cases = json.loads(
+                pickle.loads(
+                    zlib.decompress(
+                        base64.b64decode(self.private_test_cases.encode("utf-8"))  # type: ignore
+                    )
+                )
+            )  # type: ignore
+        self.private_test_cases = [Test(**t) for t in self.private_test_cases]
+
+        self.metadata = json.loads(self.metadata)  # type: ignore
+
+    def insert_output(self, output_list: list[str], code_list: list[str]) -> dict:
+        return {
+            "question_title": self.question_title,
+            "question_content": self.question_content,
+            "platform": self.platform.value,
+            "question_id": self.question_id,
+            "contest_id": self.contest_id,
+            "contest_date": self.contest_date.isoformat(),
+            "starter_code": self.starter_code,
+            "difficulty": self.difficulty.value,
+            "output_list": output_list,
+            "code_list": code_list,
+        }
+
+    def insert_output_evaluation(
+        self,
+        output_list: list[str],
+        code_list: list[str],
+        graded_list: list[bool],
+        **kwargs,
+    ) -> dict:
+        output = self.insert_output(output_list, code_list)
+        output["graded_list"] = graded_list
+        output["pass@1"] = graded_list.count(True) / len(graded_list)
+        for k, v in kwargs.items():
+            output[k] = v
+        return output
+
+    def get_evaluation_sample(self):
+        return {
+            "input_output": json.dumps(
+                {
+                    "inputs": [
+                        t.input
+                        for t in self.public_test_cases + self.private_test_cases
+                    ],
+                    "outputs": [
+                        t.output
+                        for t in self.public_test_cases + self.private_test_cases
+                    ],
+                    "fn_name": self.metadata.get("func_name", None),
+                }
+            ),
+        }
+
+
+def load_code_generation_dataset(release_version="release_v1", start_date: str = None, end_date: str = None) -> list:
+    dataset = load_dataset("livecodebench/code_generation_lite", split="test", version_tag=release_version, trust_remote_code=True)
+    dataset = filter_dataset_by_date(dataset, start_date=start_date, end_date=end_date)
+    dataset = [CodeGenerationProblem(**p) for p in dataset]  # type: ignore
+    print(f"Loaded {len(dataset)} problems")
+    return dataset
+
+def load_code_generation_dataset_not_fast(release_version="release_v1", start_date: str = None, end_date: str = None) -> list:
+    dataset = load_dataset("livecodebench/code_generation", split="test")
+    dataset = filter_dataset_by_date(dataset, start_date=start_date, end_date=end_date)
+    dataset = [CodeGenerationProblem(**p) for p in dataset]  # type: ignore
+    print(f"Loaded {len(dataset)} problems")
+    return dataset
+
+
+
+if __name__ == "__main__":
+    dataset = load_code_generation_dataset()

livecodebench/benchmarks/test_output_prediction.py

@@ -0,0 +1,90 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Original Copyright 2025 LiveCodeBench
+# For the original license and copyright information, see the LICENSE file in this repository.
+
+import json
+from datetime import datetime
+from dataclasses import dataclass
+
+from datasets import load_dataset
+
+from livecodebench.benchmarks.utils import filter_dataset_by_date
+
+
+@dataclass
+class Test:
+    input: str
+    output: str
+    testtype: str
+
+
+@dataclass
+class TestOutputPredictionProblem:
+    question_title: str
+    question_content: str
+    question_id: str
+    contest_id: str
+    contest_date: datetime
+    difficulty: str
+    test: list[Test]
+    starter_code: str
+    function_name: str
+    test_id: int
+
+    def __post_init__(self):
+        self.test = [Test(**t) for t in json.loads(self.test)]  # type: ignore
+
+    def insert_output(self, output_list: list[str], pred_list: list[str]) -> dict:
+        return {
+            "question_title": self.question_title,
+            "question_content": self.question_content,
+            "question_id": self.question_id,
+            "contest_id": self.contest_id,
+            "contest_date": self.contest_date.isoformat(),
+            "difficulty": self.difficulty,
+            "output_list": output_list,
+            "pred_list": pred_list,
+            "test_id": self.test_id,
+            "function_name": self.function_name,
+            "starter_code": self.starter_code,
+        }
+
+    def insert_output_evaluation(
+        self, output_list: list[str], code_list: list[str], graded_list: list[bool]
+    ) -> dict:
+        output = self.insert_output(output_list, code_list)
+        output["graded_list"] = graded_list
+        output["pass@1"] = graded_list.count(True) / len(graded_list)
+        return output
+
+    def get_evaluation_sample(self) -> dict:
+        return {
+            "input": self.question_content,
+            "output": self.test[0].output,
+        }
+
+
+def load_test_prediction_dataset(release_version="release_v1", start_date: str | None = None, end_date: str | None = None) -> list[TestOutputPredictionProblem]:
+    dataset = load_dataset("livecodebench/test_generation", split="test")  # type: ignore
+    dataset = filter_dataset_by_date(dataset, start_date=start_date, end_date=end_date)
+    dataset = [TestOutputPredictionProblem(**d) for d in dataset]
+    print(f"Loaded {len(dataset)} prediction problems")
+    return dataset
+
+
+if __name__ == "__main__":
+    dataset = load_test_prediction_dataset()