nvidia-scicode 25.8__py3-none-any.whl

core_evals/scicode/__init__.py

@@ -0,0 +1 @@
+

core_evals/scicode/framework.yml

@@ -0,0 +1,73 @@
+framework:
+  name: scicode
+  pkg_name: scicode
+  full_name: SciCode Benchmark
+  description: SciCode is a challenging benchmark designed to evaluate the capabilities of LLMs in generating code for solving realistic scientific research problems.
+  url: https://github.com/scicode-bench/SciCode
+  source: https://gitlab-master.nvidia.com/dl/JoC/competitive_evaluation/core_evals_frameworks/SciCode
+
+defaults:
+  command: >-
+    {% if target.api_endpoint.api_key is not none %}API_KEY=${{target.api_endpoint.api_key}}{% endif %}
+    scicode_eval --model {{target.api_endpoint.model_id}} --url {{target.api_endpoint.url}}
+    --output-dir {{config.output_dir}}
+    --log-dir {{config.output_dir}}/logs
+    {% if config.params.temperature is not none %}--temperature={{config.params.temperature}}{% endif %}
+    {% if config.params.limit_samples is not none %}--limit-samples={{config.params.limit_samples}}{% endif %}
+    --n-samples={{config.params.extra.n_samples}}
+    --extra-params top_p={{config.params.top_p}},timeout={{config.params.request_timeout}},max_tokens={{config.params.max_new_tokens}},max_retries={{config.params.max_retries}}
+    {% if config.params.extra.with_background %}--with-background {% endif %}
+    {% if config.params.extra.include_dev %}--include-dev{% endif %}
+    {% if config.params.extra.eval_threads is not none %}--eval-threads={{config.params.extra.eval_threads}}{% endif %}
+  config:
+    supported_endpoint_types:
+    - chat
+    params:
+      limit_samples: null
+      temperature: 0
+      max_new_tokens: 2048
+      top_p: 0.00001
+      request_timeout: 60
+      max_retries: 2
+      extra:
+        with_background: false
+        include_dev: false
+        n_samples: 1
+        eval_threads: null
+  target:
+    api_endpoint: # required to add: url, model_id, type
+      stream: false
+
+evaluations:
+- name: SciCode
+  description: >-
+    - SciCode is a challenging benchmark designed to evaluate the capabilities of LLMs in generating code for solving realistic scientific research problems.
+    - This variant does not include scientist-annotated background in the prompts.
+  defaults:
+    config:
+      type: "scicode"
+- name: SciCode-Background
+  description: >-
+    - SciCode is a challenging benchmark designed to evaluate the capabilities of LLMs in generating code for solving realistic scientific research problems.
+    - This variant includes scientist-annotated background in the prompts.
+  defaults:
+    config:
+      type: "scicode_background"
+      params:
+        extra:
+          with_background: true
+- name: AA-SciCode
+  description: >-
+    - SciCode is a challenging benchmark designed to evaluate the capabilities of LLMs in generating code for solving realistic scientific research problems.
+    - This variant mimicks setup used by Artificial Analysis in their Intelligence Benchmark (v2).
+    - It includes scientist-annotated background in the prompts and uses all available problems for evaluation (including "dev" set).
+  defaults:
+    config:
+      type: "aa_scicode"
+      params:
+        temperature: 0.0
+        max_new_tokens: 4096
+        extra:
+          with_background: true
+          include_dev: true
+          n_samples: 3

core_evals/scicode/framework_entrypoint.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from nvidia_eval_commons.api.run import run_eval
+
+
+def main():
+    warnings.warn(
+        "This command is deprecated and will be removed in the next release. Please use the `eval-factory` command instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    # Add this `framework_entrypoint:main` as an entrypoint to `pyproject.toml`, example
+    run_eval()

core_evals/scicode/output.py

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import pathlib
+
+from nvidia_eval_commons.api.api_dataclasses import EvaluationResult
+
+
+def parse_output(output_dir: str) -> EvaluationResult:
+    output_path = list(pathlib.Path(output_dir).glob('*.json'))
+    assert len(output_path) == 1, f"Expected 1 output file, found {len(output_path)} ({output_path})"
+    output_path = output_path[0]
+    with open(output_path) as f:
+        results = json.load(f)
+    metrics = {}
+    for metric_type in ("steps", "problems"):
+        metric_name = metric_type + "_pass@1"
+        metrics[metric_name] = dict(
+            scores={
+                metric_name: dict(
+                    value=results[metric_name],
+                    stats={}
+                )
+            }
+        )
+
+    task_results = {"metrics": metrics}
+    return EvaluationResult(tasks={'scicode': task_results}, groups={'scicode': task_results})