nvidia-profbench 25.11__py3-none-any.whl

core_evals/profbench/framework.yml

@@ -0,0 +1,98 @@
+framework:
+  name: profbench
+  pkg_name: profbench
+  full_name: ProfBench
+  description: Professional domain benchmark for evaluating LLMs on Physics PhD, Chemistry PhD, Finance MBA, and Consulting MBA tasks
+  url: https://github.com/nvidia/profbench
+
+defaults:
+  command: >
+    {% if target.api_endpoint.api_key is not none %}
+      export API_KEY=${{target.api_endpoint.api_key}} && 
+    {% endif %}
+    {% if config.params.extra.run_generation %}
+      python -m profbench.run_report_generation \
+        --model {{target.api_endpoint.model_id}} \
+        --library {{config.params.extra.library}} \
+        --timeout {{config.params.request_timeout}} \
+        --parallel {{config.params.parallelism}} \
+        --retry-attempts {{config.params.max_retries}} \
+        --folder {{config.output_dir}}{% if target.api_endpoint.url is not none %} --base-url {{target.api_endpoint.url}}{% endif %}{% if config.params.extra.version is not none %} --version {{config.params.extra.version}}{% endif %}{% if config.params.extra.web_search %} --web-search{% endif %}{% if config.params.extra.reasoning %} --reasoning{% endif %}{% if config.params.extra.reasoning_effort is not none %} --reasoning-effort {{config.params.extra.reasoning_effort}}{% endif %}{% if config.params.limit_samples is not none %} --limit-samples {{config.params.limit_samples}}{% endif %}{% if config.params.temperature is not none %} --temperature {{config.params.temperature}}{% endif %}{% if config.params.top_p is not none %} --top-p {{config.params.top_p}}{% endif %}{% if config.params.max_new_tokens is not none %} --max-tokens {{config.params.max_new_tokens}}{% endif %} && 
+      GENERATION_OUTPUT=$(ls -t {{config.output_dir}}/*.jsonl | head -1) && 
+    {% endif %}
+    {% if config.params.extra.run_judge_generated %}
+      python -m profbench.run_best_llm_judge_on_generated_reports \
+        --filename $GENERATION_OUTPUT \
+        --api-key $API_KEY \
+        --model {{target.api_endpoint.model_id}} \
+        --library {{config.params.extra.library}} \
+        --timeout {{config.params.request_timeout}} \
+        --parallel {{config.params.parallelism}} \
+        --retry-attempts {{config.params.max_retries}} \
+        --output-folder {{config.output_dir}}/judgements{% if target.api_endpoint.url is not none %} --base-url {{target.api_endpoint.url}}{% endif %}{% if config.params.limit_samples is not none %} --limit-samples {{config.params.limit_samples}}{% endif %}{% if config.params.temperature is not none %} --temperature {{config.params.temperature}}{% endif %}{% if config.params.top_p is not none %} --top-p {{config.params.top_p}}{% endif %}{% if config.params.max_new_tokens is not none %} --max-tokens {{config.params.max_new_tokens}}{% endif %} && 
+      JUDGE_OUTPUT=$(ls -t {{config.output_dir}}/judgements/*.jsonl | head -1) && 
+      python -m profbench.score_report_generation $JUDGE_OUTPUT
+    {% endif %}
+    {% if config.params.extra.run_judge_provided %}
+      python -m profbench.run_llm_judge_on_provided_reports \
+        --model {{target.api_endpoint.model_id}} \
+        --library {{config.params.extra.library}} \
+        --timeout {{config.params.request_timeout}} \
+        --parallel {{config.params.parallelism}} \
+        --retry-attempts {{config.params.max_retries}} \
+        --folder {{config.output_dir}}{% if target.api_endpoint.url is not none %} --base-url {{target.api_endpoint.url}}{% endif %}{% if config.params.extra.reasoning %} --reasoning{% endif %}{% if config.params.extra.reasoning_effort is not none %} --reasoning-effort {{config.params.extra.reasoning_effort}}{% endif %}{% if config.params.extra.debug %} --debug{% endif %}{% if config.params.limit_samples is not none %} --limit-samples {{config.params.limit_samples}}{% endif %}{% if config.params.temperature is not none %} --temperature {{config.params.temperature}}{% endif %}{% if config.params.top_p is not none %} --top-p {{config.params.top_p}}{% endif %}{% if config.params.max_new_tokens is not none %} --max-tokens {{config.params.max_new_tokens}}{% endif %} && 
+      JUDGE_OUTPUT=$(ls -t {{config.output_dir}}/*.jsonl | head -1) && 
+      python -m profbench.score_llm_judge $JUDGE_OUTPUT
+    {% endif %}
+
+  config:
+    params:
+      limit_samples: null
+      max_new_tokens: 4096
+      temperature: 0.0
+      top_p: 0.00001
+      parallelism: 10
+      max_retries: 5
+      request_timeout: 600
+      extra:
+        run_generation: false
+        run_judge_generated: false
+        run_judge_provided: false
+        library: openai
+        version: lite  # debug, lite, or full
+        web_search: false
+        reasoning: false
+        reasoning_effort: null  # low, medium, high, or minimal
+        debug: false
+
+  target:
+    api_endpoint: {}  # required to add: model_id, api_key; optional: url
+
+evaluations:
+  - name: report_generation
+    description: Generate professional reports and evaluate them (full pipeline)
+    defaults:
+      config:
+        type: report_generation
+        supported_endpoint_types:
+          - chat
+        params:
+          extra:
+            run_generation: true
+            run_judge_generated: true
+            run_judge_provided: false
+  
+  - name: llm_judge
+    description: Run LLM judge on provided ProfBench reports and score them
+    defaults:
+      config:
+        type: llm_judge
+        supported_endpoint_types:
+          - chat
+        params:
+          extra:
+            run_generation: false
+            run_judge_generated: false
+            run_judge_provided: true
+
+

core_evals/profbench/output.py

@@ -0,0 +1,77 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import pathlib
+import re
+
+from nemo_evaluator.api.api_dataclasses import EvaluationResult
+
+# This is the only required function
+def parse_output(output_dir: str) -> EvaluationResult:
+    """
+    Parse ProfBench evaluation output.
+    
+    This function looks for *_eval.json files created by the scoring scripts.
+    """
+    result_files = list(pathlib.Path(output_dir).rglob("*_eval.json"))
+    if not result_files:
+        raise FileNotFoundError(f"Failed to find `*_eval.json` with metrics in {output_dir}.")
+    if len(result_files) > 1:
+        raise ValueError(
+            f"More than 1 `*_eval.json` files found in {output_dir}. `output_dir` must contain a single evaluation."
+        )
+    
+    result_file = result_files[0]
+    
+    # Determine task name from filename
+    if "llm_judge_eval" in result_file.name:
+        task_name = "llm_judge"
+    elif "report_generation_eval" in result_file.name:
+        task_name = "report_generation"
+    else:
+        task_name = "profbench"
+    
+    with open(result_file) as fp:
+        results = json.load(fp)
+    
+    # Remove detail if present
+    if 'detail' in results:
+        results.pop('detail')
+    
+    tasks = {}
+    metrics = {}
+    
+    for metric_name, value in results.items():
+        # Skip _stderr fields as they are handled with their corresponding metrics
+        if metric_name.endswith("_stderr"):
+            continue
+        
+        # Check if there's a corresponding _stderr field
+        stderr_key = f"{metric_name}_stderr"
+        if stderr_key in results:
+            metrics[metric_name] = dict(
+                scores={metric_name: dict(value=value, stats={"stderr": results[stderr_key]})}
+            )
+        else:
+            # Provide empty stats dict as required by EvaluationResult
+            metrics[metric_name] = dict(
+                scores={metric_name: dict(value=value, stats={})}
+            )
+    
+    tasks[task_name] = dict(
+        metrics=metrics
+    )
+    return EvaluationResult(groups=tasks, tasks=tasks)

nvidia_profbench-25.11.dist-info/METADATA

@@ -0,0 +1,316 @@
+Metadata-Version: 2.4
+Name: nvidia-profbench
+Version: 25.11
+Summary: Professional domain benchmark for evaluating LLMs on Physics PhD, Chemistry PhD, Finance MBA, and Consulting MBA tasks
+Author: NVIDIA Corporation
+License: Apache License 2.0
+Project-URL: Homepage, https://github.com/nvidia/profbench
+Project-URL: Repository, https://github.com/nvidia/profbench
+Project-URL: Documentation, https://github.com/nvidia/profbench/blob/main/README.md
+Keywords: llm,evaluation,benchmark,professional,phd,mba
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: LICENSE-APACHE
+Requires-Dist: openai>=1.71.0
+Requires-Dist: datasets>=2.14.7
+Requires-Dist: huggingface-hub>=0.33.1
+Requires-Dist: numpy
+Requires-Dist: scikit-learn
+Requires-Dist: tqdm
+Requires-Dist: nemo-evaluator
+Requires-Dist: google-genai
+Provides-Extra: google
+Requires-Dist: google-genai; extra == "google"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Dynamic: license-file
+
+# NVIDIA Evals Factory
+The goal of NVIDIA Evals Factory is to advance and refine state-of-the-art methodologies for model evaluation, and deliver them as modular evaluation packages (evaluation containers and pip wheels) that teams can use as standardized building blocks.
+
+# ProfBench
+
+ProfBench is part of NVIDIA Evals Factory, providing standardized evaluation methodologies for assessing LLMs on professional domain tasks.
+
+## Overview
+
+ProfBench introduces over 3000 expert-authored response–criterion pairs across 40 tasks in four professional domains: **Physics PhD**, **Chemistry PhD**, **Finance MBA**, and **Consulting MBA**. It enables evaluation of open-ended, document-grounded professional tasks beyond exam-style QA or code/math-only settings.
+
+**Key Features:**
+- Realistic professional workflows requiring synthesis and long-form analysis
+- Robust, affordable LLM-Judge combining Macro-F1 measure with Bias Index
+- Achieves <1% cross-provider bias while reducing costs by 2-3 orders of magnitude
+- Evaluation cost: ~$12 per run (vs ~$300 for HealthBench, ~$8000 for PaperBench with OpenAI o3)
+
+Even frontier models find ProfBench challenging: the best report-generator GPT-5-high reaches only 65.9% overall, underscoring substantial headroom in realistic professional workflows.
+
+## Quick Start Guide
+
+NVIDIA Evals Factory provide you with evaluation clients, that are specifically built to evaluate model endpoints using our Standard API.
+
+### Launching an Evaluation
+
+#### List the Available Evaluations
+
+```bash
+$ nemo-evaluator ls
+```
+
+Output:
+```
+profbench: 
+  * report_generation
+  * llm_judge
+```
+
+#### Run the Evaluation of Your Choice
+
+**LLM Judge Evaluation:**
+```bash
+export API_KEY=your_nvidia_api_key_here
+
+nemo-evaluator run_eval \
+    --eval_type llm_judge \
+    --model_id meta/llama-3.1-70b-instruct \
+    --model_type chat \
+    --model_url https://integrate.api.nvidia.com/v1/chat/completions \
+    --api_key_name API_KEY \
+    --output_dir './results/profbench_llm_judge'
+```
+
+**Report Generation Evaluation:**
+```bash
+nemo-evaluator run_eval \
+    --eval_type report_generation \
+    --model_id meta/llama-3.1-70b-instruct \
+    --model_type chat \
+    --model_url https://integrate.api.nvidia.com/v1/chat/completions \
+    --api_key_name API_KEY \
+    --output_dir './results/profbench_report_generation'
+```
+
+#### Gather the Results
+
+```bash
+cat ./results/profbench_llm_judge/results.yml
+```
+
+**Example LLM Judge Output:**
+```yaml
+results:
+  tasks:
+    llm_judge:
+      metrics:
+        Overall:
+          scores:
+            Overall:
+              value: 65.3
+        Physics PhD:
+          scores:
+            Physics PhD:
+              value: 66.5
+        Chemistry PhD:
+          scores:
+            Chemistry PhD:
+              value: 60.3
+        Finance MBA:
+          scores:
+            Finance MBA:
+              value: 61.4
+        Consulting MBA:
+          scores:
+            Consulting MBA:
+              value: 63.4
+        BIAS-INDEX:
+          scores:
+            BIAS-INDEX:
+              value: 4.0
+        MF1-BI:
+          scores:
+            MF1-BI:
+              value: 61.3
+```
+
+**Example Report Generation Output:**
+```yaml
+results:
+  tasks:
+    report_generation:
+      metrics:
+        Overall:
+          scores:
+            Overall:
+              value: 11.4
+        Physics PhD:
+          scores:
+            Physics PhD:
+              value: 3.4
+        Chemistry PhD:
+          scores:
+            Chemistry PhD:
+              value: 7.1
+        Finance MBA:
+          scores:
+            Finance MBA:
+              value: 6.0
+        Consulting MBA:
+          scores:
+            Consulting MBA:
+              value: 28.9
+```
+
+## Command-Line Tool
+
+Each package comes pre-installed with a set of command-line tools, designed to simplify the execution of evaluation tasks through the `nemo-evaluator` interface.
+
+### Commands
+
+#### 1. List Evaluation Types
+```bash
+nemo-evaluator ls
+```
+Displays the evaluation types available within the ProfBench harness.
+
+#### 2. Run an Evaluation
+
+The `nemo-evaluator run_eval` command executes the evaluation process. Below are the flags and their descriptions:
+
+**Required Flags:**
+- `--eval_type <string>` - The type of evaluation to perform (`llm_judge` or `report_generation`)
+- `--model_id <string>` - The name or identifier of the model to evaluate
+- `--model_url <url>` - The API endpoint where the model is accessible
+- `--model_type <string>` - The type of the model to evaluate, currently either `chat`, `completions`, or `vlm`
+- `--output_dir <directory>` - The directory to use as the working directory for the evaluation. The results, including the `results.yml` output file, will be saved here
+
+**Optional Flags:**
+- `--api_key_name <string>` - The name of the environment variable that stores the Bearer token for the API, if authentication is required
+- `--run_config <path>` - Specifies the path to a YAML file containing the evaluation definition
+- `--overrides <string>` - Override configuration parameters (e.g., `"config.params.limit_samples=10"`)
+- `--dry_run` - Print the final run configuration and command without executing the evaluation
+
+#### Example
+
+```bash
+nemo-evaluator run_eval \
+    --eval_type llm_judge \
+    --model_id meta/llama-3.1-70b-instruct \
+    --model_type chat \
+    --model_url https://integrate.api.nvidia.com/v1/chat/completions \
+    --output_dir ./evaluation_results
+```
+
+If the model API requires authentication, set the API key in an environment variable and reference it using the `--api_key_name` flag:
+
+```bash
+export API_KEY="your_api_key_here"
+
+nemo-evaluator run_eval \
+    --eval_type llm_judge \
+    --model_id meta/llama-3.1-70b-instruct \
+    --model_type chat \
+    --model_url https://integrate.api.nvidia.com/v1/chat/completions \
+    --api_key_name API_KEY \
+    --output_dir ./evaluation_results
+```
+
+## Configuring Evaluations via YAML
+
+Evaluations in NVIDIA Evals Factory are configured using YAML files that define the parameters and settings required for the evaluation process. These configuration files follow a standard API which ensures consistency across evaluations.
+
+**Example of a YAML config:**
+
+```yaml
+config:
+  type: llm_judge
+  params:
+    parallelism: 10
+    limit_samples: 30
+    max_new_tokens: 4096
+    temperature: 0.0
+    top_p: 0.00001
+target:
+  api_endpoint:
+    model_id: meta/llama-3.1-70b-instruct
+    type: chat
+    url: https://integrate.api.nvidia.com/v1/chat/completions
+    api_key: API_KEY
+```
+
+The priority of overrides is as follows:
+
+1. Command line arguments
+2. User config (as seen above)
+3. Task defaults (defined per task type)
+4. Framework defaults
+
+**`--dry_run` option** allows you to print the final run configuration and command without executing the evaluation.
+
+**Example:**
+```bash
+nemo-evaluator run_eval \
+    --eval_type llm_judge \
+    --model_id meta/llama-3.1-70b-instruct \
+    --model_type chat \
+    --model_url https://integrate.api.nvidia.com/v1/chat/completions \
+    --output_dir ./evaluation_results \
+    --dry_run
+```
+
+## FAQ
+
+### Deploying a Model as an Endpoint
+
+NVIDIA Evals Factory utilize a client-server communication architecture to interact with the model. As a prerequisite, the model must be deployed as an endpoint with a NIM-compatible API.
+
+Users have the flexibility to deploy their model using their own infrastructure and tooling.
+
+Servers with APIs that conform to the OpenAI/NIM API standard are expected to work seamlessly out of the box.
+
+### Evaluation Types
+
+**1. LLM Judge (`llm_judge`)**
+
+Evaluates model responses against expert-authored criteria across professional domains.
+
+- Assesses professional reasoning, extraction accuracy, and writing style
+- Provides domain-specific scores (Physics PhD, Chemistry PhD, Finance MBA, Consulting MBA)
+- Includes bias metrics (BIAS-INDEX, MF1-BI) to measure self-enhancement bias
+
+**2. Report Generation (`report_generation`)**
+
+End-to-end evaluation pipeline for professional report writing.
+
+- Generates professional reports from prompts
+- Uses LLM-as-a-judge to evaluate generated reports
+- Scores across domains and skill dimensions (Reasoning, Extraction, Style)
+
+### Metrics Explained
+
+**Domain Scores:** Performance on specific professional domains:
+- Physics PhD
+- Chemistry PhD
+- Finance MBA
+- Consulting MBA
+
+
+### Supported Libraries
+
+- **`openai`** - OpenAI API and OpenAI-compatible endpoints (NVIDIA NIM, OpenRouter, vLLM, etc.)
+
+**Note:** Web search features are only available with Google GenAI library. This feature will raise `NotImplementedError` if used with OpenAI or OpenRouter APIs.
+
+### Contributing
+
+This software is distributed as a pip wheel only. The source repository is internal to NVIDIA and not publicly accessible. For issues or questions, please refer to the NVIDIA support channels.
+
+### License
+This software is distributed under the terms of both the MIT license and the Apache License (Version 2.0). See LICENSE and LICENSE-APACHE for details.

nvidia_profbench-25.11.dist-info/RECORD

@@ -0,0 +1,17 @@
+core_evals/profbench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+core_evals/profbench/framework.yml,sha256=jEA-sJU0aghJ1hZ_NIx_4ptRGXDeJOl4HzYUc4bziZY,5530
+core_evals/profbench/output.py,sha256=A4NyYvQGA1BWSxF4jRIx6v5ewVUW5hQyGWchhqUtiC4,2707
+nvidia_profbench-25.11.dist-info/licenses/LICENSE,sha256=XceIwpnulSX-tAU4MME1jFKcZ6kG5GExHiIAqJ8NX_w,1066
+nvidia_profbench-25.11.dist-info/licenses/LICENSE-APACHE,sha256=cSWJDwVqHyQgly8Zmt3pqXJ2eQbZVYwN9qd0NMssxXY,11336
+profbench/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+profbench/run_best_llm_judge_on_generated_reports.py,sha256=G09xMPdM507vsf4IACuuC1RGAJ7GmcAUHHUWahsUWZA,5012
+profbench/run_llm_judge_on_provided_reports.py,sha256=_BBfm2-xg1vKngPH6Ob6g0uDUivgACz8b1StPWmo4dA,5527
+profbench/run_report_generation.py,sha256=AfZIyrAWTeJm3I4NpVtdoTx2v2psbFvgXaagNqyAaSQ,11864
+profbench/score_llm_judge.py,sha256=CpT1HHI3fRda0h7k7fgUUBtptmC54Fhu11GbdIE-qz8,3870
+profbench/score_report_generation.py,sha256=v9w15qGrh4Wn8E5QTSD3asz8iU5Kc8EQBfHT9US81dQ,4729
+profbench/utils.py,sha256=Vqbq6M9KSkdCPQAr0vxYdX4uSySovoywRoFhTnVbgiY,5679
+nvidia_profbench-25.11.dist-info/METADATA,sha256=QvVMQtaRloQkwpfN2o0-FPAFsHTrNFc8Yq7-tSVz16U,10360
+nvidia_profbench-25.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nvidia_profbench-25.11.dist-info/entry_points.txt,sha256=AYZANph5MOOjCEj6bfLvAJgBCF45sfIUskE8zxzZRY4,167
+nvidia_profbench-25.11.dist-info/top_level.txt,sha256=X-IIup1giKjIVInf1m4vXX4FH1-dVqHtDSEIlL2UR_g,21
+nvidia_profbench-25.11.dist-info/RECORD,,

nvidia_profbench-25.11.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

nvidia_profbench-25.11.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+profbench-generate = profbench.cli:run_report_generation
+profbench-judge = profbench.cli:run_llm_judge
+profbench-score = profbench.cli:score_results

nvidia_profbench-25.11.dist-info/licenses/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2025 NVIDIA Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.