sgl-eval 0.1.0__py3-none-any.whl

sgl_eval/__init__.py

@@ -0,0 +1,10 @@
+"""sgl-eval: one-click accuracy evaluation harness for SGLang."""
+
+from pathlib import Path
+
+__version__ = "0.1.0"
+
+# Single source of truth for the vendored NeMo-Skills slice path. Used by
+# the dataset loader, the CLI's run-meta provenance reader, and any future
+# consumer that needs to reach into ``_vendored/nemo_skills/``.
+VENDORED_NS_ROOT = Path(__file__).resolve().parent / "_vendored" / "nemo_skills"

sgl_eval/_vendored/__init__.py

@@ -0,0 +1,6 @@
+"""Vendored third-party code, organized by upstream source.
+
+Each subpackage corresponds to one upstream project; ``SOURCES.yaml`` inside
+records the synced commit and import-rewrite map. See ROADMAP.md
+"Dependency policy" and ``scripts/sync_vendored.py``.
+"""

sgl_eval/_vendored/nemo_skills/SOURCES.yaml

@@ -0,0 +1,198 @@
+# Manifest for files vendored from NVIDIA/NeMo-Skills.
+#
+# Edit this file then run `python scripts/sync_vendored.py` to refresh the
+# vendored sources. Banner comments at the top of each generated file
+# point back here.
+#
+# Per ROADMAP.md "Dependency policy": external pip deps that vendored
+# files import must mirror upstream's `core/requirements.txt` at
+# `synced_from_sha`. Mirror in `pyproject.toml` directly.
+
+upstream_repo: NVIDIA/NeMo-Skills
+upstream_branch: main
+synced_from_sha: 645cf567ff08c0ae9cc3fc8e1edbb975b3067816
+synced_at: "2026-04-29"
+
+# pip deps that vendored files actually `import`. Reflected in
+# pyproject.toml. Anything upstream pulls in but our vendored files do
+# not import (torch, transformers, flask, gradio, hydra-core, ...) is
+# intentionally absent.
+pinned_deps:
+  math_verify: "[antlr4_9_3]"   # upstream pin: unversioned, with antlr4_9_3 extra
+  latex2sympy2_extended: ""     # upstream pin: unversioned
+  numpy: ""                     # upstream pin: unversioned
+  tqdm: ""                      # upstream pin: unversioned (used by evaluator/base.py)
+  editdistance: ""              # upstream pin: unversioned (evaluator/ruler.py WER)
+  datasets: ""                  # upstream pin: unversioned (every dataset/*/prepare.py)
+  # RULER2 dataset generation only. Kept out of the core install because
+  # nothing else needs them and `transformers` is heavy -- they live in the
+  # `longcontext` extra. All four are imported by dataset/ruler2/prepare_*.py;
+  # `transformers` / `tiktoken` are lazy (inside tokenizer.py's __init__).
+  nltk: ""
+  wonderwords: ""
+  inflect: ""
+  transformers: ""
+
+# Applied by sync_vendored.py to rewrite intra-nemo-skills imports so
+# vendored files resolve under sgl_eval._vendored.nemo_skills. Audited
+# by tests/test_no_untranslated_imports.py.
+import_rewrites:
+  "from nemo_skills.utils import": "from sgl_eval._vendored.nemo_skills._utils import"
+  "from nemo_skills.evaluation.metrics.base import": "from sgl_eval._vendored.nemo_skills._metrics_base import"
+  "from nemo_skills.evaluation.metrics.utils import": "from sgl_eval._vendored.nemo_skills._metrics_utils import"
+  "from nemo_skills.evaluation.math_grader import": "from sgl_eval._vendored.nemo_skills.math_grader import"
+  "from nemo_skills.evaluation.evaluator.base import": "from sgl_eval._vendored.nemo_skills.evaluator.base import"
+  "from nemo_skills.evaluation.evaluator.math import": "from sgl_eval._vendored.nemo_skills.evaluator.math import"
+  "from nemo_skills.evaluation.evaluator.mcq import": "from sgl_eval._vendored.nemo_skills.evaluator.mcq import"
+  "from nemo_skills.dataset.utils import": "from sgl_eval._vendored.nemo_skills.dataset._utils import"
+  # ruler2/prepare.py spawns its sibling generators as `python -m
+  # nemo_skills.dataset.ruler2.prepare_niah` and emits SCORE_MODULE as a
+  # string literal. Both are module paths in source text rather than import
+  # statements, so the audit regex cannot see them -- rewrite the dotted
+  # prefix so the subprocess resolves under _vendored.
+  "nemo_skills.dataset.ruler2.": "sgl_eval._vendored.nemo_skills.dataset.ruler2."
+
+# Files synced verbatim from upstream (with rewrites + drop_functions +
+# drop_imports applied, plus a provenance banner for .py / .yaml).
+# `binary: true` copies bytes verbatim with no banner (data files).
+files:
+  # --- evaluation core ---
+  - src: nemo_skills/evaluation/math_grader.py
+    dst: math_grader.py
+  - src: nemo_skills/evaluation/metrics/utils.py
+    dst: _metrics_utils.py
+  - src: nemo_skills/evaluation/metrics/base.py
+    dst: _metrics_base.py
+  - src: nemo_skills/evaluation/metrics/math_metrics.py
+    dst: math_metrics.py
+    drop_functions:
+      - _compute_reward_at_k
+
+  # --- evaluator wrappers ---
+  - src: nemo_skills/evaluation/evaluator/base.py
+    dst: evaluator/base.py
+  - src: nemo_skills/evaluation/evaluator/math.py
+    dst: evaluator/math.py
+    # Lean4ProofEvaluator pulls in code_execution.proof_utils + sandbox;
+    # we don't run Lean proofs, so drop the class and its imports.
+    drop_functions:
+      - Lean4ProofEvaluator
+    drop_imports:
+      - nemo_skills.code_execution
+  - src: nemo_skills/evaluation/evaluator/mcq.py
+    dst: evaluator/mcq.py
+  - src: nemo_skills/evaluation/evaluator/ruler.py
+    dst: evaluator/ruler.py
+  - src: nemo_skills/evaluation/metrics/ruler2_metrics.py
+    dst: ruler2_metrics.py
+
+  # --- dataset utilities ---
+  - src: nemo_skills/dataset/utils.py
+    dst: dataset/_utils.py
+
+  # --- per-benchmark dataset metadata + bundled data ---
+  - src: nemo_skills/dataset/aime25/__init__.py
+    dst: dataset/aime25/__init__.py
+  - src: nemo_skills/dataset/aime25/test.txt
+    dst: dataset/aime25/test.txt
+    binary: true
+  - src: nemo_skills/dataset/aime26/__init__.py
+    dst: dataset/aime26/__init__.py
+  - src: nemo_skills/dataset/aime24/__init__.py
+    dst: dataset/aime24/__init__.py
+  - src: nemo_skills/dataset/aime24/test.txt
+    dst: dataset/aime24/test.txt
+    binary: true
+  - src: nemo_skills/dataset/gsm8k/__init__.py
+    dst: dataset/gsm8k/__init__.py
+  - src: nemo_skills/dataset/gsm8k/prepare.py
+    dst: dataset/gsm8k/prepare.py
+  - src: nemo_skills/dataset/mmlu/__init__.py
+    dst: dataset/mmlu/__init__.py
+  - src: nemo_skills/dataset/mmlu/prepare.py
+    dst: dataset/mmlu/prepare.py
+  - src: nemo_skills/dataset/gpqa/__init__.py
+    dst: dataset/gpqa/__init__.py
+  - src: nemo_skills/dataset/gpqa/prepare.py
+    dst: dataset/gpqa/prepare.py
+  # `dst` de-hyphenates because `_resolve_upstream_metadata` imports this dir
+  # as a Python module. Unlike gpqa/mmlu, this prepare.py exposes only an
+  # argparse `main(args)`, which the registry row flags with `argparse_main`.
+  # `subsets/` is deliberately absent: it backs upstream's `10pct_opt_v1`
+  # split, and sgl-eval evaluates `test`.
+  - src: nemo_skills/dataset/mmlu-pro/__init__.py
+    dst: dataset/mmlu_pro/__init__.py
+  - src: nemo_skills/dataset/mmlu-pro/prepare.py
+    dst: dataset/mmlu_pro/prepare.py
+  # MMMU-Pro `vision` config (see `_registry.py` for what it evaluates).
+  # `dst` de-hyphenates because `_resolve_upstream_metadata` imports this dir
+  # as a Python module; it says `vision` to stay distinct from the `mmmu_pro`
+  # row, which evaluates a config upstream does not ship.
+  - src: nemo_skills/dataset/mmmu-pro/__init__.py
+    dst: dataset/mmmu_pro_vision/__init__.py
+  - src: nemo_skills/dataset/mmmu-pro/prepare.py
+    dst: dataset/mmmu_pro_vision/prepare.py
+
+  # --- RULER2 (synthetic long-context; generated per tokenizer + seq length) ---
+  # Upstream ships no test split for this one: `__init__.py` is just
+  # REQUIRES_DATA_DIR = True, and the 12 subtasks are generated locally. Note
+  # sgl-eval calls the per-task `prepare_<task>(output_folder, ...)` functions
+  # directly rather than `prepare_dataset`, whose output_folder is hardcoded to
+  # `Path(__file__).parent / setup` -- i.e. it would write into _vendored.
+  - src: nemo_skills/dataset/ruler2/__init__.py
+    dst: dataset/ruler2/__init__.py
+  - src: nemo_skills/dataset/ruler2/prepare.py
+    dst: dataset/ruler2/prepare.py
+  - src: nemo_skills/dataset/ruler2/prepare_niah.py
+    dst: dataset/ruler2/prepare_niah.py
+  - src: nemo_skills/dataset/ruler2/prepare_qa.py
+    dst: dataset/ruler2/prepare_qa.py
+  - src: nemo_skills/dataset/ruler2/prepare_mmlu.py
+    dst: dataset/ruler2/prepare_mmlu.py
+  - src: nemo_skills/dataset/ruler2/tokenizer.py
+    dst: dataset/ruler2/tokenizer.py
+    # GeminiTokenizer is the only module-level `tenacity` user and needs a
+    # GEMINI_API_KEY. sgl-eval restricts tokenizer_type to hf/openai.
+    drop_functions:
+      - GeminiTokenizer
+    drop_imports:
+      - tenacity
+  - src: nemo_skills/dataset/ruler2/ruler2_score.py
+    dst: dataset/ruler2/ruler2_score.py
+
+  # --- prompt templates ---
+  - src: nemo_skills/prompt/config/generic/math.yaml
+    dst: prompts/math.yaml
+  - src: nemo_skills/prompt/config/eval/aai/mcq-4choices.yaml
+    dst: prompts/mcq-4choices.yaml
+  # Not a default for any benchmark -- reachable only via `run --prompt
+  # matharena-aime`. Published AIME numbers are often produced with this
+  # phrasing (it states the 0-999 integer range, which `generic/math` does
+  # not), so a run reproducing one needs it available.
+  - src: nemo_skills/prompt/config/eval/matharena/aime.yaml
+    dst: prompts/matharena-aime.yaml
+  - src: nemo_skills/prompt/config/eval/aai/mcq-4choices-boxed.yaml
+    dst: prompts/mcq-4choices-boxed.yaml
+  # MMLU-Pro's default. Hardcodes A-J, so it is not interchangeable with
+  # sgl-eval's own `mmmu-pro-cot` -- see the note there.
+  - src: nemo_skills/prompt/config/eval/aai/mcq-10choices.yaml
+    dst: prompts/mcq-10choices.yaml
+  # RULER2's ++prompt_config. Pure passthrough (`user: "{question}"`) -- the
+  # long context is already assembled by the prepare scripts.
+  - src: nemo_skills/prompt/config/generic/default.yaml
+    dst: prompts/default.yaml
+  # mmmu_pro_vision's ++prompt_config (`vlm/mmmu-pro`). The `dst` basename
+  # must stay hyphenated: `_resolve_upstream_metadata` derives it from
+  # GENERATION_ARGS via `prompt_config.split("/")[-1]`. Carries
+  # `image_position: before`, which `_prompts.prompt_media_config` reads.
+  - src: nemo_skills/prompt/config/vlm/mmmu-pro.yaml
+    dst: prompts/mmmu-pro.yaml
+
+  # --- upstream's own behavior tests ---
+  # Run as part of `pytest`. These are NS's canonical regression cases,
+  # so any drift in the vendored grader / metrics / dep versions surfaces
+  # as a test failure here. Do NOT hand-edit; refresh via sync_vendored.
+  - src: tests/test_math_equal.py
+    dst: tests/test_math_equal.py
+  - src: tests/test_base_metrics.py
+    dst: tests/test_base_metrics.py

sgl_eval/_vendored/nemo_skills/__init__.py

@@ -0,0 +1,5 @@
+"""Vendored slice of NVIDIA/NeMo-Skills.
+
+See ``SOURCES.yaml`` for the synced commit and rewrite map.
+Apache-2.0 licensed (see top-level LICENSE).
+"""