slm-eval 0.0.1__py3-none-any.whl

slm_eval/__init__.py

@@ -0,0 +1,29 @@
+import importlib.metadata
+import logging
+import os
+from importlib.util import find_spec
+
+
+__version__ = importlib.metadata.version("slm_eval")
+
+
+# Enable high-performance transfers
+os.environ.setdefault("HF_XET_HIGH_PERFORMANCE", "1")  # huggingface_hub >= 0.32.0
+if find_spec("hf_transfer") is not None:
+    os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")  # legacy hf_transfer
+
+
+# Lazy-load .evaluator module to improve CLI startup
+def __getattr__(name):
+    if name == "evaluate":
+        from .evaluator import evaluate
+
+        return evaluate
+    elif name == "simple_evaluate":
+        from .evaluator import simple_evaluate
+
+        return simple_evaluate
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = ["evaluate", "simple_evaluate", "__version__"]

slm_eval/__main__.py

@@ -0,0 +1,14 @@
+from slm_eval._cli import HarnessCLI
+from slm_eval.utils import setup_logging
+
+
+def cli_evaluate() -> None:
+    """Main CLI entry point."""
+    setup_logging()
+    parser = HarnessCLI()
+    args = parser.parse_args()
+    parser.execute(args)
+
+
+if __name__ == "__main__":
+    cli_evaluate()

slm_eval/_cli/__init__.py

@@ -0,0 +1,8 @@
+"""
+CLI subcommands to run from the terminal.
+"""
+
+from .harness import HarnessCLI
+
+
+__all__ = ["HarnessCLI"]

slm_eval/_cli/harness.py

@@ -0,0 +1,60 @@
+import argparse
+import sys
+import textwrap
+
+from slm_eval._cli.ls import List
+from slm_eval._cli.run import Run
+from slm_eval._cli.validate import Validate
+
+
+class HarnessCLI:
+    """Main CLI parser that manages all subcommands."""
+
+    def __init__(self):
+        self._parser = argparse.ArgumentParser(
+            prog="slm-eval",
+            description="slm_eval: an evaluation suite for small language models",
+            epilog=textwrap.dedent("""
+                quick start:
+                  # Basic evaluation
+                  slm-eval run --model hf --model_args pretrained=gpt2 --tasks hellaswag
+
+                  # List available tasks
+                  slm-eval ls tasks
+
+                  # Validate task configurations
+                  slm-eval validate --tasks hellaswag,arc_easy
+
+                legacy compatibility:
+                  The harness maintains backward compatibility with the original interface.
+                  If no command is specified, 'run' is automatically inserted:
+
+                  slm-eval --model hf --tasks hellaswag  # Equivalent to 'slm-eval run --model hf --tasks hellaswag'
+
+                For documentation, visit: docs/interface.md
+            """),
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+        )
+        self._parser.set_defaults(func=lambda args: self._parser.print_help())
+        self._subparsers = self._parser.add_subparsers(
+            dest="command", help="Available commands", metavar="COMMAND"
+        )
+        Run.create(self._subparsers)
+        List.create(self._subparsers)
+        Validate.create(self._subparsers)
+
+    def parse_args(self) -> argparse.Namespace:
+        """Parse arguments using the main parser."""
+        if len(sys.argv) > 2 and sys.argv[1] not in self._subparsers.choices:
+            # Backward compatibility: arguments provided but no valid subcommand - insert 'run'
+            # TODO: add warning
+            sys.argv.insert(1, "run")
+        elif len(sys.argv) == 2 and "run" in sys.argv:
+            # if only 'run' is specified, ensure it is treated as a subcommand
+            self._subparsers.choices["run"].print_help()
+            sys.exit(0)
+        return self._parser.parse_args()
+
+    def execute(self, args: argparse.Namespace) -> None:
+        """Main execution method that handles subcommands and legacy support."""
+        args.func(args)

slm_eval/_cli/ls.py

@@ -0,0 +1,81 @@
+import argparse
+import textwrap
+
+from slm_eval._cli.subcommand import SubCommand
+
+
+class List(SubCommand):
+    """Command for listing available tasks."""
+
+    def __init__(self, subparsers: argparse._SubParsersAction, *args, **kwargs):
+        # Create and configure the parser
+        super().__init__(*args, **kwargs)
+        self._parser = subparsers.add_parser(
+            "ls",
+            help="List available tasks, groups, subtasks, or tags",
+            description="List available tasks, groups, subtasks, or tags from the evaluation harness.",
+            usage="slm-eval list [tasks|groups|subtasks|tags] [--include_path DIR]",
+            epilog=textwrap.dedent("""
+                examples:
+                  # List all available tasks (includes groups, subtasks, and tags)
+                  $ slm-eval ls tasks
+
+                  # List only task groups (like 'bfcl_v4', 'int_index')
+                  $ slm-eval ls groups
+
+                  # List only individual subtasks (like 'bfcl_v4_simple_python')
+                  $ slm-eval ls subtasks
+
+                  # Include external task definitions
+                  $ slm-eval ls tasks --include_path /path/to/external/tasks
+
+                  # List tasks from multiple external paths
+                  $ slm-eval ls tasks --include_path "/path/to/tasks1:/path/to/tasks2"
+
+                organization:
+                  • Groups: Collections of tasks with aggregated metric across subtasks (e.g., 'bfcl_v4')
+                  • Subtasks: Individual evaluation tasks (e.g., 'bfcl_v4_simple_python', 'hellaswag')
+                  • Tags: Similar to groups but no aggregate metric (e.g., 'reasoning', 'knowledge', 'language')
+                  • External Tasks: Custom tasks defined in external directories
+
+                evaluation usage:
+                  After listing tasks, use them with the run command!
+
+                For more information, task configs are defined in slm_eval/tasks/slm
+            """),
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+        )
+        self._add_args()
+        self._parser.set_defaults(func=self._execute)
+
+    def _add_args(self) -> None:
+        self._parser.add_argument(
+            "what",
+            choices=["tasks", "groups", "subtasks", "tags"],
+            nargs="?",
+            help="What to list: tasks (all), groups, subtasks, or tags",
+        )
+        self._parser.add_argument(
+            "--include_path",
+            type=str,
+            default=None,
+            metavar="DIR",
+            help="Additional path to include if there are external tasks.",
+        )
+
+    def _execute(self, args: argparse.Namespace) -> None:
+        """Execute the list command."""
+        from slm_eval.tasks import TaskManager
+
+        task_manager = TaskManager(include_path=args.include_path)
+
+        if args.what == "tasks":
+            print(task_manager.list_all_tasks())
+        elif args.what == "groups":
+            print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False))
+        elif args.what == "subtasks":
+            print(task_manager.list_all_tasks(list_groups=False, list_tags=False))
+        elif args.what == "tags":
+            print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False))
+        elif args.what is None:
+            self._parser.print_help()