evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/api/registry.py

@@ -0,0 +1,182 @@
+import copy
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Type, Union
+
+if TYPE_CHECKING:
+    from evalscope.api.benchmark import BenchmarkMeta, DataAdapter
+    from evalscope.api.filter import Filter
+    from evalscope.api.metric import Aggregator, Metric
+    from evalscope.api.model.model import ModelAPI
+    from evalscope.config import TaskConfig
+
+# BEGIN: Registry for benchmarks
+# Registry for benchmarks, allowing dynamic registration and retrieval of benchmark metadata and data adapters.
+BENCHMARK_REGISTRY: Dict[str, 'BenchmarkMeta'] = {}
+
+
+def register_benchmark(metadata: 'BenchmarkMeta'):
+    """Register a benchmark with its metadata."""
+
+    def register_wrapper(data_adapter: Type['DataAdapter']):
+        if metadata.name in BENCHMARK_REGISTRY:
+            raise ValueError(f'Benchmark {metadata.name} already registered')
+        metadata.data_adapter = data_adapter
+        BENCHMARK_REGISTRY[metadata.name] = metadata
+        return data_adapter
+
+    return register_wrapper
+
+
+def get_benchmark(name: str, config: Optional['TaskConfig'] = None) -> 'DataAdapter':
+    """
+    Retrieve a registered benchmark by name.
+
+    Args:
+        name (str): The name of the benchmark.
+        config (Optional['TaskConfig']): The task configuration.
+        dataset_args (Optional[dict]): The dataset-specific arguments.
+
+    """
+    # copy to avoid modifying the original metadata
+    metadata = copy.deepcopy(BENCHMARK_REGISTRY.get(name))
+    if not metadata:
+        raise ValueError(f'Benchmark {name} not found, available benchmarks: {list(sorted(BENCHMARK_REGISTRY.keys()))}')
+
+    # Update metadata with dataset-specific configuration
+    if config is not None:
+        metadata._update(config.dataset_args.get(name, {}))
+    # Return the data adapter initialized with the benchmark metadata
+    data_adapter_cls = metadata.data_adapter
+    return data_adapter_cls(benchmark_meta=metadata, task_config=config)
+
+
+# END: Registry for benchmarks
+
+# BEGIN: Registry for model APIs
+# Registry for model APIs, allowing dynamic registration and retrieval of model API classes.
+MODEL_APIS: Dict[str, Type['ModelAPI']] = {}
+
+
+def register_model_api(name: str):
+    """
+    Decorator to register a model API class with a given name.
+
+    :param name: The name of the model API.
+    """
+
+    def decorator(api_class: Type['ModelAPI']):
+        if name in MODEL_APIS:
+            raise ValueError(f"Model API '{name}' is already registered.")
+        MODEL_APIS[name] = api_class
+        return api_class
+
+    return decorator
+
+
+def get_model_api(name: str) -> Type['ModelAPI']:
+    """
+    Retrieve a registered model API class by name.
+
+    :param name: The name of the model API.
+    :return: The model API class.
+    """
+    if name not in MODEL_APIS:
+        raise ValueError(f"Model API '{name}' is not registered. Available model APIs: {list(MODEL_APIS.keys())}")
+
+    wrapped = MODEL_APIS[name]
+    if not isinstance(wrapped, type):
+        return wrapped()
+    else:
+        return wrapped
+
+
+# END: Registry for model APIs
+
+# BEGIN: Registry for metrics
+METRIC_REGISTRY: Dict[str, Type['Metric']] = {}
+
+
+def register_metric(name: str):
+
+    def decorate(fn):
+        if name in METRIC_REGISTRY:
+            raise ValueError(f"Metric named '{name}' conflicts with existing registered metric!")
+
+        METRIC_REGISTRY[name] = fn
+        return fn
+
+    return decorate
+
+
+def get_metric(name: str) -> Type['Metric']:
+    if name in METRIC_REGISTRY:
+        return METRIC_REGISTRY[name]
+    else:
+        raise ValueError(
+            f"Metric '{name}' not found in the registry. Available metrics: {list(METRIC_REGISTRY.keys())}"
+        )
+
+
+# END: Registry for metrics
+
+# BEGIN: Registry for filters
+
+FILTER_REGISTRY: Dict[str, Type['Filter']] = {}
+
+
+def register_filter(name):
+
+    def decorate(cls):
+        if name in FILTER_REGISTRY:
+            raise ValueError(f'Registering filter `{name}` that is already in Registry {FILTER_REGISTRY}')
+        FILTER_REGISTRY[name] = cls
+        return cls
+
+    return decorate
+
+
+def get_filter(filter_name: str) -> Type['Filter']:
+    if filter_name not in FILTER_REGISTRY:
+        raise KeyError(
+            f"Filter '{filter_name}' not found in the registry. Available filters: {list(FILTER_REGISTRY.keys())}"
+        )
+    return FILTER_REGISTRY[filter_name]
+
+
+# END: Registry for filters
+
+# BEGIN: Registry for aggregation functions
+AGGREGATION_REGISTRY: Dict[str, Type['Aggregator']] = {}
+
+
+def register_aggregation(name: str):
+    """
+    Decorator to register an aggregation function with a given name.
+
+    :param name: The name of the aggregation function.
+    """
+
+    def decorator(aggregation_fn: 'Aggregator'):
+        if name in AGGREGATION_REGISTRY:
+            raise ValueError(f"Aggregation function '{name}' is already registered.")
+        AGGREGATION_REGISTRY[name] = aggregation_fn
+        return aggregation_fn
+
+    return decorator
+
+
+def get_aggregation(name: str) -> Type['Aggregator']:
+    """
+    Retrieve a registered aggregation function by name.
+
+    :param name: The name of the aggregation function.
+    :return: The aggregation function.
+    """
+    if name not in AGGREGATION_REGISTRY:
+        raise ValueError(
+            f"Aggregation function '{name}' is not registered. "
+            f'Available aggregations: {list(AGGREGATION_REGISTRY.keys())}'
+        )
+    return AGGREGATION_REGISTRY[name]
+
+
+# END: Registry for aggregation functions

evalscope/api/tool/__init__.py

@@ -0,0 +1,3 @@
+from .tool_call import ToolCall, ToolCallContent, ToolCallError, ToolCallView, ToolChoice, ToolFunction
+from .tool_info import Tool, ToolDescription, ToolInfo, ToolParams, set_tool_description, tool_description
+from .utils import parse_tool_call, tool_parse_error_message

evalscope/api/tool/tool_call.py

@@ -0,0 +1,101 @@
+import json
+from pydantic import BaseModel, Field, JsonValue, field_validator
+from typing import Any, Callable, Dict, List, Literal, Optional, Union
+
+
+class ToolFunction(BaseModel):
+    """Indicate that a specific tool function should be called."""
+
+    name: str
+    """The name of the tool function to call."""
+
+    arguments: Dict[str, Any]
+    """The arguments of the tool function to call"""
+
+    @field_validator('arguments', mode='before')
+    @classmethod
+    def parse_arguments(cls, v):
+        if isinstance(v, str):
+            try:
+                v = json.loads(v)
+            except Exception as e:
+                raise ValueError(f'arguments field string is not valid JSON: {e}')
+        if not isinstance(v, dict):
+            raise ValueError('arguments must be a dict or a JSON string representing a dict')
+        return v
+
+
+class ToolCallContent(BaseModel):
+    """Content to include in tool call view."""
+
+    title: Optional[str] = Field(default=None)
+    """Optional (plain text) title for tool call content."""
+
+    format: Literal['text', 'markdown']
+    """Format (text or markdown)."""
+
+    content: str
+    """Text or markdown content."""
+
+
+class ToolCallView(BaseModel):
+    """Custom view of a tool call.
+
+    Both `context` and `call` are optional. If `call` is not specified
+    then the view will default to a syntax highlighted Python function call.
+    """
+
+    context: Optional[ToolCallContent] = Field(default=None)
+    """Context for the tool call (i.e. current tool state)."""
+
+    call: Optional[ToolCallContent] = Field(default=None)
+    """Custom representation of tool call."""
+
+
+class ToolCall(BaseModel):
+    id: str
+    """Unique identifier for tool call."""
+
+    function: ToolFunction
+    """Function to call."""
+
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+    parse_error: Optional[str] = Field(default=None)
+    """Error which occurred parsing tool call."""
+
+    view: Optional[ToolCallContent] = Field(default=None)
+    """Custom view of tool call input."""
+
+    type: Optional[str] = Field(default=None)
+    """Tool call type (deprecated)."""
+
+
+class ToolCallError(BaseModel):
+    """Error raised by a tool call."""
+
+    type: Literal[
+        'parsing',
+        'timeout',
+        'unicode_decode',
+        'permission',
+        'file_not_found',
+        'is_a_directory',
+        'limit',
+        'approval',
+        'unknown',
+    ]
+    """Error type."""
+
+    message: str
+    """Error message."""
+
+
+ToolChoice = Union[Literal['auto', 'any', 'none'], ToolFunction]
+"""Specify which tool to call.
+
+"auto" means the model decides; "any" means use at least one tool,
+"none" means never call a tool; ToolFunction instructs the model
+to call a specific function.
+"""

evalscope/api/tool/tool_info.py

@@ -0,0 +1,173 @@
+import inspect
+from dataclasses import dataclass
+from docstring_parser import Docstring, parse
+from pydantic import BaseModel, Field
+from typing import Any, Callable, Dict, List, Literal, Optional, TypeAlias, Union, get_args, get_type_hints
+
+from evalscope.utils.json_schema import JSONSchema, JSONType, json_schema, python_type_to_json_type
+
+ToolParam: TypeAlias = JSONSchema
+"""Description of tool parameter in JSON Schema format."""
+
+
+class Tool:
+
+    def __call__(
+        self,
+        *args: Any,
+        **kwargs: Any,
+    ) -> Any:
+        ...
+
+
+class ToolParams(BaseModel):
+    """Description of tool parameters object in JSON Schema format."""
+
+    type: Literal['object'] = Field(default='object')
+    """Params type (always 'object')"""
+
+    properties: Dict[str, ToolParam] = Field(default_factory=dict)
+    """Tool function parameters."""
+
+    required: List[str] = Field(default_factory=list)
+    """List of required fields."""
+
+    additionalProperties: bool = Field(default=False)
+    """Are additional object properties allowed? (always `False`)"""
+
+
+@dataclass
+class ToolDescription:
+    name: Optional[str] = None
+    description: Optional[str] = None
+    parameters: Optional[ToolParams] = None
+
+
+def tool_description(tool: Tool) -> ToolDescription:
+    return getattr(tool, TOOL_DESCRIPTION, ToolDescription())
+
+
+def set_tool_description(tool: Tool, description: ToolDescription) -> None:
+    setattr(tool, TOOL_DESCRIPTION, description)
+
+
+TOOL_DESCRIPTION = '__TOOL_DESCRIPTION__'
+
+
+class ToolInfo(BaseModel):
+    """Specification of a tool (JSON Schema compatible)
+
+    If you are implementing a ModelAPI, most LLM libraries can
+    be passed this object (dumped to a dict) directly as a function
+    specification. For example, in the OpenAI provider:
+
+    ```python
+    ChatCompletionToolParam(
+        type="function",
+        function=tool.model_dump(exclude_none=True),
+    )
+    ```
+
+    In some cases the field names don't match up exactly. In that case
+    call `model_dump()` on the `parameters` field. For example, in the
+    Anthropic provider:
+
+    ```python
+    ToolParam(
+        name=tool.name,
+        description=tool.description,
+        input_schema=tool.parameters.model_dump(exclude_none=True),
+    )
+    ```
+    """
+
+    name: str
+    """Name of tool."""
+    description: str
+    """Short description of tool."""
+    parameters: ToolParams = Field(default_factory=ToolParams)
+    """JSON Schema of tool parameters object."""
+    options: Optional[Dict[str, object]] = Field(default=None)
+    """Optional property bag that can be used by the model provider to customize the implementation of the tool"""
+
+
+def parse_tool_info(func: Callable[..., Any]) -> ToolInfo:
+    # tool may already have registry attributes w/ tool info
+    description = tool_description(func)
+    if (description.name and description.description and description.parameters is not None):
+        return ToolInfo(
+            name=description.name,
+            description=description.description,
+            parameters=description.parameters,
+        )
+
+    signature = inspect.signature(func)
+    type_hints = get_type_hints(func)
+    docstring = inspect.getdoc(func)
+    parsed_docstring: Optional[Docstring] = parse(docstring) if docstring else None
+
+    info = ToolInfo(name=func.__name__, description='')
+
+    for param_name, param in signature.parameters.items():
+        tool_param = ToolParam()
+
+        # Parse docstring
+        docstring_info = parse_docstring(docstring, param_name)
+
+        # Get type information from type annotations
+        if param_name in type_hints:
+            tool_param = json_schema(type_hints[param_name])
+        # as a fallback try to parse it from the docstring
+        # (this is minimally necessary for backwards compatiblity
+        #  with tools gen1 type parsing, which only used docstrings)
+        elif 'docstring_type' in docstring_info:
+            json_type = python_type_to_json_type(docstring_info['docstring_type'])
+            if json_type and (json_type in get_args(JSONType)):
+                tool_param = ToolParam(type=json_type)
+
+        # Get default value
+        if param.default is param.empty:
+            info.parameters.required.append(param_name)
+        else:
+            tool_param.default = param.default
+
+        # Add description from docstring
+        if 'description' in docstring_info:
+            tool_param.description = docstring_info['description']
+
+        # append the tool param
+        info.parameters.properties[param_name] = tool_param
+
+    # Add function description if available
+    if parsed_docstring:
+        if parsed_docstring.description:
+            info.description = parsed_docstring.description.strip()
+        elif parsed_docstring.long_description:
+            info.description = parsed_docstring.long_description.strip()
+        elif parsed_docstring.short_description:
+            info.description = parsed_docstring.short_description.strip()
+
+        # Add examples if available
+        if parsed_docstring.examples:
+            examples = '\n\n'.join([(example.description or '') for example in parsed_docstring.examples])
+            info.description = f'{info.description}\n\nExamples\n\n{examples}'
+
+    return info
+
+
+def parse_docstring(docstring: Optional[str], param_name: str) -> Dict[str, str]:
+    if not docstring:
+        return {}
+
+    parsed_docstring: Docstring = parse(docstring)
+
+    for param in parsed_docstring.params:
+        if param.arg_name == param_name:
+            schema: Dict[str, str] = {'description': param.description or ''}
+
+            if param.type_name:
+                schema['docstring_type'] = param.type_name
+
+            return schema
+
+    return {}

evalscope/api/tool/utils.py

@@ -0,0 +1,64 @@
+import json
+import yaml
+from typing import Any, Dict, List, Optional
+
+from evalscope.utils import get_logger
+from .tool_call import ToolCall, ToolFunction
+from .tool_info import ToolInfo
+
+logger = get_logger()
+
+
+def parse_tool_call(id: str, function: str, arguments: str, tools: Optional[List[ToolInfo]] = None) -> ToolCall:
+    """Parse a tool call from a JSON payload.
+
+    Note that this function doesn't know about internal tool names so the caller
+    should ammend the returned `ToolCall` by mapping the parsed `function` field from
+    from an internal name to an inspect tool name and fixing up the `ToolCall` object
+    as required to reflect this change.
+    """
+    error: Optional[str] = None
+    arguments_dict: Dict[str, Any] = {}
+
+    def report_parse_error(ex: Exception) -> None:
+        nonlocal error
+        error = tool_parse_error_message(arguments, ex)
+        logger.info(error)
+
+    # if the arguments is a dict, then handle it with a plain json.loads
+    arguments = arguments.strip()
+    if arguments.startswith('{'):
+        try:
+            arguments_dict = json.loads(arguments)
+        except json.JSONDecodeError as ex:
+            report_parse_error(ex)
+
+    # otherwise parse it as yaml (which will pickup unquoted strings, numbers, and true/false)
+    # and then create a dict that maps it to the first function argument
+    elif function and tools:
+        tool_info = next(
+            (tool for tool in tools if tool.name == function and len(tool.parameters.properties) > 0),
+            None,
+        )
+        if tool_info:
+            param_names = list(tool_info.parameters.properties.keys())
+            try:
+                value = yaml.safe_load(arguments)
+                arguments_dict[param_names[0]] = value
+            except yaml.error.YAMLError:
+                # If the yaml parser fails, we treat it as a string argument.
+                arguments_dict[param_names[0]] = arguments
+
+    # return ToolCall with error payload
+    return ToolCall(
+        id=id,
+        function=ToolFunction(
+            name=function,
+            arguments=arguments_dict,
+        ),
+        parse_error=error,
+    )
+
+
+def tool_parse_error_message(arguments: str, ex: Exception) -> str:
+    return f'Error parsing the following tool call arguments:\n\n{arguments}\n\nError details: {ex}'

evalscope/app/ui/app_ui.py

@@ -32,7 +32,8 @@ def create_app_ui(args: argparse.Namespace):
 
         @sidebar.load_btn.click(
             inputs=[sidebar.reports_dropdown],
-            outputs=[visualization.single_model.report_name, visualization.multi_model.multi_report_name])
+            outputs=[visualization.single_model.report_name, visualization.multi_model.multi_report_name]
+        )
         def update_displays(reports_dropdown):
             if not reports_dropdown:
                 gr.Warning(locale_dict['note'], duration=3)