evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/api/tool/tool_info.py

@@ -0,0 +1,173 @@
+import inspect
+from dataclasses import dataclass
+from docstring_parser import Docstring, parse
+from pydantic import BaseModel, Field, field_validator
+from typing import Any, Callable, Dict, List, Literal, Optional, TypeAlias, Union, get_args, get_type_hints
+
+from evalscope.utils.json_schema import JSONSchema, JSONType, json_schema, python_type_to_json_type
+
+ToolParam: TypeAlias = JSONSchema
+"""Description of tool parameter in JSON Schema format."""
+
+
+class Tool:
+
+    def __call__(
+        self,
+        *args: Any,
+        **kwargs: Any,
+    ) -> Any:
+        ...
+
+
+class ToolParams(BaseModel):
+    """Description of tool parameters object in JSON Schema format."""
+
+    type: Literal['object'] = Field(default='object')
+    """Params type (always 'object')"""
+
+    properties: Dict[str, ToolParam] = Field(default_factory=dict)
+    """Tool function parameters."""
+
+    required: List[str] = Field(default_factory=list)
+    """List of required fields."""
+
+    additionalProperties: bool = Field(default=False)
+    """Are additional object properties allowed? (always `False`)"""
+
+
+@dataclass
+class ToolDescription:
+    name: Optional[str] = None
+    description: Optional[str] = None
+    parameters: Optional[ToolParams] = None
+
+
+def tool_description(tool: Tool) -> ToolDescription:
+    return getattr(tool, TOOL_DESCRIPTION, ToolDescription())
+
+
+def set_tool_description(tool: Tool, description: ToolDescription) -> None:
+    setattr(tool, TOOL_DESCRIPTION, description)
+
+
+TOOL_DESCRIPTION = '__TOOL_DESCRIPTION__'
+
+
+class ToolInfo(BaseModel):
+    """Specification of a tool (JSON Schema compatible)
+
+    If you are implementing a ModelAPI, most LLM libraries can
+    be passed this object (dumped to a dict) directly as a function
+    specification. For example, in the OpenAI provider:
+
+    ```python
+    ChatCompletionToolParam(
+        type="function",
+        function=tool.model_dump(exclude_none=True),
+    )
+    ```
+
+    In some cases the field names don't match up exactly. In that case
+    call `model_dump()` on the `parameters` field. For example, in the
+    Anthropic provider:
+
+    ```python
+    ToolParam(
+        name=tool.name,
+        description=tool.description,
+        input_schema=tool.parameters.model_dump(exclude_none=True),
+    )
+    ```
+    """
+
+    name: str
+    """Name of tool."""
+    description: str
+    """Short description of tool."""
+    parameters: ToolParams = Field(default_factory=ToolParams)
+    """JSON Schema of tool parameters object."""
+    options: Optional[Dict[str, object]] = Field(default=None)
+    """Optional property bag that can be used by the model provider to customize the implementation of the tool"""
+
+
+def parse_tool_info(func: Callable[..., Any]) -> ToolInfo:
+    # tool may already have registry attributes w/ tool info
+    description = tool_description(func)
+    if (description.name and description.description and description.parameters is not None):
+        return ToolInfo(
+            name=description.name,
+            description=description.description,
+            parameters=description.parameters,
+        )
+
+    signature = inspect.signature(func)
+    type_hints = get_type_hints(func)
+    docstring = inspect.getdoc(func)
+    parsed_docstring: Optional[Docstring] = parse(docstring) if docstring else None
+
+    info = ToolInfo(name=func.__name__, description='')
+
+    for param_name, param in signature.parameters.items():
+        tool_param = ToolParam()
+
+        # Parse docstring
+        docstring_info = parse_docstring(docstring, param_name)
+
+        # Get type information from type annotations
+        if param_name in type_hints:
+            tool_param = json_schema(type_hints[param_name])
+        # as a fallback try to parse it from the docstring
+        # (this is minimally necessary for backwards compatiblity
+        #  with tools gen1 type parsing, which only used docstrings)
+        elif 'docstring_type' in docstring_info:
+            json_type = python_type_to_json_type(docstring_info['docstring_type'])
+            if json_type and (json_type in get_args(JSONType)):
+                tool_param = ToolParam(type=json_type)
+
+        # Get default value
+        if param.default is param.empty:
+            info.parameters.required.append(param_name)
+        else:
+            tool_param.default = param.default
+
+        # Add description from docstring
+        if 'description' in docstring_info:
+            tool_param.description = docstring_info['description']
+
+        # append the tool param
+        info.parameters.properties[param_name] = tool_param
+
+    # Add function description if available
+    if parsed_docstring:
+        if parsed_docstring.description:
+            info.description = parsed_docstring.description.strip()
+        elif parsed_docstring.long_description:
+            info.description = parsed_docstring.long_description.strip()
+        elif parsed_docstring.short_description:
+            info.description = parsed_docstring.short_description.strip()
+
+        # Add examples if available
+        if parsed_docstring.examples:
+            examples = '\n\n'.join([(example.description or '') for example in parsed_docstring.examples])
+            info.description = f'{info.description}\n\nExamples\n\n{examples}'
+
+    return info
+
+
+def parse_docstring(docstring: Optional[str], param_name: str) -> Dict[str, str]:
+    if not docstring:
+        return {}
+
+    parsed_docstring: Docstring = parse(docstring)
+
+    for param in parsed_docstring.params:
+        if param.arg_name == param_name:
+            schema: Dict[str, str] = {'description': param.description or ''}
+
+            if param.type_name:
+                schema['docstring_type'] = param.type_name
+
+            return schema
+
+    return {}

evalscope/api/tool/utils.py

@@ -0,0 +1,64 @@
+import json
+import yaml
+from typing import Any, Dict, List, Optional
+
+from evalscope.utils import get_logger
+from .tool_call import ToolCall, ToolFunction
+from .tool_info import ToolInfo
+
+logger = get_logger()
+
+
+def parse_tool_call(id: str, function: str, arguments: str, tools: Optional[List[ToolInfo]] = None) -> ToolCall:
+    """Parse a tool call from a JSON payload.
+
+    Note that this function doesn't know about internal tool names so the caller
+    should ammend the returned `ToolCall` by mapping the parsed `function` field from
+    from an internal name to an inspect tool name and fixing up the `ToolCall` object
+    as required to reflect this change.
+    """
+    error: Optional[str] = None
+    arguments_dict: Dict[str, Any] = {}
+
+    def report_parse_error(ex: Exception) -> None:
+        nonlocal error
+        error = tool_parse_error_message(arguments, ex)
+        logger.info(error)
+
+    # if the arguments is a dict, then handle it with a plain json.loads
+    arguments = arguments.strip()
+    if arguments.startswith('{'):
+        try:
+            arguments_dict = json.loads(arguments)
+        except json.JSONDecodeError as ex:
+            report_parse_error(ex)
+
+    # otherwise parse it as yaml (which will pickup unquoted strings, numbers, and true/false)
+    # and then create a dict that maps it to the first function argument
+    elif function and tools:
+        tool_info = next(
+            (tool for tool in tools if tool.name == function and len(tool.parameters.properties) > 0),
+            None,
+        )
+        if tool_info:
+            param_names = list(tool_info.parameters.properties.keys())
+            try:
+                value = yaml.safe_load(arguments)
+                arguments_dict[param_names[0]] = value
+            except yaml.error.YAMLError:
+                # If the yaml parser fails, we treat it as a string argument.
+                arguments_dict[param_names[0]] = arguments
+
+    # return ToolCall with error payload
+    return ToolCall(
+        id=id,
+        function=ToolFunction(
+            name=function,
+            arguments=arguments_dict,
+        ),
+        parse_error=error,
+    )
+
+
+def tool_parse_error_message(arguments: str, ex: Exception) -> str:
+    return f'Error parsing the following tool call arguments:\n\n{arguments}\n\nError details: {ex}'

evalscope/app/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .app import create_app
+    from .arguments import add_argument
+
+else:
+    _import_structure = {
+        'app': [
+            'create_app',
+        ],
+        'arguments': [
+            'add_argument',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/app/app.py

@@ -0,0 +1,38 @@
+"""
+Main application module for the Evalscope dashboard.
+"""
+import argparse
+
+from evalscope.utils.logger import configure_logging
+from .arguments import add_argument
+from .ui import create_app_ui
+from .utils.env_utils import setup_env
+
+
+def create_app(args: argparse.Namespace):
+    """
+    Create and launch the Evalscope dashboard application.
+
+    Args:
+        args: Command line arguments.
+    """
+    configure_logging(debug=args.debug)
+
+    setup_env(args)
+
+    demo = create_app_ui(args)
+
+    demo.launch(
+        share=args.share,
+        server_name=args.server_name,
+        server_port=args.server_port,
+        debug=args.debug,
+        allowed_paths=args.allowed_paths,
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    add_argument(parser)
+    args = parser.parse_args()
+    create_app(args)

evalscope/app/arguments.py

@@ -0,0 +1,11 @@
+import argparse
+
+
+def add_argument(parser: argparse.ArgumentParser):
+    parser.add_argument('--share', action='store_true', help='Share the app.')
+    parser.add_argument('--server-name', type=str, default='0.0.0.0', help='The server name.')
+    parser.add_argument('--server-port', type=int, default=None, help='The server port.')
+    parser.add_argument('--debug', action='store_true', help='Debug the app.')
+    parser.add_argument('--lang', type=str, default='zh', help='The locale.', choices=['zh', 'en'])
+    parser.add_argument('--outputs', type=str, default='./outputs', help='The outputs dir.')
+    parser.add_argument('--allowed-paths', nargs='+', default=['/'], help='The outputs dir.')

evalscope/app/constants.py

@@ -0,0 +1,22 @@
+PLOTLY_THEME = 'plotly_dark'
+REPORT_TOKEN = '@@'
+MODEL_TOKEN = '::'
+DATASET_TOKEN = ', '
+DEFAULT_BAR_WIDTH = 0.2
+LATEX_DELIMITERS = [{
+    'left': '$$',
+    'right': '$$',
+    'display': True
+}, {
+    'left': '$',
+    'right': '$',
+    'display': False
+}, {
+    'left': '\\(',
+    'right': '\\)',
+    'display': False
+}, {
+    'left': '\\[',
+    'right': '\\]',
+    'display': True
+}]

evalscope/app/ui/__init__.py

@@ -0,0 +1,20 @@
+"""
+UI components for the Evalscope dashboard.
+"""
+from .app_ui import create_app_ui
+from .multi_model import MultiModelComponents, create_multi_model_tab
+from .sidebar import SidebarComponents, create_sidebar
+from .single_model import SingleModelComponents, create_single_model_tab
+from .visualization import VisualizationComponents, create_visualization
+
+__all__ = [
+    'create_app_ui',
+    'SidebarComponents',
+    'create_sidebar',
+    'VisualizationComponents',
+    'create_visualization',
+    'SingleModelComponents',
+    'create_single_model_tab',
+    'MultiModelComponents',
+    'create_multi_model_tab',
+]

evalscope/app/ui/app_ui.py

@@ -0,0 +1,53 @@
+"""
+Main UI application for the Evalscope dashboard.
+"""
+import argparse
+import gradio as gr
+
+from evalscope.version import __version__
+from ..utils.localization import get_app_locale
+from .sidebar import create_sidebar
+from .visualization import create_visualization
+
+
+def create_app_ui(args: argparse.Namespace):
+    lang = args.lang
+    locale_dict = get_app_locale(lang)
+
+    with gr.Blocks(title='Evalscope Dashboard') as demo:
+        gr.HTML(f'<h1 style="text-align: left;">{locale_dict["title"]} (v{__version__})</h1>')
+        with gr.Row():
+            with gr.Column(scale=0, min_width=35):
+                toggle_btn = gr.Button('<')
+            with gr.Column(scale=1):
+                gr.HTML(f'<h3 style="text-align: left;">{locale_dict["star_beggar"]}</h3>')
+
+        with gr.Row():
+            with gr.Column(scale=1) as sidebar_column:
+                sidebar_visible = gr.State(True)
+                sidebar = create_sidebar(args.outputs, lang)
+
+            with gr.Column(scale=5):
+                visualization = create_visualization(sidebar, lang)
+
+        @sidebar.load_btn.click(
+            inputs=[sidebar.reports_dropdown],
+            outputs=[visualization.single_model.report_name, visualization.multi_model.multi_report_name]
+        )
+        def update_displays(reports_dropdown):
+            if not reports_dropdown:
+                gr.Warning(locale_dict['note'], duration=3)
+                return gr.skip()
+
+            return (
+                gr.update(choices=reports_dropdown, value=reports_dropdown[0]),  # update single model dropdown
+                gr.update(choices=reports_dropdown, value=reports_dropdown)  # update multi model dropdown
+            )
+
+        @toggle_btn.click(inputs=[sidebar_visible], outputs=[sidebar_column, sidebar_visible, toggle_btn])
+        def toggle_sidebar(visible):
+            new_visible = not visible
+            text = '<' if new_visible else '>'
+            return gr.update(visible=new_visible), new_visible, gr.update(value=text)
+
+    return demo