evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/api/model/model_output.py

@@ -0,0 +1,285 @@
+import uuid
+from pydantic import BaseModel, Field, JsonValue, model_validator
+from typing import Any, Dict, List, Literal, Optional, Type, Union
+
+from evalscope.api.messages import ChatMessageAssistant, Content
+from evalscope.api.tool import ToolCall, ToolFunction
+
+
+class ModelUsage(BaseModel):
+    """Token usage for completion."""
+
+    input_tokens: int = Field(default=0)
+    """Total input tokens used."""
+
+    output_tokens: int = Field(default=0)
+    """Total output tokens used."""
+
+    total_tokens: int = Field(default=0)
+    """Total tokens used."""
+
+    input_tokens_cache_write: Optional[int] = Field(default=None)
+    """Number of tokens written to the cache."""
+
+    input_tokens_cache_read: Optional[int] = Field(default=None)
+    """Number of tokens retrieved from the cache."""
+
+    reasoning_tokens: Optional[int] = Field(default=None)
+    """Number of tokens used for reasoning."""
+
+    def __add__(self, other: 'ModelUsage') -> 'ModelUsage':
+
+        def optional_sum(a: Optional[int], b: Optional[int]) -> Optional[int]:
+            if a is not None and b is not None:
+                return a + b
+            if a is not None:
+                return a
+            if b is not None:
+                return b
+            return None
+
+        return ModelUsage(
+            input_tokens=self.input_tokens + other.input_tokens,
+            output_tokens=self.output_tokens + other.output_tokens,
+            total_tokens=self.total_tokens + other.total_tokens,
+            input_tokens_cache_write=optional_sum(self.input_tokens_cache_write, other.input_tokens_cache_write),
+            input_tokens_cache_read=optional_sum(self.input_tokens_cache_read, other.input_tokens_cache_read),
+            reasoning_tokens=optional_sum(self.reasoning_tokens, other.reasoning_tokens),
+        )
+
+
+StopReason = Literal[
+    'stop',
+    'max_tokens',
+    'model_length',
+    'tool_calls',
+    'content_filter',
+    'unknown',
+]
+"""Reason that the model stopped or failed to generate."""
+
+
+class TopLogprob(BaseModel):
+    """List of the most likely tokens and their log probability, at this token position."""
+
+    token: str
+    """The top-kth token represented as a string."""
+
+    logprob: float
+    """The log probability value of the model for the top-kth token."""
+
+    bytes: Optional[List[int]] = Field(default=None)
+    """The top-kth token represented as a byte array (a list of integers)."""
+
+
+class Logprob(BaseModel):
+    """Log probability for a token."""
+
+    token: str
+    """The predicted token represented as a string."""
+
+    logprob: float
+    """The log probability value of the model for the predicted token."""
+
+    bytes: Optional[List[int]] = Field(default=None)
+    """The predicted token represented as a byte array (a list of integers)."""
+
+    top_logprobs: Optional[List[TopLogprob]] = Field(default=None)
+    """If the `top_logprobs` argument is greater than 0, this will contain an ordered list of the top K most likely tokens and their log probabilities."""  # noqa: E501
+
+
+class Logprobs(BaseModel):
+    """Log probability information for a completion choice."""
+
+    content: List[Logprob]
+    """a (num_generated_tokens,) length list containing the individual log probabilities for each generated token."""
+
+
+class ChatCompletionChoice(BaseModel):
+    """Choice generated for completion."""
+
+    message: ChatMessageAssistant
+    """Assistant message."""
+
+    stop_reason: StopReason = Field(default='unknown')
+    """Reason that the model stopped generating."""
+
+    logprobs: Optional[Logprobs] = Field(default=None)
+    """Logprobs."""
+
+    @model_validator(mode='before')
+    @classmethod
+    def migrate_stop_reason(cls: Type['ChatCompletionChoice'], values: Dict[str, Any]) -> Dict[str, Any]:
+        if 'stop_reason' in values:
+            stop_reason = values['stop_reason']
+            if stop_reason == 'length':
+                values['stop_reason'] = 'max_tokens'
+
+        return values
+
+    @classmethod
+    def from_content(cls, content: Union[str, List[Content]]) -> 'ChatCompletionChoice':
+        """Create a ChatCompletionChoice from content string."""
+        return cls(
+            message=ChatMessageAssistant(content=content),
+            stop_reason='stop',
+        )
+
+
+class ModelOutput(BaseModel):
+    """Output from model generation."""
+
+    model: str = Field(default_factory=str)
+    """Model used for generation."""
+
+    choices: List[ChatCompletionChoice] = Field(default=[])
+    """Completion choices."""
+
+    usage: Optional[ModelUsage] = Field(default=None)
+    """Model token usage"""
+
+    time: Optional[float] = Field(default=None)
+    """Time elapsed (in seconds) for call to generate."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional metadata associated with model output."""
+
+    error: Optional[str] = Field(default=None)
+    """Error message in the case of content moderation refusals."""
+
+    @property
+    def empty(self) -> bool:
+        return len(self.choices) == 0
+
+    @property
+    def stop_reason(self) -> StopReason:
+        """First message stop reason."""
+        return self.choices[0].stop_reason
+
+    @property
+    def message(self) -> ChatMessageAssistant:
+        """First message choice."""
+        return self.choices[0].message
+
+    @property
+    def completion(self) -> str:
+        """Text of first message choice text."""
+        if len(self.choices) > 0:
+            return self.choices[0].message.text
+        else:
+            return '\n'.join(choice.message.text for choice in self.choices)
+
+    @completion.setter
+    def completion(self, completion: str) -> None:
+        """Set the text of the first message choice.
+
+        Args:
+          completion (str): Text for first message.
+        """
+        if len(self.choices) > 0:
+            self.choices[0].message.text = completion
+        else:
+            self.choices.append(
+                ChatCompletionChoice(
+                    message=ChatMessageAssistant(content=completion, model=self.model),
+                    stop_reason='stop',
+                )
+            )
+
+    @property
+    def completions(self) -> List[str]:
+        """List of all message choices text."""
+        return [choice.message.text for choice in self.choices]
+
+    @staticmethod
+    def from_content(
+        model: str,
+        content: Union[str, List[Content]],
+        stop_reason: StopReason = 'stop',
+        error: Optional[str] = None,
+    ) -> 'ModelOutput':
+        """Create ModelOutput from simple text content.
+
+        Args:
+           model: Model name.
+           content: Text content from generation.
+           stop_reason: Stop reason for generation.
+           error: Error message.
+        """
+        return ModelOutput(
+            model=model,
+            choices=[
+                ChatCompletionChoice(
+                    message=ChatMessageAssistant(content=content, model=model, source='generate'),
+                    stop_reason=stop_reason,
+                )
+            ],
+            error=error,
+        )
+
+    @staticmethod
+    def for_tool_call(
+        model: str,
+        tool_name: str,
+        tool_arguments: Dict[str, Any],
+        internal: Optional[JsonValue] = None,
+        tool_call_id: Optional[str] = None,
+        content: Optional[str] = None,
+    ) -> 'ModelOutput':
+        """
+        Returns a ModelOutput for requesting a tool call.
+
+        Args:
+            model: model name
+            tool_name: The name of the tool.
+            internal: The model's internal info for the tool (if any).
+            tool_arguments: The arguments passed to the tool.
+            tool_call_id: Optional ID for the tool call. Defaults to a random UUID.
+            content: Optional content to include in the message. Defaults to "tool call for tool {tool_name}".
+
+        Returns:
+            A ModelOutput corresponding to the tool call
+        """
+        if content is None:
+            content = f'tool call for tool {tool_name}'
+
+        if tool_call_id is None:
+            tool_call_id = f'for_tool_call_{uuid.uuid4()}'
+
+        return ModelOutput(
+            model=model,
+            choices=[
+                ChatCompletionChoice(
+                    message=ChatMessageAssistant(
+                        content=content,
+                        model=model,
+                        source='generate',
+                        tool_calls=[
+                            ToolCall(
+                                id=tool_call_id,
+                                internal=internal,
+                                function=ToolFunction(
+                                    name=tool_name,
+                                    arguments=tool_arguments,
+                                )
+                            )
+                        ],
+                    ),
+                    stop_reason='tool_calls',
+                )
+            ],
+        )
+
+
+def as_stop_reason(reason: Optional[str]) -> StopReason:
+    """Encode common reason strings into standard StopReason."""
+    if reason in ['stop', 'eos']:
+        return 'stop'
+    elif reason == 'length':
+        return 'max_tokens'
+    elif reason in ['tool_calls', 'function_call']:
+        return 'tool_calls'
+    elif reason in ['content_filter', 'model_length', 'max_tokens']:
+        return reason
+    else:
+        return 'unknown'

evalscope/api/registry.py

@@ -0,0 +1,182 @@
+import copy
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Type, Union
+
+if TYPE_CHECKING:
+    from evalscope.api.benchmark import BenchmarkMeta, DataAdapter
+    from evalscope.api.filter import Filter
+    from evalscope.api.metric import Aggregator, Metric
+    from evalscope.api.model.model import ModelAPI
+    from evalscope.config import TaskConfig
+
+# BEGIN: Registry for benchmarks
+# Registry for benchmarks, allowing dynamic registration and retrieval of benchmark metadata and data adapters.
+BENCHMARK_REGISTRY: Dict[str, 'BenchmarkMeta'] = {}
+
+
+def register_benchmark(metadata: 'BenchmarkMeta'):
+    """Register a benchmark with its metadata."""
+
+    def register_wrapper(data_adapter: Type['DataAdapter']):
+        if metadata.name in BENCHMARK_REGISTRY:
+            raise ValueError(f'Benchmark {metadata.name} already registered')
+        metadata.data_adapter = data_adapter
+        BENCHMARK_REGISTRY[metadata.name] = metadata
+        return data_adapter
+
+    return register_wrapper
+
+
+def get_benchmark(name: str, config: Optional['TaskConfig'] = None) -> 'DataAdapter':
+    """
+    Retrieve a registered benchmark by name.
+
+    Args:
+        name (str): The name of the benchmark.
+        config (Optional['TaskConfig']): The task configuration.
+        dataset_args (Optional[dict]): The dataset-specific arguments.
+
+    """
+    # copy to avoid modifying the original metadata
+    metadata = copy.deepcopy(BENCHMARK_REGISTRY.get(name))
+    if not metadata:
+        raise ValueError(f'Benchmark {name} not found, available benchmarks: {list(sorted(BENCHMARK_REGISTRY.keys()))}')
+
+    # Update metadata with dataset-specific configuration
+    if config is not None:
+        metadata._update(config.dataset_args.get(name, {}))
+    # Return the data adapter initialized with the benchmark metadata
+    data_adapter_cls = metadata.data_adapter
+    return data_adapter_cls(benchmark_meta=metadata, task_config=config)
+
+
+# END: Registry for benchmarks
+
+# BEGIN: Registry for model APIs
+# Registry for model APIs, allowing dynamic registration and retrieval of model API classes.
+MODEL_APIS: Dict[str, Type['ModelAPI']] = {}
+
+
+def register_model_api(name: str):
+    """
+    Decorator to register a model API class with a given name.
+
+    :param name: The name of the model API.
+    """
+
+    def decorator(api_class: Type['ModelAPI']):
+        if name in MODEL_APIS:
+            raise ValueError(f"Model API '{name}' is already registered.")
+        MODEL_APIS[name] = api_class
+        return api_class
+
+    return decorator
+
+
+def get_model_api(name: str) -> Type['ModelAPI']:
+    """
+    Retrieve a registered model API class by name.
+
+    :param name: The name of the model API.
+    :return: The model API class.
+    """
+    if name not in MODEL_APIS:
+        raise ValueError(f"Model API '{name}' is not registered. Available model APIs: {list(MODEL_APIS.keys())}")
+
+    wrapped = MODEL_APIS[name]
+    if not isinstance(wrapped, type):
+        return wrapped()
+    else:
+        return wrapped
+
+
+# END: Registry for model APIs
+
+# BEGIN: Registry for metrics
+METRIC_REGISTRY: Dict[str, Type['Metric']] = {}
+
+
+def register_metric(name: str):
+
+    def decorate(fn):
+        if name in METRIC_REGISTRY:
+            raise ValueError(f"Metric named '{name}' conflicts with existing registered metric!")
+
+        METRIC_REGISTRY[name] = fn
+        return fn
+
+    return decorate
+
+
+def get_metric(name: str) -> Type['Metric']:
+    if name in METRIC_REGISTRY:
+        return METRIC_REGISTRY[name]
+    else:
+        raise ValueError(
+            f"Metric '{name}' not found in the registry. Available metrics: {list(METRIC_REGISTRY.keys())}"
+        )
+
+
+# END: Registry for metrics
+
+# BEGIN: Registry for filters
+
+FILTER_REGISTRY: Dict[str, Type['Filter']] = {}
+
+
+def register_filter(name):
+
+    def decorate(cls):
+        if name in FILTER_REGISTRY:
+            raise ValueError(f'Registering filter `{name}` that is already in Registry {FILTER_REGISTRY}')
+        FILTER_REGISTRY[name] = cls
+        return cls
+
+    return decorate
+
+
+def get_filter(filter_name: str) -> Type['Filter']:
+    if filter_name not in FILTER_REGISTRY:
+        raise KeyError(
+            f"Filter '{filter_name}' not found in the registry. Available filters: {list(FILTER_REGISTRY.keys())}"
+        )
+    return FILTER_REGISTRY[filter_name]
+
+
+# END: Registry for filters
+
+# BEGIN: Registry for aggregation functions
+AGGREGATION_REGISTRY: Dict[str, Type['Aggregator']] = {}
+
+
+def register_aggregation(name: str):
+    """
+    Decorator to register an aggregation function with a given name.
+
+    :param name: The name of the aggregation function.
+    """
+
+    def decorator(aggregation_fn: 'Aggregator'):
+        if name in AGGREGATION_REGISTRY:
+            raise ValueError(f"Aggregation function '{name}' is already registered.")
+        AGGREGATION_REGISTRY[name] = aggregation_fn
+        return aggregation_fn
+
+    return decorator
+
+
+def get_aggregation(name: str) -> Type['Aggregator']:
+    """
+    Retrieve a registered aggregation function by name.
+
+    :param name: The name of the aggregation function.
+    :return: The aggregation function.
+    """
+    if name not in AGGREGATION_REGISTRY:
+        raise ValueError(
+            f"Aggregation function '{name}' is not registered. "
+            f'Available aggregations: {list(AGGREGATION_REGISTRY.keys())}'
+        )
+    return AGGREGATION_REGISTRY[name]
+
+
+# END: Registry for aggregation functions

evalscope/api/tool/__init__.py

@@ -0,0 +1,3 @@
+from .tool_call import ToolCall, ToolCallContent, ToolCallError, ToolCallView, ToolChoice, ToolFunction
+from .tool_info import Tool, ToolDescription, ToolInfo, ToolParams, set_tool_description, tool_description
+from .utils import parse_tool_call, tool_parse_error_message

evalscope/api/tool/tool_call.py

@@ -0,0 +1,101 @@
+import json
+from pydantic import BaseModel, Field, JsonValue, field_validator
+from typing import Any, Callable, Dict, List, Literal, Optional, Union
+
+
+class ToolFunction(BaseModel):
+    """Indicate that a specific tool function should be called."""
+
+    name: str
+    """The name of the tool function to call."""
+
+    arguments: Dict[str, Any]
+    """The arguments of the tool function to call"""
+
+    @field_validator('arguments', mode='before')
+    @classmethod
+    def parse_arguments(cls, v):
+        if isinstance(v, str):
+            try:
+                v = json.loads(v)
+            except Exception as e:
+                raise ValueError(f'arguments field string is not valid JSON: {e}')
+        if not isinstance(v, dict):
+            raise ValueError('arguments must be a dict or a JSON string representing a dict')
+        return v
+
+
+class ToolCallContent(BaseModel):
+    """Content to include in tool call view."""
+
+    title: Optional[str] = Field(default=None)
+    """Optional (plain text) title for tool call content."""
+
+    format: Literal['text', 'markdown']
+    """Format (text or markdown)."""
+
+    content: str
+    """Text or markdown content."""
+
+
+class ToolCallView(BaseModel):
+    """Custom view of a tool call.
+
+    Both `context` and `call` are optional. If `call` is not specified
+    then the view will default to a syntax highlighted Python function call.
+    """
+
+    context: Optional[ToolCallContent] = Field(default=None)
+    """Context for the tool call (i.e. current tool state)."""
+
+    call: Optional[ToolCallContent] = Field(default=None)
+    """Custom representation of tool call."""
+
+
+class ToolCall(BaseModel):
+    id: str
+    """Unique identifier for tool call."""
+
+    function: ToolFunction
+    """Function to call."""
+
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+    parse_error: Optional[str] = Field(default=None)
+    """Error which occurred parsing tool call."""
+
+    view: Optional[ToolCallContent] = Field(default=None)
+    """Custom view of tool call input."""
+
+    type: Optional[str] = Field(default=None)
+    """Tool call type (deprecated)."""
+
+
+class ToolCallError(BaseModel):
+    """Error raised by a tool call."""
+
+    type: Literal[
+        'parsing',
+        'timeout',
+        'unicode_decode',
+        'permission',
+        'file_not_found',
+        'is_a_directory',
+        'limit',
+        'approval',
+        'unknown',
+    ]
+    """Error type."""
+
+    message: str
+    """Error message."""
+
+
+ToolChoice = Union[Literal['auto', 'any', 'none'], ToolFunction]
+"""Specify which tool to call.
+
+"auto" means the model decides; "any" means use at least one tool,
+"none" means never call a tool; ToolFunction instructs the model
+to call a specific function.
+"""