evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/api/model/model.py

@@ -0,0 +1,386 @@
+import abc
+from pydantic_core import to_jsonable_python
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Literal, Optional, Sequence, Union
+
+from evalscope.api.messages import ChatMessage, ChatMessageAssistant, ChatMessageSystem, ChatMessageUser
+from evalscope.api.registry import get_model_api
+from evalscope.api.tool import ToolChoice, ToolFunction, ToolInfo
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+from .generate_config import GenerateConfig
+from .model_output import ModelOutput
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class ModelAPI(abc.ABC):
+    """Model API provider."""
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        **kwargs
+    ) -> None:
+        """Create a model API provider.
+
+        Args:
+           model_name (str): Model name.
+           base_url (str | None): Alternate base URL for model.
+           api_key (str | None): API key for model.
+           api_key_vars (list[str]): Environment variables that
+              may contain keys for this provider (used for override)
+           config (GenerateConfig): Model configuration.
+        """
+        self.model_name = model_name
+        self.base_url = base_url
+        self.api_key = api_key
+        self.config = config
+
+    @abc.abstractmethod
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+        """Generate output from the model.
+
+        Args:
+          input (str | list[ChatMessage]): Chat message
+            input (if a `str` is passed it is converted
+            to a `ChatUserMessage`).
+          tools (list[ToolInfo]): Tools available for the
+            model to call.
+          tool_choice (ToolChoice): Directives to the model
+            as to which tools to prefer.
+          config (GenerateConfig): Model configuration.
+
+        Returns:
+           ModelOutput
+        """
+        ...
+
+    def batch_generate(
+        self,
+        inputs: List[List[ChatMessage]],
+        tools: List[List[ToolInfo]],
+        tool_choices: List[ToolChoice],
+        configs: List[GenerateConfig],
+    ) -> Generator[ModelOutput, None, None]:
+        """Default batch implementation using individual generate calls.
+
+        ModelAPI implementations can override this for optimized batch processing.
+
+        Args:
+          inputs: List of preprocessed chat message inputs.
+          tools: List of tools for each input.
+          tool_choices: List of tool choices for each input.
+          configs: List of configs for each input.
+
+        Returns:
+            Generator yielding ModelOutput for each input.
+        """
+        from concurrent.futures import ThreadPoolExecutor
+
+        def single_generate(args):
+            input_msgs, input_tools, tool_choice, config = args
+            return self.generate(input_msgs, input_tools, tool_choice, config)
+
+        with ThreadPoolExecutor(max_workers=self.config.batch_size) as executor:
+            futures = []
+            for input_msgs, input_tools, tool_choice, config in zip(inputs, tools, tool_choices, configs):
+                future = executor.submit(single_generate, (input_msgs, input_tools, tool_choice, config))
+                futures.append(future)
+
+            for future in futures:
+                yield future.result()
+
+    def supports_batch(self) -> bool:
+        """Whether this ModelAPI supports optimized batch processing."""
+        return False
+
+    def max_tokens(self) -> Optional[int]:
+        """Default max_tokens."""
+        return None
+
+    def max_tokens_for_config(self, config: GenerateConfig) -> Optional[int]:
+        """Default max_tokens for a given config.
+
+        Args:
+           config: Generation config.
+
+        Returns:
+           Default maximum tokens for specified configuration.
+        """
+        return None
+
+    def tools_required(self) -> bool:
+        """Any tool use in a message stream means that tools must be passed."""
+        return False
+
+    def tool_result_images(self) -> bool:
+        """Tool results can contain images"""
+        return False
+
+
+class Model:
+    """Model interface.
+
+    Use `get_model()` to get an instance of a model.
+    """
+
+    api: ModelAPI
+    """Model API."""
+
+    config: GenerateConfig
+    """Generation config."""
+
+    def __init__(self, api: ModelAPI, config: GenerateConfig, model_args: Dict[str, Any] = {}) -> None:
+        """Create a model.
+
+        Args:
+           api: Model API provider.
+           config: Model configuration.
+           model_args: Optional model args
+        """
+        self.api = api
+        self.config = config
+        self.model_args = model_args
+
+    @property
+    def name(self) -> str:
+        """Model name or path to model."""
+        return self.api.model_name
+
+    @property
+    def role(self) -> Optional[str]:
+        """Model role."""
+        return self._role
+
+    @role.setter
+    def role(self, role: str) -> None:
+        self._role = role
+
+    def __str__(self) -> str:
+        return f'Model(name={self.model_id}, role={self.role})'
+
+    def generate(
+        self,
+        input: Union[str, List[ChatMessage]],
+        tools: Optional[Sequence[ToolInfo]] = None,
+        tool_choice: Optional[ToolChoice] = None,
+        config: Optional[GenerateConfig] = None,
+    ) -> ModelOutput:
+        """Generate output from the model.
+
+        Args:
+          input: Chat message input (if a `str` is passed it is converted
+            to a `ChatMessageUser`).
+          tools: Tools available for the model to call.
+          tool_choice: Directives to the model as to which tools to prefer.
+          config: Model configuration.
+
+        Returns:
+           ModelOutput
+        """
+        processed_input, processed_tools, processed_tool_choice, processed_config = self._preprocess_input(
+            input, tools, tool_choice, config
+        )
+
+        # Call the model's generate method
+        output = self.api.generate(
+            input=processed_input,
+            tools=processed_tools,
+            tool_choice=processed_tool_choice,
+            config=processed_config,
+        )
+
+        # return output
+        return output
+
+    def batch_generate(
+        self,
+        inputs: List[List[ChatMessage]],
+        tools: List[List[ToolInfo]],
+        tool_choices: List[ToolChoice],
+        configs: List[GenerateConfig],
+    ) -> Generator[ModelOutput, None, None]:
+        """Generate output from the model for a batch of inputs.
+
+        Args:
+          inputs (List[List[ChatMessage]]): Batch of chat message inputs.
+          tools (List[List[ToolInfo]]): Batch of tools for each input.
+          tool_choices (List[ToolChoice]): Batch of tool choices for each input.
+          configs (List[GenerateConfig]): Batch of configs for each input.
+        """
+        preprocessed_data = []
+
+        for input_item, input_tools, input_tool_choice, input_config in zip(inputs, tools, tool_choices, configs):
+            processed_input, processed_tools, processed_tool_choice, processed_config = self._preprocess_input(
+                input=input_item, tools=input_tools, tool_choice=input_tool_choice, config=input_config
+            )
+            preprocessed_data.append((processed_input, processed_tools, processed_tool_choice, processed_config))
+
+        # check if ModelAPI supports batch processing
+        if self.api.supports_batch() and len(preprocessed_data) > 1:
+            # use the batch_generate method of the ModelAPI
+            inputs, tools, tool_choices, configs = zip(*preprocessed_data)
+            batch_results = self.api.batch_generate(
+                inputs=list(inputs), tools=list(tools), tool_choices=list(tool_choices), configs=list(configs)
+            )
+            for result in batch_results:
+                yield result
+        else:
+            # fall back to processing each input individually
+            for input_msgs, input_tools, tool_choice, config in preprocessed_data:
+                result = self.api.generate(input_msgs, input_tools, tool_choice, config)
+                yield result
+
+    def _preprocess_input(
+        self,
+        input: Union[str, List[ChatMessage]],
+        tools: Optional[Sequence[ToolInfo]] = None,
+        tool_choice: Optional[ToolChoice] = None,
+        config: Optional[GenerateConfig] = None,
+    ) -> tuple[List[ChatMessage], List[ToolInfo], ToolChoice, GenerateConfig]:
+        """pre process input for generate."""
+
+        # merge passed config
+        if config is not None:
+            config = self.config.merge(config)
+        else:
+            config = self.config.model_copy(deep=True)
+
+        # provide max_tokens from the model api if required
+        if config.max_tokens is None:
+            config.max_tokens = self.api.max_tokens_for_config(config)
+            if config.max_tokens is None:
+                config.max_tokens = self.api.max_tokens()
+
+        # normalize input to chat
+        if isinstance(input, str):
+            input = [ChatMessageUser(content=input)]
+
+        # handle tools and tool_choice
+        tool_choice = tool_choice if tool_choice is not None else 'auto'
+        tools_info = list(tools) if tools is not None else []
+
+        if isinstance(tool_choice, ToolFunction):
+            tools_info = [tool for tool in tools_info if tool.name == tool_choice.name]
+
+        if tool_choice == 'none' or len(tools_info) == 0:
+            if not self.api.tools_required():
+                tools_info = []
+            tool_choice = 'none'
+
+        return input, tools_info, tool_choice, config
+
+
+class ModelCache:
+    _models: Dict[str, 'Model'] = {}
+
+    @classmethod
+    def get(cls, key: str) -> Optional['Model']:
+        return cls._models.get(key, None)
+
+    @classmethod
+    def set(cls, key: str, model: 'Model') -> None:
+        cls._models[key] = model
+
+
+def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
+    """Get an instance of a model with the specified task configuration.
+
+    Args:
+        task_config (TaskConfig): Task configuration.
+
+    Returns:
+        Model: An instance of the model.
+    """
+    model = task_config.model
+    eval_type = task_config.eval_type
+    base_url = task_config.api_url
+    api_key = task_config.api_key
+    config = task_config.generation_config
+    model_args = task_config.model_args or {}
+
+    return get_model(
+        model=model, eval_type=eval_type, base_url=base_url, api_key=api_key, config=config, model_args=model_args
+    )
+
+
+@thread_safe
+def get_model(
+    model: Union[str, Model, ModelAPI],
+    eval_type: str,
+    base_url: Optional[str] = None,
+    api_key: Optional[str] = None,
+    config: GenerateConfig = GenerateConfig(),
+    model_args: dict = {},
+    role: Optional[str] = None,
+    memoize: bool = True,
+) -> Model:
+    """Get an instance of a model.
+
+    Calls to get_model() are memoized (i.e. a call with the same arguments
+    will return an existing instance of the model rather than creating a
+    new one). You can disable this with `memoize=False`.
+
+    Args:
+        task_config (TaskConfig): Task configuration.
+        memoize (bool): Whether to memoize the model instance.
+
+    Returns:
+        Model instance.
+
+    """
+
+    # start with seeing if a model was passed
+    if isinstance(model, Model):
+        return model
+
+    if isinstance(model, ModelAPI):
+        return Model(model, config, model_args)
+
+    # see if we can return a memoized model instance
+    # (exclude mockllm since custom_outputs is an infinite generator)
+    model_cache_key: str = ''
+    if eval_type.startswith('mock_llm'):
+        memoize = False
+    if memoize:
+        model_cache_key = (
+            model + str(role) + config.model_dump_json(exclude_none=True) + str(base_url) + str(api_key)
+            + str(to_jsonable_python(model_args, fallback=lambda _: None))
+        )
+        cached = ModelCache.get(model_cache_key)
+        if cached is not None:
+            return cached
+
+    logger.info(
+        f'Creating model {model} with eval_type={eval_type} '
+        f'base_url={base_url}, api_key={api_key}, config={config}, model_args={model_args}'
+    )
+
+    # find a matching model type
+    modelapi_type = get_model_api(eval_type)
+
+    modelapi_instance = modelapi_type(
+        model_name=model,
+        base_url=base_url,
+        api_key=api_key,
+        config=config,
+        **model_args,
+    )
+    m = Model(modelapi_instance, config, model_args)
+    if role is not None:
+        m.role = role
+    if memoize:
+        ModelCache.set(model_cache_key, m)
+    return m

evalscope/api/model/model_output.py

@@ -0,0 +1,285 @@
+import uuid
+from pydantic import BaseModel, Field, JsonValue, model_validator
+from typing import Any, Dict, List, Literal, Optional, Type, Union
+
+from evalscope.api.messages import ChatMessageAssistant, Content
+from evalscope.api.tool import ToolCall, ToolFunction
+
+
+class ModelUsage(BaseModel):
+    """Token usage for completion."""
+
+    input_tokens: int = Field(default=0)
+    """Total input tokens used."""
+
+    output_tokens: int = Field(default=0)
+    """Total output tokens used."""
+
+    total_tokens: int = Field(default=0)
+    """Total tokens used."""
+
+    input_tokens_cache_write: Optional[int] = Field(default=None)
+    """Number of tokens written to the cache."""
+
+    input_tokens_cache_read: Optional[int] = Field(default=None)
+    """Number of tokens retrieved from the cache."""
+
+    reasoning_tokens: Optional[int] = Field(default=None)
+    """Number of tokens used for reasoning."""
+
+    def __add__(self, other: 'ModelUsage') -> 'ModelUsage':
+
+        def optional_sum(a: Optional[int], b: Optional[int]) -> Optional[int]:
+            if a is not None and b is not None:
+                return a + b
+            if a is not None:
+                return a
+            if b is not None:
+                return b
+            return None
+
+        return ModelUsage(
+            input_tokens=self.input_tokens + other.input_tokens,
+            output_tokens=self.output_tokens + other.output_tokens,
+            total_tokens=self.total_tokens + other.total_tokens,
+            input_tokens_cache_write=optional_sum(self.input_tokens_cache_write, other.input_tokens_cache_write),
+            input_tokens_cache_read=optional_sum(self.input_tokens_cache_read, other.input_tokens_cache_read),
+            reasoning_tokens=optional_sum(self.reasoning_tokens, other.reasoning_tokens),
+        )
+
+
+StopReason = Literal[
+    'stop',
+    'max_tokens',
+    'model_length',
+    'tool_calls',
+    'content_filter',
+    'unknown',
+]
+"""Reason that the model stopped or failed to generate."""
+
+
+class TopLogprob(BaseModel):
+    """List of the most likely tokens and their log probability, at this token position."""
+
+    token: str
+    """The top-kth token represented as a string."""
+
+    logprob: float
+    """The log probability value of the model for the top-kth token."""
+
+    bytes: Optional[List[int]] = Field(default=None)
+    """The top-kth token represented as a byte array (a list of integers)."""
+
+
+class Logprob(BaseModel):
+    """Log probability for a token."""
+
+    token: str
+    """The predicted token represented as a string."""
+
+    logprob: float
+    """The log probability value of the model for the predicted token."""
+
+    bytes: Optional[List[int]] = Field(default=None)
+    """The predicted token represented as a byte array (a list of integers)."""
+
+    top_logprobs: Optional[List[TopLogprob]] = Field(default=None)
+    """If the `top_logprobs` argument is greater than 0, this will contain an ordered list of the top K most likely tokens and their log probabilities."""  # noqa: E501
+
+
+class Logprobs(BaseModel):
+    """Log probability information for a completion choice."""
+
+    content: List[Logprob]
+    """a (num_generated_tokens,) length list containing the individual log probabilities for each generated token."""
+
+
+class ChatCompletionChoice(BaseModel):
+    """Choice generated for completion."""
+
+    message: ChatMessageAssistant
+    """Assistant message."""
+
+    stop_reason: StopReason = Field(default='unknown')
+    """Reason that the model stopped generating."""
+
+    logprobs: Optional[Logprobs] = Field(default=None)
+    """Logprobs."""
+
+    @model_validator(mode='before')
+    @classmethod
+    def migrate_stop_reason(cls: Type['ChatCompletionChoice'], values: Dict[str, Any]) -> Dict[str, Any]:
+        if 'stop_reason' in values:
+            stop_reason = values['stop_reason']
+            if stop_reason == 'length':
+                values['stop_reason'] = 'max_tokens'
+
+        return values
+
+    @classmethod
+    def from_content(cls, content: Union[str, List[Content]]) -> 'ChatCompletionChoice':
+        """Create a ChatCompletionChoice from content string."""
+        return cls(
+            message=ChatMessageAssistant(content=content),
+            stop_reason='stop',
+        )
+
+
+class ModelOutput(BaseModel):
+    """Output from model generation."""
+
+    model: str = Field(default_factory=str)
+    """Model used for generation."""
+
+    choices: List[ChatCompletionChoice] = Field(default=[])
+    """Completion choices."""
+
+    usage: Optional[ModelUsage] = Field(default=None)
+    """Model token usage"""
+
+    time: Optional[float] = Field(default=None)
+    """Time elapsed (in seconds) for call to generate."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional metadata associated with model output."""
+
+    error: Optional[str] = Field(default=None)
+    """Error message in the case of content moderation refusals."""
+
+    @property
+    def empty(self) -> bool:
+        return len(self.choices) == 0
+
+    @property
+    def stop_reason(self) -> StopReason:
+        """First message stop reason."""
+        return self.choices[0].stop_reason
+
+    @property
+    def message(self) -> ChatMessageAssistant:
+        """First message choice."""
+        return self.choices[0].message
+
+    @property
+    def completion(self) -> str:
+        """Text of first message choice text."""
+        if len(self.choices) > 0:
+            return self.choices[0].message.text
+        else:
+            return '\n'.join(choice.message.text for choice in self.choices)
+
+    @completion.setter
+    def completion(self, completion: str) -> None:
+        """Set the text of the first message choice.
+
+        Args:
+          completion (str): Text for first message.
+        """
+        if len(self.choices) > 0:
+            self.choices[0].message.text = completion
+        else:
+            self.choices.append(
+                ChatCompletionChoice(
+                    message=ChatMessageAssistant(content=completion, model=self.model),
+                    stop_reason='stop',
+                )
+            )
+
+    @property
+    def completions(self) -> List[str]:
+        """List of all message choices text."""
+        return [choice.message.text for choice in self.choices]
+
+    @staticmethod
+    def from_content(
+        model: str,
+        content: Union[str, List[Content]],
+        stop_reason: StopReason = 'stop',
+        error: Optional[str] = None,
+    ) -> 'ModelOutput':
+        """Create ModelOutput from simple text content.
+
+        Args:
+           model: Model name.
+           content: Text content from generation.
+           stop_reason: Stop reason for generation.
+           error: Error message.
+        """
+        return ModelOutput(
+            model=model,
+            choices=[
+                ChatCompletionChoice(
+                    message=ChatMessageAssistant(content=content, model=model, source='generate'),
+                    stop_reason=stop_reason,
+                )
+            ],
+            error=error,
+        )
+
+    @staticmethod
+    def for_tool_call(
+        model: str,
+        tool_name: str,
+        tool_arguments: Dict[str, Any],
+        internal: Optional[JsonValue] = None,
+        tool_call_id: Optional[str] = None,
+        content: Optional[str] = None,
+    ) -> 'ModelOutput':
+        """
+        Returns a ModelOutput for requesting a tool call.
+
+        Args:
+            model: model name
+            tool_name: The name of the tool.
+            internal: The model's internal info for the tool (if any).
+            tool_arguments: The arguments passed to the tool.
+            tool_call_id: Optional ID for the tool call. Defaults to a random UUID.
+            content: Optional content to include in the message. Defaults to "tool call for tool {tool_name}".
+
+        Returns:
+            A ModelOutput corresponding to the tool call
+        """
+        if content is None:
+            content = f'tool call for tool {tool_name}'
+
+        if tool_call_id is None:
+            tool_call_id = f'for_tool_call_{uuid.uuid4()}'
+
+        return ModelOutput(
+            model=model,
+            choices=[
+                ChatCompletionChoice(
+                    message=ChatMessageAssistant(
+                        content=content,
+                        model=model,
+                        source='generate',
+                        tool_calls=[
+                            ToolCall(
+                                id=tool_call_id,
+                                internal=internal,
+                                function=ToolFunction(
+                                    name=tool_name,
+                                    arguments=tool_arguments,
+                                )
+                            )
+                        ],
+                    ),
+                    stop_reason='tool_calls',
+                )
+            ],
+        )
+
+
+def as_stop_reason(reason: Optional[str]) -> StopReason:
+    """Encode common reason strings into standard StopReason."""
+    if reason in ['stop', 'eos']:
+        return 'stop'
+    elif reason == 'length':
+        return 'max_tokens'
+    elif reason in ['tool_calls', 'function_call']:
+        return 'tool_calls'
+    elif reason in ['content_filter', 'model_length', 'max_tokens']:
+        return reason
+    else:
+        return 'unknown'