agentscope-runtime 0.2.0b2__py3-none-any.whl → 1.0.0__py3-none-any.whl

agentscope_runtime/tools/generations/text_to_video.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+# pylint:disable=abstract-method, deprecated-module, wrong-import-order
+# pylint:disable=no-else-break, too-many-branches
+
+import asyncio
+import os
+import time
+import uuid
+from http import HTTPStatus
+from typing import Any, Optional
+
+from dashscope.aigc.video_synthesis import AioVideoSynthesis
+from mcp.server.fastmcp import Context
+from pydantic import BaseModel, Field
+
+from ..base import Tool
+from ..utils.api_key_util import get_api_key, ApiNames
+from ...engine.tracing import trace, TracingUtil
+
+
+class TextToVideoInput(BaseModel):
+    """
+    Text to video generation input model
+    """
+
+    prompt: str = Field(
+        ...,
+        description="正向提示词，用来描述生成视频中期望包含的元素和视觉特点, 超过800个字符自动截断",
+    )
+    negative_prompt: Optional[str] = Field(
+        default=None,
+        description="反向提示词，用来描述不希望在视频画面中看到的内容，可以对视频画面进行限制，超过500个字符自动截断",
+    )
+    size: Optional[str] = Field(
+        default=None,
+        description="视频分辨率，默认不设置",
+    )
+    duration: Optional[int] = Field(
+        default=None,
+        description="视频生成时长，单位为秒",
+    )
+    prompt_extend: Optional[bool] = Field(
+        default=None,
+        description="是否开启prompt智能改写，开启后使用大模型对输入prompt进行智能改写",
+    )
+    watermark: Optional[bool] = Field(
+        default=None,
+        description="是否添加水印，默认不设置",
+    )
+    ctx: Optional[Context] = Field(
+        default=None,
+        description="HTTP request context containing headers for mcp only, "
+        "don't generate it",
+    )
+
+
+class TextToVideoOutput(BaseModel):
+    """
+    Text to video generation output model
+    """
+
+    video_url: str = Field(
+        title="Video URL",
+        description="输出的视频url",
+    )
+    request_id: Optional[str] = Field(
+        default=None,
+        title="Request ID",
+        description="请求ID",
+    )
+
+
+class TextToVideo(Tool[TextToVideoInput, TextToVideoOutput]):
+    """
+    Text to video generation service that converts text into videos
+    using DashScope's VideoSynthesis API.
+    """
+
+    name: str = "modelstudio_text_to_video"
+    description: str = (
+        "通义万相-文生视频模型可根据文本生成5秒无声视频，支持 480P、720P、1080P 多种分辨率档位，"
+        "并在各档位下提供多个具体尺寸选项，以适配不同业务场景。"
+    )
+
+    @trace(trace_type="AIGC", trace_name="text_to_video")
+    async def arun(
+        self,
+        args: TextToVideoInput,
+        **kwargs: Any,
+    ) -> TextToVideoOutput:
+        """
+        Generate video from text prompt using DashScope VideoSynthesis
+
+        This method wraps DashScope's VideoSynthesis service to generate videos
+        based on text descriptions. It uses async call pattern for better
+        performance and supports polling for task completion.
+
+        Args:
+            args: TextToVideoInput containing optional parameters
+            **kwargs: Additional keyword arguments including:
+                - request_id: Optional request ID for tracking
+                - model_name: Model name to use (defaults to wan2.2-t2v-plus)
+                - api_key: DashScope API key for authentication
+
+        Returns:
+            TextToVideoOutput containing the generated video URL and request ID
+
+        Raises:
+            ValueError: If DASHSCOPE_API_KEY is not set or invalid
+            TimeoutError: If video generation takes too long
+            RuntimeError: If video generation fails
+        """
+        trace_event = kwargs.pop("trace_event", None)
+        request_id = TracingUtil.get_request_id()
+
+        try:
+            api_key = get_api_key(ApiNames.dashscope_api_key, **kwargs)
+        except AssertionError as e:
+            raise ValueError("Please set valid DASHSCOPE_API_KEY!") from e
+
+        model_name = kwargs.get(
+            "model_name",
+            os.getenv("TEXT_TO_VIDEO_MODEL_NAME", "wan2.2-t2v-plus"),
+        )
+
+        parameters = {}
+        if args.prompt_extend is not None:
+            parameters["prompt_extend"] = args.prompt_extend
+        if args.size:
+            parameters["size"] = args.size
+        if args.duration is not None:
+            parameters["duration"] = args.duration
+        if args.watermark is not None:
+            parameters["watermark"] = args.watermark
+
+        # Create AioVideoSynthesis instance
+        aio_video_synthesis = AioVideoSynthesis()
+
+        # Submit async task
+        task_response = await aio_video_synthesis.async_call(
+            model=model_name,
+            api_key=api_key,
+            prompt=args.prompt,
+            negative_prompt=args.negative_prompt,
+            **parameters,
+        )
+
+        if (
+            task_response.status_code != HTTPStatus.OK
+            or not task_response.output
+            or task_response.output.task_status in ["FAILED", "CANCELED"]
+        ):
+            raise RuntimeError(f"Failed to submit task: {task_response}")
+
+        # Poll for task completion using async methods
+        max_wait_time = 600  # 10 minutes timeout for video generation
+        poll_interval = 5  # 5 seconds polling interval
+        start_time = time.time()
+
+        while True:
+            # Wait before polling
+            await asyncio.sleep(poll_interval)
+
+            # Fetch task result using async method
+            res = await aio_video_synthesis.fetch(
+                api_key=api_key,
+                task=task_response,
+            )
+
+            if (
+                res.status_code != HTTPStatus.OK
+                or not res.output
+                or res.output.task_status in ["FAILED", "CANCELED"]
+            ):
+                raise RuntimeError(f"Failed to fetch result: {res}")
+
+            # Check task completion status
+            if res.status_code == HTTPStatus.OK:
+                if hasattr(res.output, "task_status"):
+                    if res.output.task_status == "SUCCEEDED":
+                        break
+                    elif res.output.task_status in ["FAILED", "CANCELED"]:
+                        raise RuntimeError(f"Failed to generate: {res}")
+                else:
+                    # If no task_status field, assume completed
+                    break
+
+            # Check timeout
+            if time.time() - start_time > max_wait_time:
+                raise TimeoutError(
+                    f"Video generation timeout after {max_wait_time}s",
+                )
+
+        # Handle request ID
+        if not request_id:
+            request_id = (
+                res.request_id if res.request_id else str(uuid.uuid4())
+            )
+
+        # Log trace event if provided
+        if trace_event:
+            trace_event.on_log(
+                "",
+                **{
+                    "step_suffix": "results",
+                    "payload": {
+                        "request_id": request_id,
+                        "text_to_video_result": res,
+                    },
+                },
+            )
+
+        # Extract video URL from response
+        if res.status_code == HTTPStatus.OK:
+            video_url = res.output.video_url
+            return TextToVideoOutput(
+                video_url=video_url,
+                request_id=request_id,
+            )
+        else:
+            raise RuntimeError(f"Failed to get video URL: {res.message}")

agentscope_runtime/tools/mcp_wrapper.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+# pylint:disable=too-many-branches, protected-access
+
+from typing import Any, Callable, Generic, Optional, Type, TypeVar
+
+from mcp.server.fastmcp import FastMCP
+from pydantic import BaseModel
+from pydantic_core import PydanticUndefined
+
+T = TypeVar("T", bound=BaseModel)
+U = TypeVar("U", bound=BaseModel)
+
+
+class MCPWrapper(Generic[T, U]):
+    """
+    A wrapper class for integrating zh with MCP (Model Context Protocol)
+    servers.
+
+    This class provides functionality to wrap tool classes and expose
+    them as MCP tools with proper type annotations and parameter handling.
+
+    Attributes:
+        mcp (FastMCP): The MCP server instance.
+        tool_class (Type): The tool class to wrap.
+    """
+
+    def __init__(self, mcp: FastMCP, tool_class: Type):
+        """Initialize the MCP wrapper.
+
+        Args:
+            mcp (FastMCP): The FastMCP server instance to register tools with.
+            tool_class (Type): The tool class to wrap as an MCP tool.
+        """
+        self.mcp = mcp
+        self.tool_class = tool_class
+
+    def wrap(
+        self,
+        name: str,
+        description: str,
+        method_name: str = "arun",
+    ) -> Callable[..., Any]:
+        """Wrap a tool as an MCP tool.
+
+        This method creates a tool instance and wraps it as an MCP tool
+        with proper parameter handling and type annotations derived from the
+        tool's input model.
+
+        Args:
+            name (str): The name for the tool instance.
+            description (str): The description for the tool instance.
+            method_name (str, optional): The method name to call on the
+                 tool.  Defaults to "arun".
+
+        Returns:
+            Callable[..., Any]: The wrapped tool function that can be called
+               by MCP.
+        """
+        tool = self.tool_class(name=name, description=description)
+
+        def create_decorated_async_function(
+            params: list[str],
+            func_name: str = "wrapped_tool",
+            decorator: Optional[Callable] = None,
+        ) -> Callable[..., Any]:
+            """Create a dynamically generated async function with proper
+            type annotations.
+
+            This internal function generates Python code for an async
+            function that matches the tool's input schema,
+            then executes it to create the actual function.
+
+            Args:
+                params (list[str]): List of parameter names from the
+                    tool's input model.
+                func_name (str, optional): Name for the generated function.
+                    Defaults to "wrapped_tool".
+                decorator (Optional[Callable], optional): Decorator to apply
+                    to the generated function. Defaults to None.
+
+            Returns:
+                Callable[..., Any]: The dynamically created async function.
+            """
+            # Generate parameter list with type annotations
+            params_types_with_default = []
+            params_types_without_default = []
+
+            for param in params:
+                # Get field information from Pydantic model
+                field_info = tool.input_type.model_fields[param]
+                # Extract type annotation
+                param_type = field_info.annotation
+
+                # Special handling for ctx parameter
+                if param == "ctx":
+                    # Keep ctx in function signature for FastMCP auto-injection
+                    # but use Context type directly
+                    param_line = f"{param}: Context = None"
+                    params_types_with_default.append(param_line)
+                    continue
+
+                # Convert type to string representation
+                if hasattr(param_type, "__name__"):
+                    type_str = param_type.__name__
+                    if type_str == "Optional":
+                        type_str = ""
+                else:
+                    type_str = str(param_type)
+
+                # Check for default value
+                if not field_info.is_required():
+                    # All non-required fields get None as default
+                    default_repr = (
+                        repr(field_info.default)
+                        if field_info.default is not PydanticUndefined
+                        else "None"
+                    )
+                    if type_str == "":
+                        param_line = f"{param} = {default_repr}"
+                    else:
+                        param_line = f"{param}: {type_str} = {default_repr}"
+                    params_types_with_default.append(param_line)
+                else:
+                    if type_str == "":
+                        param_line = f"{param}"
+                    else:
+                        param_line = f"{param}: {type_str}"
+                    params_types_without_default.append(param_line)
+
+            args_str_with_default = ", ".join(params_types_with_default)
+            args_str_without_default = ", ".join(params_types_without_default)
+
+            args_str = args_str_without_default
+            if len(args_str_with_default) > 0:
+                args_str += f", {args_str_with_default}"
+            # dynamic generate functions
+            code = f"""
+async def {func_name}({args_str}):
+    # Build kwargs dict dynamically,
+    # only including non-None values for optional params
+    kwargs_dict = {{}}
+    locals_dict = locals()
+    field_infos = tool.input_type.model_fields
+
+    for param_name in {params}:
+        param_value = locals_dict[param_name]
+        field_info = field_infos[param_name]
+
+        # Include required fields always, optional fields only if not None
+        if field_info.is_required():
+            kwargs_dict[param_name] = param_value
+        elif param_value is not None:
+            kwargs_dict[param_name] = param_value
+        # Skip optional fields with None values - let Pydantic use defaults
+
+    input_model = tool.input_type(**kwargs_dict)
+
+    # Set request_id from MCP context before calling tool method
+    if 'ctx' in locals_dict and locals_dict['ctx'] is not None:
+        request_id = get_mcp_dash_request_id(locals_dict['ctx'])
+        TracingUtil.set_request_id(request_id)
+
+    method = getattr(tool, method_name)
+    result = await method(input_model)
+    import json
+    return json.dumps(result.model_dump(), ensure_ascii=False)
+"""
+
+            # make namespace for tool
+            from mcp.server.fastmcp import Context
+            from .utils.mcp_util import get_mcp_dash_request_id
+            from ..engine.tracing.tracing_util import TracingUtil
+
+            namespace = {
+                "tool": tool,
+                "method_name": method_name,
+                "Context": Context,
+                "PydanticUndefined": PydanticUndefined,
+                "get_mcp_dash_request_id": get_mcp_dash_request_id,
+                "TracingUtil": TracingUtil,
+            }
+
+            # generate code generations
+            exec(code, namespace)
+
+            raw_function = namespace[func_name]
+
+            # apply decorator
+            if decorator:
+                return decorator(raw_function)
+            return raw_function
+
+        # define the mcp tool decorator
+        tool_decorator = self.mcp.tool(
+            name=tool.name,
+            description=tool.description,
+        )
+
+        # wrap the tool with mcp decorator with respect of the tool
+        # input type
+
+        wrapped_tool = create_decorated_async_function(
+            params=list(tool.input_type.model_fields.keys()),
+            decorator=tool_decorator,
+        )
+
+        # Update schema and remove ctx parameter
+        schema = tool.function_schema.parameters.model_dump()
+        if "properties" in schema and "ctx" in schema["properties"]:
+            schema["properties"].pop("ctx")
+        if "required" in schema and "ctx" in schema["required"]:
+            schema["required"].remove("ctx")
+
+        self.mcp._tool_manager._tools[tool.name].parameters.update(schema)
+        return wrapped_tool

agentscope_runtime/tools/realtime_clients/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from .azure_asr_client import AzureAsrCallbacks, AzureAsrClient
+from .modelstudio_asr_client import (
+    ModelstudioAsrClient,
+    ModelstudioAsrConfig,
+    ModelstudioAsrCallbacks,
+)
+from .azure_tts_client import AzureTtsClient, AzureTtsCallbacks
+from .modelstudio_tts_client import (
+    ModelstudioTtsClient,
+    ModelstudioTtsConfig,
+    ModelstudioTtsCallbacks,
+)

agentscope_runtime/tools/realtime_clients/asr_client.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from typing import Any
+
+from pydantic import BaseModel
+
+from .realtime_tool import (
+    RealtimeComponent,
+    RealtimeType,
+)
+from ...engine.schemas.realtime import AsrConfig
+
+
+class AsrClient(RealtimeComponent):
+    def __init__(self, config: AsrConfig, callbacks: BaseModel):
+        super().__init__(RealtimeType.ASR, config, callbacks)
+
+    def start(self, **kwargs: Any) -> None:
+        pass
+
+    def stop(self, **kwargs: Any) -> None:
+        pass
+
+    def close(self, **kwargs: Any) -> None:
+        pass
+
+    def send_audio_data(self, data: bytes) -> None:
+        pass

agentscope_runtime/tools/realtime_clients/azure_asr_client.py

@@ -0,0 +1,195 @@
+# -*- coding: utf-8 -*-
+# pylint:disable=logging-not-lazy, consider-using-f-string
+
+import json
+import logging
+import os
+from typing import Optional, Callable, Any
+
+import azure.cognitiveservices.speech as speech_sdk
+from azure.cognitiveservices.speech import (
+    SessionEventArgs,
+    SpeechRecognitionEventArgs,
+    SpeechRecognitionCanceledEventArgs,
+)
+from pydantic import BaseModel
+
+from .asr_client import AsrClient
+from .realtime_tool import (
+    RealtimeState,
+)
+from ...engine.schemas.realtime import AzureAsrConfig
+
+logger = logging.getLogger(__name__)
+
+
+class AzureAsrCallbacks(BaseModel):
+    on_started: Optional[Callable] = None
+    on_stopped: Optional[Callable] = None
+    on_canceled: Optional[Callable] = None
+    on_event: Optional[Callable] = None
+
+
+class AzureAsrClient(AsrClient):
+    def __init__(
+        self,
+        config: AzureAsrConfig,
+        callbacks: AzureAsrCallbacks,
+    ):
+        super().__init__(config, callbacks)
+        self.asr_request_id = None
+        self.is_first_audio_data = True
+
+        stream_format = speech_sdk.audio.AudioStreamFormat(
+            samples_per_second=config.sample_rate,
+            bits_per_sample=config.bits_per_sample,
+            channels=config.nb_channels,
+        )
+
+        speech_config = speech_sdk.SpeechConfig(
+            subscription=config.key if config.key else os.getenv("AZURE_KEY"),
+            region=(
+                config.region if config.region else os.getenv("AZURE_REGION")
+            ),
+            speech_recognition_language=config.language,
+        )
+
+        if config.initial_silence_timeout is not None:
+            speech_config.set_property(
+                speech_sdk.PropertyId.SpeechServiceConnection_InitialSilenceTimeoutMs,  # noqa
+                str(config.initial_silence_timeout),
+            )
+
+        if (
+            config.max_end_silence is not None
+            or config.fast_vad_min_duration is not None
+        ):
+            speech_config.set_property(
+                speech_sdk.PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs,  # noqa
+                str(
+                    (
+                        config.fast_vad_min_duration
+                        if config.fast_vad_min_duration is not None
+                        else config.max_end_silence
+                    ),
+                ),
+            )
+
+        self.push_stream = speech_sdk.audio.PushAudioInputStream(
+            stream_format=stream_format,
+        )
+        audio_config = speech_sdk.audio.AudioConfig(stream=self.push_stream)
+
+        self.recognizer = speech_sdk.SpeechRecognizer(
+            speech_config=speech_config,
+            audio_config=audio_config,
+        )
+
+        self.recognizer.session_started.connect(self.on_session_started)
+        self.recognizer.session_stopped.connect(self.on_session_stopped)
+        self.recognizer.speech_start_detected.connect(self.on_speech_start)
+        self.recognizer.speech_end_detected.connect(self.on_speech_end)
+        self.recognizer.canceled.connect(self.on_canceled)
+        self.recognizer.recognizing.connect(self.on_recognizing)
+        self.recognizer.recognized.connect(self.on_recognized)
+
+        logger.info(
+            f"azure_asr_config: {json.dumps(self.config.model_dump())}",
+        )
+
+    def start(self, **kwargs: Any) -> None:
+        logger.info("asr_start: config=%s" % self.config)
+        self.recognizer.start_continuous_recognition()
+
+    def stop(self, **kwargs: Any) -> None:
+        # TODO(zhiyi): blocking
+        if self.state == RealtimeState.IDLE:
+            return
+
+        logger.info("asr_stop: asr_request_id=%s" % self.asr_request_id)
+
+        self.recognizer.stop_continuous_recognition()
+        logger.info("asr_stop 2: asr_request_id=%s" % self.asr_request_id)
+        self.push_stream.close()
+
+    def close(self, **kwargs: Any) -> None:
+        logger.info("asr_close: asr_request_id=%s" % self.asr_request_id)
+        self.push_stream.close()
+        self.recognizer.stop_continuous_recognition()
+
+    def send_audio_data(self, data: bytes) -> None:
+        # logger.info("send_audio_data: asr_request_id=%s"
+        # % self.asr_request_id)
+        if self.state == RealtimeState.IDLE:
+            logger.error(
+                "send_audio_data failed: asr_request_id=%s"
+                % self.asr_request_id,
+            )
+            return
+
+        self.push_stream.write(data)
+
+    def on_session_started(self, event: SessionEventArgs) -> None:
+        self.state = RealtimeState.RUNNING
+        self.asr_request_id = event.session_id
+        logger.info(
+            f"asr_on_started: asr_request_id={event.session_id},"
+            f" event={event}",
+        )
+        if self.callbacks and self.callbacks.on_started:
+            self.callbacks.on_started()
+
+    def on_session_stopped(self, event: SessionEventArgs) -> None:
+        self.state = RealtimeState.IDLE
+        logger.info(
+            f"asr_on_stopped: asr_request_id={event.session_id},"
+            f" event={event}",
+        )
+        if self.callbacks and self.callbacks.on_stopped:
+            self.callbacks.on_stopped()
+
+    def on_speech_start(self, event: SessionEventArgs) -> None:
+        self.state = RealtimeState.RUNNING
+        logger.info(f"asr_on_speech_start: asr_request_id={event.session_id}")
+
+    def on_speech_end(self, event: SessionEventArgs) -> None:
+        # self.state = RealtimeState.IDLE
+        logger.info(
+            f"asr_on_speech_end: asr_request_id={event.session_id},"
+            f" event={event}",
+        )
+
+    def on_canceled(self, event: SpeechRecognitionCanceledEventArgs) -> None:
+        # self.state = RealtimeState.IDLE
+        logger.warning(
+            f"asr_on_canceled: asr_request_id={event.session_id},"
+            f" event={event}",
+        )
+        details = event.cancellation_details
+        logger.info(
+            f"asr_cancellation_details: reason={details.reason},"
+            f" error_code={details.code},"
+            f" error_details={details.error_details}, ",
+        )
+        if self.callbacks and self.callbacks.on_canceled:
+            self.callbacks.on_canceled()
+
+    def on_recognizing(self, event: SpeechRecognitionEventArgs) -> None:
+        logger.info(f"asr_on_recognizing: {event}")
+        if event.result.reason == speech_sdk.ResultReason.RecognizingSpeech:
+            if (
+                self.callbacks
+                and self.callbacks.on_event
+                and event.result.text
+            ):
+                self.callbacks.on_event(False, event.result.text)
+
+    def on_recognized(self, event: SpeechRecognitionEventArgs) -> None:
+        logger.info(f"asr_on_recognized: {event}")
+        if event.result.reason == speech_sdk.ResultReason.RecognizedSpeech:
+            if (
+                self.callbacks
+                and self.callbacks.on_event
+                and event.result.text
+            ):
+                self.callbacks.on_event(True, event.result.text)