veadk-python 0.2.27__py3-none-any.whl

veadk/tools/builtin_tools/tts.py

@@ -0,0 +1,253 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import requests
+import json
+import base64
+import time
+import queue
+import threading
+import tempfile
+from typing import Dict, Any
+from google.adk.tools import ToolContext
+from veadk.config import getenv, settings
+from veadk.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def text_to_speech(text: str, tool_context: ToolContext) -> Dict[str, Any]:
+    """TTS provides users with the ability to convert text to speech, turning the text content of LLM into audio.
+    Use this tool when you need to convert text content into audible speech.
+    It transforms plain text into natural-sounding speech, as well as exporting the generated audio in pcm format.
+
+    Args:
+        text: The text to convert.
+
+    Returns:
+        A dict with the saved audio path.
+    """
+    url = "https://openspeech.bytedance.com/api/v3/tts/unidirectional"
+    temp_dir = getenv("TOOL_VESPEECH_AUDIO_OUTPUT_PATH", tempfile.gettempdir())
+
+    app_id = getenv("TOOL_VESPEECH_APP_ID")
+    speaker = getenv(
+        "TOOL_VESPEECH_SPEAKER", "zh_female_vv_uranus_bigtts"
+    )  # e.g. zh_female_vv_mars_bigtts
+    api_key = settings.tool.vespeech.api_key
+    if not all([app_id, api_key, speaker]):
+        return {
+            "error": (
+                "Tool text_to_speech execution failed. Missing required env vars: "
+                "TOOL_VESPEECH_APP_ID, TOOL_VESPEECH_API_KEY, TOOL_VESPEECH_SPEAKER"
+            )
+        }
+
+    headers = {
+        "X-Api-App-Id": app_id,
+        "X-Api-Key": api_key,
+        "X-Api-Resource-Id": "seed-tts-2.0",  # seed-tts-1.0 or seed-tts-2.0
+        "Content-Type": "application/json",
+        "Connection": "keep-alive",
+    }
+    additions = {
+        "explicit_language": "zh",
+        "disable_markdown_filter": True,
+        "enable_timestamp": True,
+    }
+    payload = {
+        "user": {"uid": tool_context._invocation_context.user_id},
+        "req_params": {
+            "text": text,
+            "speaker": speaker,
+            "audio_params": {
+                "format": "pcm",
+                "bit_rate": 16000,
+                "sample_rate": 24000,
+                "enable_timestamp": True,
+            },
+            "additions": json.dumps(additions),
+        },
+    }
+
+    session = requests.Session()
+    response = None
+
+    try:
+        logger.debug(f"Request TTS server with payload: {payload}.")
+        response = session.post(url, headers=headers, json=payload, stream=True)
+
+        os.makedirs(temp_dir, exist_ok=True)
+        with tempfile.NamedTemporaryFile(
+            suffix=".pcm", delete=False, dir=temp_dir
+        ) as tmp:
+            audio_save_path = tmp.name  # e.g. /tmp/tts_12345.pcm
+            logger.debug(f"Created temporary file: {audio_save_path}")
+
+        handle_server_response(response, audio_save_path)
+
+    except Exception as e:
+        logger.error(
+            f"Failed to convert text to speech: {e}Response content: {response}"
+        )
+        return {
+            "error": f"Tool text_to_speech execution failed. "
+            f"Response content: {response}"
+            f"Execution Error: {e}"
+        }
+    finally:
+        if response:
+            response.close()
+        session.close()
+    logger.debug("Finish convert text to speech")
+    return {"saved_audio_path": audio_save_path}
+
+
+def handle_server_response(
+    response: requests.models.Response, audio_save_path: str
+) -> None:
+    """
+    Handle the server response for TTS.
+
+    Args:
+        response: The server response as a requests.models.Response object.
+
+    Returns:
+        None
+    """
+
+    # audio data buffer
+    audio_data = bytearray()
+    # audio data queue for player thread
+    audio_queue = queue.Queue()
+    total_audio_size = 0
+
+    output_stream, player_thread = None, None
+    stop_event = threading.Event()
+    try:
+        from veadk.utils.audio_manager import (
+            AudioDeviceManager,
+            AudioConfig,
+            input_audio_config,
+            output_audio_config,
+        )
+
+        audio_device = AudioDeviceManager(
+            AudioConfig(**input_audio_config), AudioConfig(**output_audio_config)
+        )
+
+        # init output stream
+        output_stream = audio_device.open_output_stream()
+        player_thread = threading.Thread(
+            target=_audio_player_thread, args=(audio_queue, output_stream, stop_event)
+        )
+        player_thread.daemon = True
+        player_thread.start()
+    except Exception as e:
+        logger.error(f"Failed to initialize audio device: {e}")
+
+    try:
+        for chunk in response.iter_lines(decode_unicode=True):
+            if not chunk:
+                continue
+            data = json.loads(chunk)
+
+            if data.get("code", 0) == 0 and "data" in data and data["data"]:
+                chunk_audio = base64.b64decode(data["data"])
+                audio_size = len(chunk_audio)
+                total_audio_size += audio_size
+                audio_queue.put(chunk_audio)
+                audio_data.extend(chunk_audio)
+                continue
+            if data.get("code", 0) == 0 and "sentence" in data and data["sentence"]:
+                logger.debug(f"sentence_data: {data}")
+                continue
+            if data.get("code", 0) == 20000000:
+                logger.debug(
+                    f"successfully get audio data, total size: {total_audio_size / 1024:.2f} KB"
+                )
+                break
+            if data.get("code", 0) > 0:
+                logger.debug(f"tts response error:{data}")
+                break
+
+        # save audio data to file
+        save_output_to_file(audio_data, audio_save_path)
+    except Exception as e:
+        logger.error(f"handle tts failed: {e}, response: {response}")
+        raise
+    finally:
+        if output_stream:
+            audio_queue.join()
+            stop_event.set()
+            if player_thread and player_thread.is_alive():
+                player_thread.join()
+            output_stream.close()
+
+
+def _audio_player_thread(audio_queue, output_stream, stop_event):
+    """
+    Play audio data from queue.
+    Args:
+        audio_queue: The queue to store audio data.
+        output_stream: The output stream to play audio.
+        stop_event: The event to stop the thread.
+
+    Returns:
+
+    """
+    while not stop_event.is_set():
+        try:
+            # write audio data to output stream
+            audio_data = audio_queue.get(timeout=1.0)
+            if audio_data:
+                output_stream.write(audio_data)
+            audio_queue.task_done()
+        except queue.Empty:
+            # if queue is empty, sleep for a while
+            time.sleep(0.1)
+        except Exception as e:
+            logger.error(f"Failed to play audio data: {e}")
+            time.sleep(0.1)
+    logger.debug("audio player thread exited")
+
+
+def save_output_to_file(audio_data: bytearray, filename: str) -> None:
+    """
+    Save audio data to file.
+
+    Args:
+        audio_data: The audio data as bytes.
+        filename: The filename to save the audio data.
+
+    Returns:
+        None
+    """
+
+    if not audio_data:
+        logger.debug("No audio data to save.")
+        return
+    if not filename:
+        logger.debug("No filename to save audio data.")
+        return
+
+    try:
+        with open(filename, "wb") as f:
+            f.write(audio_data)
+            logger.debug(
+                f"Successfully save audio file to {filename},file size: {len(audio_data) / 1024:.2f} KB"
+            )
+    except IOError as e:
+        logger.error(f"Failed to save pcm file: {e}")

veadk/tools/builtin_tools/vesearch.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+
+from veadk.config import getenv, settings
+
+
+def vesearch(query: str) -> str:
+    """Search information from Internet, social media, news sites, etc.
+
+    Args:
+        query: The query string to search.
+
+    Returns:
+        Summarized search results.
+    """
+    api_key = settings.tool.vesearch.api_key
+    bot_id = str(getenv("TOOL_VESEARCH_ENDPOINT"))
+
+    if api_key == "":
+        return "Invoke `vesearch` failed. Please set TOOL_VESEARCH_API_KEY as your environment variable."
+    if bot_id == "":
+        return "Invoke `vesearch` failed. Please set TOOL_VESEARCH_ENDPOINT as your environment variable."
+
+    URL = "https://open.feedcoopapi.com/agent_api/agent/chat/completion"
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    data = {
+        "bot_id": bot_id,
+        "messages": [{"role": "user", "content": query}],
+    }
+
+    response = requests.post(URL, json=data, headers=headers)
+    if response.status_code == 200:
+        result = response.json()
+        return result["choices"][0]["message"]["content"]
+    else:
+        return response.text

veadk/tools/builtin_tools/video_generate.py

@@ -0,0 +1,363 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import time
+import traceback
+from typing import Dict, cast
+
+from google.adk.tools import ToolContext
+from opentelemetry import trace
+from opentelemetry.trace import Span
+from volcenginesdkarkruntime import Ark
+from volcenginesdkarkruntime.types.content_generation.create_task_content_param import (
+    CreateTaskContentParam,
+)
+
+from veadk.config import getenv, settings
+from veadk.consts import DEFAULT_VIDEO_MODEL_API_BASE, DEFAULT_VIDEO_MODEL_NAME
+from veadk.utils.logger import get_logger
+from veadk.version import VERSION
+
+logger = get_logger(__name__)
+
+client = Ark(
+    api_key=getenv(
+        "MODEL_VIDEO_API_KEY", getenv("MODEL_AGENT_API_KEY", settings.model.api_key)
+    ),
+    base_url=getenv("MODEL_VIDEO_API_BASE", DEFAULT_VIDEO_MODEL_API_BASE),
+)
+
+
+async def generate(prompt, first_frame_image=None, last_frame_image=None):
+    try:
+        if first_frame_image is None:
+            response = client.content_generation.tasks.create(
+                model=getenv("MODEL_VIDEO_NAME", DEFAULT_VIDEO_MODEL_NAME),
+                content=[
+                    {"type": "text", "text": prompt},
+                ],
+            )
+        elif last_frame_image is None:
+            response = client.content_generation.tasks.create(
+                model=getenv("MODEL_VIDEO_NAME", DEFAULT_VIDEO_MODEL_NAME),
+                content=cast(
+                    list[CreateTaskContentParam],  # avoid IDE warning
+                    [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": first_frame_image},
+                        },
+                    ],
+                ),
+            )
+        else:
+            response = client.content_generation.tasks.create(
+                model=getenv("MODEL_VIDEO_NAME", DEFAULT_VIDEO_MODEL_NAME),
+                content=[
+                    {"type": "text", "text": prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": first_frame_image},
+                        "role": "first_frame",
+                    },
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": last_frame_image},
+                        "role": "last_frame",
+                    },
+                ],
+            )
+    except:
+        traceback.print_exc()
+        raise
+    return response
+
+
+async def video_generate(
+    params: list, tool_context: ToolContext, batch_size: int = 10
+) -> Dict:
+    """
+    Generate videos in **batch** from text prompts, optionally guided by a first/last frame,
+    and fine-tuned via *model text commands* (a.k.a. `parameters` appended to the prompt).
+
+    This API creates video-generation tasks. Each item in `params` describes a single video.
+    The function submits all items in one call and returns task metadata for tracking.
+
+    Args:
+        params (list[dict]):
+            A list of video generation requests. Each item supports the fields below.
+        batch_size (int):
+            The number of videos to generate in a batch. Defaults to 10.
+
+            Required per item:
+                - video_name (str):
+                    Name/identifier of the output video file.
+
+                - prompt (str):
+                    Text describing the video to generate. Supports zh/EN.
+                    You may append **model text commands** after the prompt to control resolution,
+                    aspect ratio, duration, fps, watermark, seed, camera lock, etc.
+                    Format: `... --rs <resolution> --rt <ratio> --dur <seconds> --fps <fps> --wm <bool> --seed <int> --cf <bool>`
+                    Example:
+                        "小猫骑着滑板穿过公园。 --rs 720p --rt 16:9 --dur 5 --fps 24 --wm true --seed 11 --cf false"
+
+            Optional per item:
+                - first_frame (str | None):
+                    URL or Base64 string (data URL) for the **first frame** (role = `first_frame`).
+                    Use when you want the clip to start from a specific image.
+
+                - last_frame (str | None):
+                    URL or Base64 string (data URL) for the **last frame** (role = `last_frame`).
+                    Use when you want the clip to end on a specific image.
+
+            Notes on first/last frame:
+                * When both frames are provided, **match width/height** to avoid cropping; if they differ,
+                  the tail frame may be auto-cropped to fit.
+                * If you only need one guided frame, provide either `first_frame` or `last_frame` (not both).
+
+            Image input constraints (for first/last frame):
+                - Formats: jpeg, png, webp, bmp, tiff, gif
+                - Aspect ratio (宽:高): 0.4–2.5
+                - Width/Height (px): 300–6000
+                - Size: < 30 MB
+                - Base64 data URL example: `data:image/png;base64,<BASE64>`
+
+    Model text commands (append after the prompt; unsupported keys are ignored by some models):
+        --rs / --resolution <value>       Video resolution. Common values: 480p, 720p, 1080p.
+                                          Default depends on model (e.g., doubao-seedance-1-0-pro: 1080p,
+                                          some others default 720p).
+
+        --rt / --ratio <value>            Aspect ratio. Typical: 16:9 (default), 9:16, 4:3, 3:4, 1:1, 2:1, 21:9.
+                                          Some models support `keep_ratio` (keep source image ratio) or `adaptive`
+                                          (auto choose suitable ratio).
+
+        --dur / --duration <seconds>      Clip length in seconds. Seedance supports **3–12 s**;
+                                          Wan2.1 仅支持 5 s。Default varies by model.
+
+        --fps / --framespersecond <int>   Frame rate. Common: 16 or 24 (model-dependent; e.g., seaweed=24, wan2.1=16).
+
+        --wm / --watermark <true|false>   Whether to add watermark. Default: **false** (per doc).
+
+        --seed <int>                      Random seed in [-1, 2^32-1]. Default **-1** = auto seed.
+                                          Same seed may yield similar (not guaranteed identical) results across runs.
+
+        --cf / --camerafixed <true|false> Lock camera movement. Some models support this flag.
+                                          true: try to keep camera fixed; false: allow movement. Default: **false**.
+
+    Returns:
+        Dict:
+            API response containing task creation results for each input item. A typical shape is:
+            {
+                "status": "success",
+                "success_list": [{"video_name": "video_url"}],
+                "error_list": []
+            }
+
+    Constraints & Tips:
+        - Keep prompt concise and focused (建议 ≤ 500 字); too many details may distract the model.
+        - If using first/last frames, ensure their **aspect ratio matches** your chosen `--rt` to minimize cropping.
+        - If you must reproduce results, specify an explicit `--seed`.
+        - Unsupported parameters are ignored silently or may cause validation errors (model-specific).
+
+    Minimal examples:
+        1) Text-only batch of two 5-second clips at 720p, 16:9, 24 fps:
+            params = [
+                {
+                    "video_name": "cat_park.mp4",
+                    "prompt": "小猫骑着滑板穿过公园。 --rs 720p --rt 16:9 --dur 5 --fps 24 --wm false"
+                },
+                {
+                    "video_name": "city_night.mp4",
+                    "prompt": "霓虹灯下的城市延时摄影风。 --rs 720p --rt 16:9 --dur 5 --fps 24 --seed 7"
+                },
+            ]
+
+        2) With guided first/last frame (square, 6 s, camera fixed):
+            params = [
+                {
+                    "video_name": "logo_reveal.mp4",
+                    "first_frame": "https://cdn.example.com/brand/logo_start.png",
+                    "last_frame": "https://cdn.example.com/brand/logo_end.png",
+                    "prompt": "品牌 Logo 从线稿到上色的变化。 --rs 1080p --rt 1:1 --dur 6 --fps 24 --cf true"
+                }
+            ]
+    """
+    success_list = []
+    error_list = []
+    logger.debug(f"Using model: {getenv('MODEL_VIDEO_NAME', DEFAULT_VIDEO_MODEL_NAME)}")
+    logger.debug(f"video_generate params: {params}")
+
+    for start_idx in range(0, len(params), batch_size):
+        batch = params[start_idx : start_idx + batch_size]
+        logger.debug(f"video_generate batch {start_idx // batch_size}: {batch}")
+
+        task_dict = {}
+        tracer = trace.get_tracer("gcp.vertex.agent")
+        with tracer.start_as_current_span("call_llm") as span:
+            input_part = {"role": "user"}
+            output_part = {"message.role": "model"}
+            total_tokens = 0
+            for idx, item in enumerate(batch):
+                input_part[f"parts.{idx}.type"] = "text"
+                input_part[f"parts.{idx}.text"] = json.dumps(item, ensure_ascii=False)
+
+                video_name = item["video_name"]
+                prompt = item["prompt"]
+                first_frame = item.get("first_frame", None)
+                last_frame = item.get("last_frame", None)
+                try:
+                    if not first_frame:
+                        logger.debug(
+                            f"video_generate task_{idx} text generation: prompt={prompt}"
+                        )
+                        response = await generate(prompt)
+                    elif not last_frame:
+                        logger.debug(
+                            f"video_generate task_{idx} first frame generation: prompt={prompt}, first_frame={first_frame}"
+                        )
+                        response = await generate(prompt, first_frame)
+                    else:
+                        logger.debug(
+                            f"video_generate task_{idx} first and last frame generation: prompt={prompt}, first_frame={first_frame}, last_frame={last_frame}"
+                        )
+                        response = await generate(prompt, first_frame, last_frame)
+                    logger.debug(
+                        f"batch_{start_idx // batch_size} video_generate task_{idx} response: {response}"
+                    )
+                    task_dict[response.id] = video_name
+                except Exception as e:
+                    logger.error(f"Error: {e}")
+                    error_list.append(video_name)
+                    continue
+
+            logger.debug("begin query video_generate task status...")
+
+            while True:
+                task_list = list(task_dict.keys())
+                if len(task_list) == 0:
+                    break
+                for idx, task_id in enumerate(task_list):
+                    result = client.content_generation.tasks.get(task_id=task_id)
+                    status = result.status
+                    if status == "succeeded":
+                        logger.debug(
+                            f"{task_dict[task_id]} video_generate {status}. Video URL: {result.content.video_url}"
+                        )
+                        tool_context.state[f"{task_dict[task_id]}_video_url"] = (
+                            result.content.video_url
+                        )
+                        total_tokens += result.usage.completion_tokens
+                        output_part[f"message.parts.{idx}.type"] = "text"
+                        output_part[f"message.parts.{idx}.text"] = (
+                            f"{task_dict[task_id]}: {result.content.video_url}"
+                        )
+                        success_list.append(
+                            {task_dict[task_id]: result.content.video_url}
+                        )
+                        task_dict.pop(task_id, None)
+                    elif status == "failed":
+                        logger.error(
+                            f"{task_dict[task_id]} video_generate {status}. Error: {result.error}"
+                        )
+                        error_list.append(task_dict[task_id])
+                        task_dict.pop(task_id, None)
+                    else:
+                        logger.debug(
+                            f"{task_dict[task_id]} video_generate current status: {status}, Retrying after 10 seconds..."
+                        )
+                time.sleep(10)
+
+            add_span_attributes(
+                span,
+                tool_context,
+                input_part=input_part,
+                output_part=output_part,
+                output_tokens=total_tokens,
+                total_tokens=total_tokens,
+                request_model=getenv("MODEL_VIDEO_NAME", DEFAULT_VIDEO_MODEL_NAME),
+                response_model=getenv("MODEL_VIDEO_NAME", DEFAULT_VIDEO_MODEL_NAME),
+            )
+
+    if len(success_list) == 0:
+        logger.debug(
+            f"video_generate success_list: {success_list}\nerror_list: {error_list}"
+        )
+        return {
+            "status": "error",
+            "success_list": success_list,
+            "error_list": error_list,
+        }
+    else:
+        logger.debug(
+            f"video_generate success_list: {success_list}\nerror_list: {error_list}"
+        )
+        return {
+            "status": "success",
+            "success_list": success_list,
+            "error_list": error_list,
+        }
+
+
+def add_span_attributes(
+    span: Span,
+    tool_context: ToolContext,
+    input_part: dict | None = None,
+    output_part: dict | None = None,
+    input_tokens: int | None = None,
+    output_tokens: int | None = None,
+    total_tokens: int | None = None,
+    request_model: str | None = None,
+    response_model: str | None = None,
+):
+    try:
+        # common attributes
+        app_name = tool_context._invocation_context.app_name
+        user_id = tool_context._invocation_context.user_id
+        agent_name = tool_context.agent_name
+        session_id = tool_context._invocation_context.session.id
+        span.set_attribute("gen_ai.agent.name", agent_name)
+        span.set_attribute("openinference.instrumentation.veadk", VERSION)
+        span.set_attribute("gen_ai.app.name", app_name)
+        span.set_attribute("gen_ai.user.id", user_id)
+        span.set_attribute("gen_ai.session.id", session_id)
+        span.set_attribute("agent_name", agent_name)
+        span.set_attribute("agent.name", agent_name)
+        span.set_attribute("app_name", app_name)
+        span.set_attribute("app.name", app_name)
+        span.set_attribute("user.id", user_id)
+        span.set_attribute("session.id", session_id)
+        span.set_attribute("cozeloop.report.source", "veadk")
+
+        # llm attributes
+        span.set_attribute("gen_ai.system", "openai")
+        span.set_attribute("gen_ai.operation.name", "chat")
+        if request_model:
+            span.set_attribute("gen_ai.request.model", request_model)
+        if response_model:
+            span.set_attribute("gen_ai.response.model", response_model)
+        if total_tokens:
+            span.set_attribute("gen_ai.usage.total_tokens", total_tokens)
+        if output_tokens:
+            span.set_attribute("gen_ai.usage.output_tokens", output_tokens)
+        if input_tokens:
+            span.set_attribute("gen_ai.usage.input_tokens", input_tokens)
+        if input_part:
+            span.add_event("gen_ai.user.message", input_part)
+        if output_part:
+            span.add_event("gen_ai.choice", output_part)
+
+    except Exception:
+        traceback.print_exc()