veadk-python 0.2.27__py3-none-any.whl

veadk/cli/cli_deploy.py

@@ -0,0 +1,233 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import click
+
+from veadk.version import VERSION
+
+TEMP_PATH = "/tmp"
+
+
+@click.command()
+@click.option(
+    "--volcengine-access-key",
+    default=None,
+    help="Volcengine access key",
+)
+@click.option(
+    "--volcengine-secret-key",
+    default=None,
+    help="Volcengine secret key",
+)
+@click.option(
+    "--vefaas-app-name", required=True, help="Expected Volcengine FaaS application name"
+)
+@click.option(
+    "--veapig-instance-name", default="", help="Expected Volcengine APIG instance name"
+)
+@click.option(
+    "--veapig-service-name", default="", help="Expected Volcengine APIG service name"
+)
+@click.option(
+    "--veapig-upstream-name", default="", help="Expected Volcengine APIG upstream name"
+)
+@click.option(
+    "--short-term-memory-backend",
+    default="local",
+    type=click.Choice(["local", "mysql"]),
+    help="Backend for short-term memory",
+)
+@click.option("--use-adk-web", is_flag=True, help="Whether to use ADK Web")
+@click.option(
+    "--auth-method",
+    default="none",
+    type=click.Choice(["none", "api-key", "oauth2"]),
+    help="=Authentication method for agent",
+)
+@click.option(
+    "--user-pool-name",
+    default="",
+    help="Expected Volcengine Identity user pool name",
+)
+@click.option(
+    "--client-name",
+    default="",
+    help="Expected Volcengine Identity client name",
+)
+@click.option("--path", default=".", help="Local project path")
+@click.option("--iam-role", default=None, help="iam role for the vefaas function")
+def deploy(
+    volcengine_access_key: str,
+    volcengine_secret_key: str,
+    vefaas_app_name: str,
+    veapig_instance_name: str,
+    veapig_service_name: str,
+    veapig_upstream_name: str,
+    short_term_memory_backend: str,
+    use_adk_web: bool,
+    auth_method: str,
+    user_pool_name: str,
+    client_name: str,
+    path: str,
+    iam_role: str,
+) -> None:
+    """Deploy a user project to Volcengine FaaS application.
+
+    This command deploys a VeADK agent project to Volcengine's Function as a Service (FaaS)
+    platform. It creates a deployment package from the local project, configures the necessary
+    cloud resources, and manages the deployment process including template generation,
+    file copying, and cloud resource provisioning.
+
+    The deployment process includes:
+    1. Creating a temporary deployment package using cookiecutter templates
+    2. Copying the user's project files to the deployment structure
+    3. Processing configuration files and requirements
+    4. Executing the deployment to Volcengine FaaS
+    5. Cleaning up temporary files
+
+    Args:
+        volcengine_access_key: Volcengine access key for API authentication. If not provided,
+            will use VOLCENGINE_ACCESS_KEY environment variable
+        volcengine_secret_key: Volcengine secret key for API authentication. If not provided,
+            will use VOLCENGINE_SECRET_KEY environment variable
+        vefaas_app_name: Name of the target Volcengine FaaS application where the
+            project will be deployed
+        veapig_instance_name: Optional Volcengine API Gateway instance name for
+            external API access configuration
+        veapig_service_name: Optional Volcengine API Gateway service name
+        veapig_upstream_name: Optional Volcengine API Gateway upstream name
+        short_term_memory_backend: Backend type for short-term memory storage.
+            Choices are 'local' or 'mysql'
+        use_adk_web: Flag to enable ADK Web interface for the deployed agent
+        auth_method: Authentication for the agent.
+            Choices are 'none', 'api-key' or 'oauth2'.
+        veidentity_user_pool_name: Optional Volcengine Identity user pool name
+        veidentity_client_name: Optional Volcengine Identity client name
+        path: Local directory path containing the VeADK project to deploy
+
+    Note:
+        - The function automatically processes and copies requirements.txt if present in the project
+        - config.yaml files are excluded from deployment for security reasons
+        - Temporary files are created in /tmp and cleaned up after deployment
+        - The deployment uses cookiecutter templates for standardized project structure
+    """
+    import asyncio
+    import shutil
+    from pathlib import Path
+
+    from cookiecutter.main import cookiecutter
+
+    import veadk.integrations.ve_faas as vefaas
+    from veadk.config import getenv
+    from veadk.utils.logger import get_logger
+    from veadk.utils.misc import formatted_timestamp, load_module_from_file
+    import os
+    from veadk.config import veadk_environments
+
+    logger = get_logger(__name__)
+
+    if not volcengine_access_key:
+        volcengine_access_key = getenv("VOLCENGINE_ACCESS_KEY")
+    if not volcengine_secret_key:
+        volcengine_secret_key = getenv("VOLCENGINE_SECRET_KEY")
+    if not iam_role:
+        iam_role = getenv("IAM_ROLE", None, allow_false_values=True)
+    else:
+        os.environ["IAM_ROLE"] = iam_role
+        veadk_environments["IAM_ROLE"] = iam_role
+
+    user_proj_abs_path = Path(path).resolve()
+    template_dir_path = Path(vefaas.__file__).parent / "template"
+
+    tmp_dir_name = f"{user_proj_abs_path.name}_{formatted_timestamp()}"
+
+    settings = {
+        "local_dir_name": tmp_dir_name.replace("-", "_"),
+        "app_name": user_proj_abs_path.name.replace("-", "_"),
+        "agent_module_name": user_proj_abs_path.name,
+        "short_term_memory_backend": short_term_memory_backend,
+        "vefaas_application_name": vefaas_app_name,
+        "veapig_instance_name": veapig_instance_name,
+        "veapig_service_name": veapig_service_name,
+        "veapig_upstream_name": veapig_upstream_name,
+        "use_adk_web": use_adk_web,
+        "auth_method": auth_method,
+        "veidentity_user_pool_name": user_pool_name,
+        "veidentity_client_name": client_name,
+        "veadk_version": VERSION,
+    }
+
+    cookiecutter(
+        template=str(template_dir_path),
+        output_dir=TEMP_PATH,
+        no_input=True,
+        extra_context=settings,
+    )
+    logger.debug(f"Create a template project at {TEMP_PATH}/{tmp_dir_name}")
+
+    agent_dir = (
+        Path(TEMP_PATH)
+        / tmp_dir_name
+        / "src"
+        / user_proj_abs_path.name.replace("-", "_")
+    )
+
+    # remove /tmp/tmp_dir_name/src/user_proj_abs_path.name
+    shutil.rmtree(agent_dir)
+    agent_dir.mkdir(parents=True, exist_ok=True)
+
+    # copy
+    shutil.copytree(user_proj_abs_path, agent_dir, dirs_exist_ok=True)
+    logger.debug(f"Remove agent module from {user_proj_abs_path} to {agent_dir}")
+
+    # copy requirements.txt
+    if (user_proj_abs_path / "requirements.txt").exists():
+        logger.debug(
+            f"Find a requirements.txt in {user_proj_abs_path}/requirements.txt, copy it to temp project."
+        )
+        shutil.copy(
+            user_proj_abs_path / "requirements.txt",
+            Path(TEMP_PATH) / tmp_dir_name / "src" / "requirements.txt",
+        )
+    else:
+        logger.warning(
+            "No requirements.txt found in the user project, we will use a default one."
+        )
+
+    # avoid upload user's config.yaml
+    if (user_proj_abs_path / "config.yaml").exists():
+        logger.warning(
+            f"Find a config.yaml in {user_proj_abs_path}/config.yaml, we will not upload it by default."
+        )
+        shutil.move(agent_dir / "config.yaml", Path(TEMP_PATH) / tmp_dir_name)
+    else:
+        logger.info(
+            "No config.yaml found in the user project. Some environment variables may not be set."
+        )
+
+    # load
+    logger.debug(
+        f"Load deploy module from {Path(TEMP_PATH) / tmp_dir_name / 'deploy.py'}"
+    )
+    deploy_module = load_module_from_file(
+        module_name="deploy_module",
+        file_path=str(Path(TEMP_PATH) / tmp_dir_name / "deploy.py"),
+    )
+    logger.info(f"Begin deploy from {Path(TEMP_PATH) / tmp_dir_name / 'src'}")
+    asyncio.run(deploy_module.main())
+
+    # remove tmp file
+    logger.info("Deploy done. Delete temp dir.")
+    shutil.rmtree(Path(TEMP_PATH) / tmp_dir_name)

veadk/cli/cli_eval.py

@@ -0,0 +1,215 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from veadk.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+@click.command()
+@click.option(
+    "--agent-dir",
+    default=".",
+    help="To-be-evaluated agent directory. Must export `root_agent` in `agent.py`",
+)
+@click.option(
+    "--agent-a2a-url",
+    default=None,
+    help="To-be-evaluated agent URL. The agent should be deployed as A2A mode.",
+)
+@click.option(
+    "--evalset-file",
+    required=True,
+    help="Google ADK formatted evalset file path",
+)
+@click.option(
+    "--evaluator",
+    type=click.Choice(["adk", "deepeval"], case_sensitive=False),
+    help="Evaluator type, choose `adk` or `deepeval`",
+)
+@click.option(
+    "--judge-model-name",
+    default="doubao-1-5-pro-256k-250115",
+    help="Judge model name, default is `doubao-1-5-pro-256k-250115`. Useless under `adk` evaluator.",
+)
+@click.option(
+    "--volcengine-access-key",
+    default=None,
+    help="Volcengine access key for using Volcengine models",
+)
+@click.option(
+    "--volcengine-secret-key",
+    default=None,
+    help="Volcengine secret key for using Volcengine models",
+)
+def eval(
+    agent_dir: str,
+    agent_a2a_url: str,
+    evalset_file: str,
+    evaluator: str,
+    judge_model_name: str,
+    volcengine_access_key: str,
+    volcengine_secret_key: str,
+) -> None:
+    """Evaluate an agent using specified evaluation datasets and metrics.
+
+    This command provides comprehensive agent evaluation capabilities using either Google ADK
+    or DeepEval frameworks. It supports both local agent evaluation (from source code) and
+    remote agent evaluation (via A2A deployment URLs), making it flexible for different
+    development and deployment scenarios.
+
+    The evaluation process includes:
+    1. Loading the target agent from local directory or remote A2A endpoint
+    2. Configuring the evaluation environment and credentials
+    3. Setting up the chosen evaluator with appropriate metrics
+    4. Running evaluation tests against the provided dataset
+    5. Generating detailed performance reports and scores
+
+    Evaluation Modes:
+    - Local Evaluation: Loads agent code from a local directory containing 'agent.py'
+      with exported 'root_agent' variable. Suitable for development and testing.
+    - Remote Evaluation: Connects to a deployed agent via A2A (Agent-to-Agent) URL.
+      Ideal for evaluating production deployments or distributed agents.
+
+    Evaluator Options:
+    - ADK Evaluator: Uses Google's Agent Development Kit evaluation framework.
+      Provides standardized metrics and comprehensive evaluation reports.
+    - DeepEval: Advanced evaluation framework with customizable metrics including
+      GEval for general performance and ToolCorrectnessMetric for tool usage accuracy.
+
+    Args:
+        agent_dir: Local directory path containing the agent implementation.
+            Must include an 'agent.py' file with exported 'root_agent' variable.
+            Defaults to current directory if not specified
+        agent_a2a_url: Complete URL of the deployed agent in A2A mode.
+            If provided alongside agent_dir, this URL takes precedence
+        evalset_file: Path to the evaluation dataset file in Google ADK format.
+            Should contain test cases with inputs, expected outputs, and metadata
+        evaluator: Evaluation framework to use. Available options:
+            - 'adk': Google ADK evaluator with built-in metrics
+            - 'deepeval': Advanced evaluator with customizable metrics and thresholds
+        judge_model_name: Name of the language model used for evaluation judgment.
+            Defaults to 'doubao-1-5-pro-256k-250115'. Only applicable for DeepEval;
+            ignored when using ADK evaluator
+        volcengine_access_key: Volcengine platform access key for model authentication.
+            If not provided, uses VOLCENGINE_ACCESS_KEY environment variable
+        volcengine_secret_key: Volcengine platform secret key for model authentication.
+            If not provided, uses VOLCENGINE_SECRET_KEY environment variable
+
+    Note:
+        - At least one of --agent-dir or --agent-a2a-url must be provided
+        - If both are provided, --agent-a2a-url takes precedence
+        - Judge model name is ignored when using ADK evaluator
+        - Evaluation results are logged and may be saved to output files
+
+    Raises:
+        ImportError: If DeepEval dependencies are not installed when using DeepEval evaluator.
+        ValueError: If neither agent_dir nor agent_a2a_url is provided.
+    """
+    import asyncio
+    import os
+    from pathlib import Path
+
+    from google.adk.cli.utils.agent_loader import AgentLoader
+
+    from veadk.a2a.remote_ve_agent import RemoteVeAgent
+    from veadk.config import getenv, settings
+    from veadk.prompts.prompt_evaluator import eval_principle_prompt
+
+    try:
+        from deepeval.metrics import GEval, ToolCorrectnessMetric
+        from deepeval.test_case import LLMTestCaseParams
+
+        from veadk.evaluation.adk_evaluator import ADKEvaluator
+        from veadk.evaluation.deepeval_evaluator import DeepevalEvaluator
+    except ImportError:
+        raise ImportError(
+            "Please install veadk with `[evaluation]` extras, e.g., `pip install veadk-python[eval]`"
+        )
+
+    # ====== prepare agent instance ======
+    if not agent_dir and not agent_a2a_url:
+        raise ValueError(
+            "Option `--agent-dir` or  `--agent-a2a-url` should be provided one of them."
+        )
+
+    if agent_dir and agent_a2a_url:
+        logger.warning(
+            "`--agent-dir` and `--agent-a2a-url` are both provided, will use `--agent-a2a-url`."
+        )
+        agent_instance = RemoteVeAgent(name="a2a_agent", url=agent_a2a_url)
+        logger.info(f"Loaded agent from {agent_a2a_url}")
+
+    if not agent_dir and agent_a2a_url:
+        agent_instance = RemoteVeAgent(name="a2a_agent", url=agent_a2a_url)
+        logger.info(f"Loaded agent from {agent_a2a_url}")
+
+    if agent_dir and not agent_a2a_url:
+        agent_instance = AgentLoader(str(Path(agent_dir).parent.resolve())).load_agent(
+            str(Path(agent_dir).name)
+        )
+        logger.info(f"Loaded agent from {agent_dir}, agent name: {agent_instance.name}")
+
+    # ====== prepare envs ======
+    if volcengine_access_key and "VOLCENGINE_ACCESS_KEY" not in os.environ:
+        os.environ["VOLCENGINE_ACCESS_KEY"] = volcengine_access_key
+    if volcengine_secret_key and "VOLCENGINE_SECRET_KEY" not in os.environ:
+        os.environ["VOLCENGINE_SECRET_KEY"] = volcengine_secret_key
+
+    # ====== prepare evaluator instance ======
+    evaluator_instance = None
+    if evaluator == "adk" and judge_model_name:
+        logger.warning(
+            "Using Google ADK evaluator, `--judge-model-name` will be ignored."
+        )
+        evaluator_instance = ADKEvaluator(agent=agent_instance)
+
+        asyncio.run(evaluator_instance.evaluate(eval_set_file_path=evalset_file))
+
+    if evaluator == "deepeval":
+        if not volcengine_access_key:
+            volcengine_access_key = getenv("VOLCENGINE_ACCESS_KEY")
+        if not volcengine_secret_key:
+            volcengine_secret_key = getenv("VOLCENGINE_SECRET_KEY")
+
+        evaluator_instance = DeepevalEvaluator(
+            agent=agent_instance,
+            judge_model_api_key=settings.model.api_key,
+            judge_model_name=judge_model_name,
+        )
+
+        judge_model = evaluator_instance.judge_model
+
+        metrics = [
+            GEval(
+                threshold=0.8,
+                name="Base Evaluation",
+                criteria=eval_principle_prompt,
+                evaluation_params=[
+                    LLMTestCaseParams.INPUT,
+                    LLMTestCaseParams.ACTUAL_OUTPUT,
+                    LLMTestCaseParams.EXPECTED_OUTPUT,
+                ],
+                model=judge_model,
+            ),
+            ToolCorrectnessMetric(threshold=0.5, model=judge_model),
+        ]
+
+        asyncio.run(
+            evaluator_instance.evaluate(
+                eval_set_file_path=evalset_file, metrics=metrics
+            )
+        )

veadk/cli/cli_init.py

@@ -0,0 +1,214 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from typing import Any
+
+import click
+
+from veadk.version import VERSION
+
+warnings.filterwarnings(
+    "ignore", category=UserWarning, module="pydantic._internal._fields"
+)
+
+
+def _render_prompts() -> dict[str, Any]:
+    """Render interactive prompts to collect user configuration for project initialization.
+
+    This function prompts the user for various configuration options including
+    Volcengine FaaS application name, API Gateway settings, and deployment mode.
+
+    Returns:
+        dict[str, Any]: A dictionary containing all the collected configuration values
+    """
+    vefaas_application_name = click.prompt(
+        "Volcengine FaaS application name", default="veadk-cloud-agent"
+    )
+
+    veapig_instance_name = click.prompt(
+        "Volcengine API Gateway instance name", default="", show_default=True
+    )
+
+    veapig_service_name = click.prompt(
+        "Volcengine API Gateway service name", default="", show_default=True
+    )
+
+    veapig_upstream_name = click.prompt(
+        "Volcengine API Gateway upstream name", default="", show_default=True
+    )
+
+    deploy_mode_options = {
+        "1": "A2A/MCP Server",
+        "2": "VeADK Web / Google ADK Web",
+    }
+
+    click.echo("Choose a deploy mode:")
+    for key, value in deploy_mode_options.items():
+        click.echo(f"  {key}. {value}")
+
+    deploy_mode = click.prompt(
+        "Enter your choice", type=click.Choice(deploy_mode_options.keys())
+    )
+    use_adk_web = deploy_mode == "2"
+
+    auth_method_options = {}
+    auth_methods = {}
+    if use_adk_web:
+        auth_method_options = {
+            "1": "None",
+            "2": "OAuth2",
+        }
+        auth_methods = {
+            "1": "none",
+            "2": "oauth2",
+        }
+    else:
+        auth_method_options = {
+            "1": "None",
+            "2": "API key",
+            "3": "OAuth2",
+        }
+        auth_methods = {
+            "1": "none",
+            "2": "api-key",
+            "3": "oauth2",
+        }
+
+    click.echo("Choose an authentication method:")
+    for key, value in auth_method_options.items():
+        click.echo(f"  {key}. {value}")
+
+    auth_method_idx = click.prompt(
+        "Enter your choice", type=click.Choice(auth_method_options.keys())
+    )
+    auth_method = auth_methods[auth_method_idx]
+
+    veidentity_user_pool_name = ""
+    veidentity_client_name = ""
+    if auth_method == "oauth2":
+        veidentity_user_pool_name = click.prompt(
+            "Volcengine Identity user pool name", default="", show_default=True
+        )
+
+        if use_adk_web:
+            veidentity_client_name = click.prompt(
+                "Volcengine Identity client name", default="", show_default=True
+            )
+
+    return {
+        "vefaas_application_name": vefaas_application_name,
+        "veapig_instance_name": veapig_instance_name,
+        "veapig_service_name": veapig_service_name,
+        "veapig_upstream_name": veapig_upstream_name,
+        "use_adk_web": use_adk_web,
+        "auth_method": auth_method,
+        "veidentity_user_pool_name": veidentity_user_pool_name,
+        "veidentity_client_name": veidentity_client_name,
+        "veadk_version": VERSION,
+    }
+
+
+@click.command()
+@click.option(
+    "--vefaas-template-type", default="template", help="Expected template type"
+)
+def init(
+    vefaas_template_type: str,
+) -> None:
+    """Initialize a new VeADK project that can be deployed to Volcengine FaaS.
+
+    This command creates a new VeADK project from predefined templates using an interactive
+    setup process. It generates a complete project structure with all necessary files,
+    configurations, and deployment scripts ready for Volcengine cloud deployment.
+
+    The initialization process includes:
+    1. Interactive prompts for collecting deployment configuration
+    2. Template selection based on the specified template type
+    3. Project directory creation with proper structure
+    4. Configuration file generation with user preferences
+    5. Ready-to-use deployment scripts and source code structure
+
+    Available template types:
+    - 'template' (default): Creates an A2A/MCP/Web server template with a weather-reporter
+      example application. Suitable for most agent development scenarios.
+    - 'web_template': Creates a web application template with a simple-blog example.
+      Designed for web-based agent applications with UI components.
+
+    The generated project structure includes:
+    - src/ directory containing agent source code
+    - deploy.py script for cloud deployment
+    - Configuration files for various deployment scenarios
+    - Example implementations based on the selected template
+
+    Args:
+        vefaas_template_type: The type of template to use for project initialization.
+            Defaults to 'template'. Valid options are:
+            - 'template': Standard agent template (weather-reporter example)
+            - 'web_template': Web application template (simple-blog example)
+
+    Note:
+        - If the target directory already exists, you will be prompted to confirm overwrite
+        - The generated project includes example code that can be modified for your use case
+        - All deployment configurations can be customized after project creation
+        - The deploy.py script provides automated deployment to Volcengine FaaS platform
+    """
+    import shutil
+    from pathlib import Path
+
+    from cookiecutter.main import cookiecutter
+
+    import veadk.integrations.ve_faas as vefaas
+
+    if vefaas_template_type == "web_template":
+        click.echo(
+            "Welcome use VeADK to create your project. We will generate a `simple-blog` web application for you."
+        )
+    else:
+        click.echo(
+            "Welcome use VeADK to create your project. We will generate a `weather-reporter` application for you."
+        )
+
+    cwd = Path.cwd()
+    local_dir_name = click.prompt("Local directory name", default="veadk-cloud-proj")
+    target_dir_path = cwd / local_dir_name
+
+    if target_dir_path.exists():
+        click.confirm(
+            f"Directory '{target_dir_path}' already exists, do you want to overwrite it",
+            abort=True,
+        )
+        shutil.rmtree(target_dir_path)
+
+    settings = _render_prompts()
+    settings["local_dir_name"] = local_dir_name
+
+    if not vefaas_template_type:
+        vefaas_template_type = "template"
+
+    template_dir_path = Path(vefaas.__file__).parent / vefaas_template_type
+
+    cookiecutter(
+        template=str(template_dir_path),
+        output_dir=str(cwd),
+        extra_context=settings,
+        no_input=True,
+    )
+
+    click.echo(f"Template project has been generated at {target_dir_path}")
+    click.echo(f"Edit {target_dir_path / 'src/'} to define your agents")
+    click.echo(
+        f"Edit {target_dir_path / 'deploy.py'} to define your deployment attributes"
+    )
+    click.echo("Run python `deploy.py` for deployment on Volcengine FaaS platform.")