autoglm-gui 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

phone_agent/agent_ios.py

@@ -0,0 +1,277 @@
+"""iOS PhoneAgent class for orchestrating iOS phone automation."""
+
+import json
+import traceback
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from phone_agent.actions.handler import finish, parse_action
+from phone_agent.actions.handler_ios import IOSActionHandler
+from phone_agent.config import get_messages, get_system_prompt
+from phone_agent.model import ModelClient, ModelConfig
+from phone_agent.model.client import MessageBuilder
+from phone_agent.xctest import XCTestConnection, get_current_app, get_screenshot
+
+
+@dataclass
+class IOSAgentConfig:
+    """Configuration for the iOS PhoneAgent."""
+
+    max_steps: int = 100
+    wda_url: str = "http://localhost:8100"
+    session_id: str | None = None
+    device_id: str | None = None  # iOS device UDID
+    lang: str = "cn"
+    system_prompt: str | None = None
+    verbose: bool = True
+
+    def __post_init__(self):
+        if self.system_prompt is None:
+            self.system_prompt = get_system_prompt(self.lang)
+
+
+@dataclass
+class StepResult:
+    """Result of a single agent step."""
+
+    success: bool
+    finished: bool
+    action: dict[str, Any] | None
+    thinking: str
+    message: str | None = None
+
+
+class IOSPhoneAgent:
+    """
+    AI-powered agent for automating iOS phone interactions.
+
+    The agent uses a vision-language model to understand screen content
+    and decide on actions to complete user tasks via WebDriverAgent.
+
+    Args:
+        model_config: Configuration for the AI model.
+        agent_config: Configuration for the iOS agent behavior.
+        confirmation_callback: Optional callback for sensitive action confirmation.
+        takeover_callback: Optional callback for takeover requests.
+
+    Example:
+        >>> from phone_agent.agent_ios import IOSPhoneAgent, IOSAgentConfig
+        >>> from phone_agent.model import ModelConfig
+        >>>
+        >>> model_config = ModelConfig(base_url="http://localhost:8000/v1")
+        >>> agent_config = IOSAgentConfig(wda_url="http://localhost:8100")
+        >>> agent = IOSPhoneAgent(model_config, agent_config)
+        >>> agent.run("Open Safari and search for Apple")
+    """
+
+    def __init__(
+        self,
+        model_config: ModelConfig | None = None,
+        agent_config: IOSAgentConfig | None = None,
+        confirmation_callback: Callable[[str], bool] | None = None,
+        takeover_callback: Callable[[str], None] | None = None,
+    ):
+        self.model_config = model_config or ModelConfig()
+        self.agent_config = agent_config or IOSAgentConfig()
+
+        self.model_client = ModelClient(self.model_config)
+
+        # Initialize WDA connection and create session if needed
+        self.wda_connection = XCTestConnection(wda_url=self.agent_config.wda_url)
+
+        # Auto-create session if not provided
+        if self.agent_config.session_id is None:
+            success, session_id = self.wda_connection.start_wda_session()
+            if success and session_id != "session_started":
+                self.agent_config.session_id = session_id
+                if self.agent_config.verbose:
+                    print(f"✅ Created WDA session: {session_id}")
+            elif self.agent_config.verbose:
+                print("⚠️  Using default WDA session (no explicit session ID)")
+
+        self.action_handler = IOSActionHandler(
+            wda_url=self.agent_config.wda_url,
+            session_id=self.agent_config.session_id,
+            confirmation_callback=confirmation_callback,
+            takeover_callback=takeover_callback,
+        )
+
+        self._context: list[dict[str, Any]] = []
+        self._step_count = 0
+
+    def run(self, task: str) -> str:
+        """
+        Run the agent to complete a task.
+
+        Args:
+            task: Natural language description of the task.
+
+        Returns:
+            Final message from the agent.
+        """
+        self._context = []
+        self._step_count = 0
+
+        # First step with user prompt
+        result = self._execute_step(task, is_first=True)
+
+        if result.finished:
+            return result.message or "Task completed"
+
+        # Continue until finished or max steps reached
+        while self._step_count < self.agent_config.max_steps:
+            result = self._execute_step(is_first=False)
+
+            if result.finished:
+                return result.message or "Task completed"
+
+        return "Max steps reached"
+
+    def step(self, task: str | None = None) -> StepResult:
+        """
+        Execute a single step of the agent.
+
+        Useful for manual control or debugging.
+
+        Args:
+            task: Task description (only needed for first step).
+
+        Returns:
+            StepResult with step details.
+        """
+        is_first = len(self._context) == 0
+
+        if is_first and not task:
+            raise ValueError("Task is required for the first step")
+
+        return self._execute_step(task, is_first)
+
+    def reset(self) -> None:
+        """Reset the agent state for a new task."""
+        self._context = []
+        self._step_count = 0
+
+    def _execute_step(
+        self, user_prompt: str | None = None, is_first: bool = False
+    ) -> StepResult:
+        """Execute a single step of the agent loop."""
+        self._step_count += 1
+
+        # Capture current screen state
+        screenshot = get_screenshot(
+            wda_url=self.agent_config.wda_url,
+            session_id=self.agent_config.session_id,
+            device_id=self.agent_config.device_id,
+        )
+        current_app = get_current_app(
+            wda_url=self.agent_config.wda_url, session_id=self.agent_config.session_id
+        )
+
+        # Build messages
+        if is_first:
+            self._context.append(
+                MessageBuilder.create_system_message(self.agent_config.system_prompt)
+            )
+
+            screen_info = MessageBuilder.build_screen_info(current_app)
+            text_content = f"{user_prompt}\n\n{screen_info}"
+
+            self._context.append(
+                MessageBuilder.create_user_message(
+                    text=text_content, image_base64=screenshot.base64_data
+                )
+            )
+        else:
+            screen_info = MessageBuilder.build_screen_info(current_app)
+            text_content = f"** Screen Info **\n\n{screen_info}"
+
+            self._context.append(
+                MessageBuilder.create_user_message(
+                    text=text_content, image_base64=screenshot.base64_data
+                )
+            )
+
+        # Get model response
+        try:
+            response = self.model_client.request(self._context)
+        except Exception as e:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            return StepResult(
+                success=False,
+                finished=True,
+                action=None,
+                thinking="",
+                message=f"Model error: {e}",
+            )
+
+        # Parse action from response
+        try:
+            action = parse_action(response.action)
+        except ValueError:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            action = finish(message=response.action)
+
+        if self.agent_config.verbose:
+            # Print thinking process
+            msgs = get_messages(self.agent_config.lang)
+            print("\n" + "=" * 50)
+            print(f"💭 {msgs['thinking']}:")
+            print("-" * 50)
+            print(response.thinking)
+            print("-" * 50)
+            print(f"🎯 {msgs['action']}:")
+            print(json.dumps(action, ensure_ascii=False, indent=2))
+            print("=" * 50 + "\n")
+
+        # Remove image from context to save space
+        self._context[-1] = MessageBuilder.remove_images_from_message(self._context[-1])
+
+        # Execute action
+        try:
+            result = self.action_handler.execute(
+                action, screenshot.width, screenshot.height
+            )
+        except Exception as e:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            result = self.action_handler.execute(
+                finish(message=str(e)), screenshot.width, screenshot.height
+            )
+
+        # Add assistant response to context
+        self._context.append(
+            MessageBuilder.create_assistant_message(
+                f"<think>{response.thinking}</think><answer>{response.action}</answer>"
+            )
+        )
+
+        # Check if finished
+        finished = action.get("_metadata") == "finish" or result.should_finish
+
+        if finished and self.agent_config.verbose:
+            msgs = get_messages(self.agent_config.lang)
+            print("\n" + "🎉 " + "=" * 48)
+            print(
+                f"✅ {msgs['task_completed']}: {result.message or action.get('message', msgs['done'])}"
+            )
+            print("=" * 50 + "\n")
+
+        return StepResult(
+            success=result.success,
+            finished=finished,
+            action=action,
+            thinking=response.thinking,
+            message=result.message or action.get("message"),
+        )
+
+    @property
+    def context(self) -> list[dict[str, Any]]:
+        """Get the current conversation context."""
+        return self._context.copy()
+
+    @property
+    def step_count(self) -> int:
+        """Get the current step count."""
+        return self._step_count

phone_agent/config/__init__.py

@@ -1,9 +1,19 @@
 """Configuration module for Phone Agent."""
 
 from phone_agent.config.apps import APP_PACKAGES
+from phone_agent.config.apps_ios import APP_PACKAGES_IOS
 from phone_agent.config.i18n import get_message, get_messages
 from phone_agent.config.prompts_en import SYSTEM_PROMPT as SYSTEM_PROMPT_EN
 from phone_agent.config.prompts_zh import SYSTEM_PROMPT as SYSTEM_PROMPT_ZH
+from phone_agent.config.timing import (
+    TIMING_CONFIG,
+    ActionTimingConfig,
+    ConnectionTimingConfig,
+    DeviceTimingConfig,
+    TimingConfig,
+    get_timing_config,
+    update_timing_config,
+)
 
 
 def get_system_prompt(lang: str = "cn") -> str:
@@ -26,10 +36,18 @@ SYSTEM_PROMPT = SYSTEM_PROMPT_ZH
 
 __all__ = [
     "APP_PACKAGES",
+    "APP_PACKAGES_IOS",
     "SYSTEM_PROMPT",
     "SYSTEM_PROMPT_ZH",
     "SYSTEM_PROMPT_EN",
     "get_system_prompt",
     "get_messages",
     "get_message",
+    "TIMING_CONFIG",
+    "TimingConfig",
+    "ActionTimingConfig",
+    "DeviceTimingConfig",
+    "ConnectionTimingConfig",
+    "get_timing_config",
+    "update_timing_config",
 ]

phone_agent/config/apps.py

@@ -29,7 +29,7 @@ APP_PACKAGES: dict[str, str] = {
     "12306": "com.MobileTicket",
     "去哪儿": "com.Qunar",
     "去哪儿旅行": "com.Qunar",
-    "滴滴出行": "com.sdu.did.psnger",
+    "滴滴出行": "com.sdu.didi.psnger",
     # Video & Entertainment
     "bilibili": "tv.danmaku.bili",
     "抖音": "com.ss.android.ugc.aweme",

phone_agent/config/apps_harmonyos.py

@@ -0,0 +1,256 @@
+"""HarmonyOS application package name mappings.
+
+Maps user-friendly app names to HarmonyOS bundle names.
+These bundle names are used with the 'hdc shell aa start -b <bundle>' command.
+"""
+
+# Custom ability names for apps that don't use the default "EntryAbility"
+# Maps bundle_name -> ability_name
+# Generated by: python test/find_abilities.py
+APP_ABILITIES: dict[str, str] = {
+    # Third-party apps
+    "cn.wps.mobileoffice.hap": "DocumentAbility",
+    "com.ccb.mobilebank.hm": "CcbMainAbility",
+    "com.dewu.hos": "HomeAbility",
+    "com.larus.nova.hm": "MainAbility",
+    "com.luna.hm.music": "MainAbility",
+    "com.meitu.meitupic": "MainAbility",
+    "com.ss.hm.article.news": "MainAbility",
+    "com.ss.hm.ugc.aweme": "MainAbility",
+    "com.taobao.taobao4hmos": "Taobao_mainAbility",
+    "com.tencent.videohm": "AppAbility",
+    "com.ximalaya.ting.xmharmony": "MainBundleAbility",
+    "com.zhihu.hmos": "PhoneAbility",
+    # Huawei system apps
+    "com.huawei.hmos.browser": "MainAbility",
+    "com.huawei.hmos.calculator": "com.huawei.hmos.calculator.CalculatorAbility",
+    "com.huawei.hmos.calendar": "MainAbility",
+    "com.huawei.hmos.camera": "com.huawei.hmos.camera.MainAbility",
+    "com.huawei.hmos.clock": "com.huawei.hmos.clock.phone",
+    "com.huawei.hmos.clouddrive": "MainAbility",
+    "com.huawei.hmos.email": "ApplicationAbility",
+    "com.huawei.hmos.filemanager": "MainAbility",
+    "com.huawei.hmos.health": "Activity_card_entryAbility",
+    "com.huawei.hmos.notepad": "MainAbility",
+    "com.huawei.hmos.photos": "MainAbility",
+    "com.huawei.hmos.screenrecorder": "com.huawei.hmos.screenrecorder.ServiceExtAbility",
+    "com.huawei.hmos.screenshot": "com.huawei.hmos.screenshot.ServiceExtAbility",
+    "com.huawei.hmos.settings": "com.huawei.hmos.settings.MainAbility",
+    "com.huawei.hmos.soundrecorder": "MainAbility",
+    "com.huawei.hmos.vassistant": "AiCaptionServiceExtAbility",
+    "com.huawei.hmos.wallet": "MainAbility",
+    # Huawei services
+    "com.huawei.hmsapp.appgallery": "MainAbility",
+    "com.huawei.hmsapp.books": "MainAbility",
+    "com.huawei.hmsapp.himovie": "MainAbility",
+    "com.huawei.hmsapp.hisearch": "MainAbility",
+    "com.huawei.hmsapp.music": "MainAbility",
+    "com.huawei.hmsapp.thememanager": "MainAbility",
+    "com.huawei.hmsapp.totemweather": "com.huawei.hmsapp.totemweather.MainAbility",
+    # OHOS system apps
+    "com.ohos.callui": "com.ohos.callui.ServiceAbility",
+    "com.ohos.contacts": "com.ohos.contacts.MainAbility",
+    "com.ohos.mms": "com.ohos.mms.MainAbility",
+}
+
+APP_PACKAGES: dict[str, str] = {
+    # Social & Messaging
+    "微信": "com.tencent.wechat",
+    "QQ": "com.tencent.mqq",
+    "微博": "com.sina.weibo.stage",
+    # E-commerce
+    "淘宝": "com.taobao.taobao4hmos",
+    "京东": "com.jd.hm.mall",
+    "拼多多": "com.xunmeng.pinduoduo.hos",
+    "淘宝闪购": "com.taobao.taobao4hmos",
+    "京东秒送": "com.jd.hm.mall",
+    # Lifestyle & Social
+    "小红书": "com.xingin.xhs_hos",
+    "知乎": "com.zhihu.hmos",
+    # "豆瓣": "com.douban.frodo",  # 未在 hdc 列表中找到
+    # Maps & Navigation
+    "高德地图": "com.amap.hmapp",
+    "百度地图": "com.baidu.hmmap",
+    # Food & Services
+    "美团": "com.sankuai.hmeituan",
+    "美团外卖": "com.meituan.takeaway",
+    "大众点评": "com.sankuai.dianping",
+    # "肯德基": "com.yek.android.kfc.activitys",  # 未在 hdc 列表中找到
+    # Travel
+    # "携程": "ctrip.android.view",  # 未在 hdc 列表中找到
+    "铁路12306": "com.chinarailway.ticketingHM",
+    "12306": "com.chinarailway.ticketingHM",
+    # "去哪儿": "com.Qunar",  # 未在 hdc 列表中找到
+    # "去哪儿旅行": "com.Qunar",  # 未在 hdc 列表中找到
+    "滴滴出行": "com.sdu.didi.hmos.psnger",
+    # Video & Entertainment
+    "bilibili": "yylx.danmaku.bili",
+    "抖音": "com.ss.hm.ugc.aweme",
+    "快手": "com.kuaishou.hmapp",
+    "腾讯视频": "com.tencent.videohm",
+    "爱奇艺": "com.qiyi.video.hmy",
+    "芒果TV": "com.mgtv.phone",
+    # "优酷视频": "com.youku.phone",  # 未在 hdc 列表中找到
+    # "红果短剧": "com.phoenix.read",  # 未在 hdc 列表中找到
+    # Music & Audio
+    # "网易云音乐": "com.netease.cloudmusic",  # 未在 hdc 列表中找到
+    "QQ音乐": "com.tencent.hm.qqmusic",
+    "汽水音乐": "com.luna.hm.music",
+    "喜马拉雅": "com.ximalaya.ting.xmharmony",
+    # Reading
+    # "番茄小说": "com.dragon.read",  # 未在 hdc 列表中找到
+    # "番茄免费小说": "com.dragon.read",  # 未在 hdc 列表中找到
+    # "七猫免费小说": "com.kmxs.reader",  # 未在 hdc 列表中找到
+    # Productivity
+    "飞书": "com.ss.feishu",
+    # "QQ邮箱": "com.tencent.androidqqmail",  # 未在 hdc 列表中找到
+    # AI & Tools
+    "豆包": "com.larus.nova.hm",
+    # Health & Fitness
+    # "keep": "com.gotokeep.keep",  # 未在 hdc 列表中找到
+    # "美柚": "com.lingan.seeyou",  # 未在 hdc 列表中找到
+    # News & Information
+    # "腾讯新闻": "com.tencent.news",  # 未在 hdc 列表中找到
+    "今日头条": "com.ss.hm.article.news",
+    # Real Estate
+    # "贝壳找房": "com.lianjia.beike",  # 未在 hdc 列表中找到
+    # "安居客": "com.anjuke.android.app",  # 未在 hdc 列表中找到
+    # Finance
+    # "同花顺": "com.hexin.plat.android",  # 未在 hdc 列表中找到
+    # Games
+    # "星穹铁道": "com.miHoYo.hkrpg",  # 未在 hdc 列表中找到
+    # "崩坏：星穹铁道": "com.miHoYo.hkrpg",  # 未在 hdc 列表中找到
+    # "恋与深空": "com.papegames.lysk.cn",  # 未在 hdc 列表中找到
+    # HarmonyOS 第三方应用
+    "百度": "com.baidu.baiduapp",
+    "阿里巴巴": "com.alibaba.wireless_hmos",
+    "WPS": "cn.wps.mobileoffice.hap",
+    "企业微信": "com.tencent.wework.hmos",
+    "同程": "com.tongcheng.hmos",
+    "同程旅行": "com.tongcheng.hmos",
+    "唯品会": "com.vip.hosapp",
+    "支付宝": "com.alipay.mobile.client",
+    "UC浏览器": "com.uc.mobile",
+    "闲鱼": "com.taobao.idlefish4ohos",
+    "转转": "com.zhuanzhuan.hmoszz",
+    "迅雷": "com.xunlei.thunder",
+    "搜狗输入法": "com.sogou.input",
+    "扫描全能王": "com.intsig.camscanner.hap",
+    "美图秀秀": "com.meitu.meitupic",
+    "58同城": "com.wuba.life",
+    "得物": "com.dewu.hos",
+    "海底捞": "com.haidilao.haros",
+    "中国移动": "com.droi.tong",
+    "中国联通": "com.sinovatech.unicom.ha",
+    "国家税务总局": "cn.gov.chinatax.gt4.hm",
+    "建设银行": "com.ccb.mobilebank.hm",
+    "快手极速版": "com.kuaishou.hmnebula",
+    # HarmonyOS 系统应用 - 工具类
+    "浏览器": "com.huawei.hmos.browser",
+    "计算器": "com.huawei.hmos.calculator",
+    "日历": "com.huawei.hmos.calendar",
+    "相机": "com.huawei.hmos.camera",
+    "时钟": "com.huawei.hmos.clock",
+    "云盘": "com.huawei.hmos.clouddrive",
+    "云空间": "com.huawei.hmos.clouddrive",
+    "邮件": "com.huawei.hmos.email",
+    "文件管理器": "com.huawei.hmos.filemanager",
+    "文件": "com.huawei.hmos.files",
+    "查找设备": "com.huawei.hmos.finddevice",
+    "查找手机": "com.huawei.hmos.finddevice",
+    "录音机": "com.huawei.hmos.soundrecorder",
+    "录音": "com.huawei.hmos.soundrecorder",
+    "录屏": "com.huawei.hmos.screenrecorder",
+    "截屏": "com.huawei.hmos.screenshot",
+    "笔记": "com.huawei.hmos.notepad",
+    "备忘录": "com.huawei.hmos.notepad",
+    # HarmonyOS 系统应用 - 媒体类
+    "相册": "com.huawei.hmos.photos",
+    "图库": "com.huawei.hmos.photos",
+    # "视频": "com.huawei.hmos.mediaplayer",  # 未在 hdc 列表中找到，但有 com.huawei.hmsapp.himovie
+    # HarmonyOS 系统应用 - 通讯类
+    "联系人": "com.ohos.contacts",
+    "通讯录": "com.ohos.contacts",
+    "短信": "com.ohos.mms",
+    "信息": "com.ohos.mms",
+    "电话": "com.ohos.callui",
+    "拨号": "com.ohos.callui",
+    # HarmonyOS 系统应用 - 设置类
+    "设置": "com.huawei.hmos.settings",
+    "系统设置": "com.huawei.hmos.settings",
+    "AndroidSystemSettings": "com.huawei.hmos.settings",
+    "Android System Settings": "com.huawei.hmos.settings",
+    "Android  System Settings": "com.huawei.hmos.settings",
+    "Android-System-Settings": "com.huawei.hmos.settings",
+    "Settings": "com.huawei.hmos.settings",
+    # HarmonyOS 系统应用 - 生活服务
+    "健康": "com.huawei.hmos.health",
+    "运动健康": "com.huawei.hmos.health",
+    "地图": "com.huawei.hmos.maps.app",
+    "华为地图": "com.huawei.hmos.maps.app",
+    "钱包": "com.huawei.hmos.wallet",
+    "华为钱包": "com.huawei.hmos.wallet",
+    "智慧生活": "com.huawei.hmos.ailife",
+    "智能助手": "com.huawei.hmos.vassistant",
+    "小艺": "com.huawei.hmos.vassistant",
+    # HarmonyOS 服务
+    "应用市场": "com.huawei.hmsapp.appgallery",
+    "华为应用市场": "com.huawei.hmsapp.appgallery",
+    "音乐": "com.huawei.hmsapp.music",
+    "华为音乐": "com.huawei.hmsapp.music",
+    "主题": "com.huawei.hmsapp.thememanager",
+    "主题管理": "com.huawei.hmsapp.thememanager",
+    "天气": "com.huawei.hmsapp.totemweather",
+    "华为天气": "com.huawei.hmsapp.totemweather",
+    "视频": "com.huawei.hmsapp.himovie",
+    "华为视频": "com.huawei.hmsapp.himovie",
+    "阅读": "com.huawei.hmsapp.books",
+    "华为阅读": "com.huawei.hmsapp.books",
+    "游戏中心": "com.huawei.hmsapp.gamecenter",
+    "华为游戏中心": "com.huawei.hmsapp.gamecenter",
+    "搜索": "com.huawei.hmsapp.hisearch",
+    "华为搜索": "com.huawei.hmsapp.hisearch",
+    "指南针": "com.huawei.hmsapp.compass",
+    "会员中心": "com.huawei.hmos.myhuawei",
+    "我的华为": "com.huawei.hmos.myhuawei",
+    "华为会员": "com.huawei.hmos.myhuawei",
+}
+
+
+def get_package_name(app_name: str) -> str | None:
+    """
+    Get the package name for an app.
+
+    Args:
+        app_name: The display name of the app.
+
+    Returns:
+        The HarmonyOS bundle name, or None if not found.
+    """
+    return APP_PACKAGES.get(app_name)
+
+
+def get_app_name(package_name: str) -> str | None:
+    """
+    Get the app name from a package name.
+
+    Args:
+        package_name: The HarmonyOS bundle name.
+
+    Returns:
+        The display name of the app, or None if not found.
+    """
+    for name, package in APP_PACKAGES.items():
+        if package == package_name:
+            return name
+    return None
+
+
+def list_supported_apps() -> list[str]:
+    """
+    Get a list of all supported app names.
+
+    Returns:
+        List of app names.
+    """
+    return list(APP_PACKAGES.keys())