autoglm-gui 0.2.0__py3-none-any.whl → 0.2.3__py3-none-any.whl

phone_agent/adb/screenshot.py

@@ -0,0 +1,109 @@
+"""Screenshot utilities for capturing Android device screen."""
+
+import base64
+import os
+import subprocess
+import tempfile
+import uuid
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Tuple
+
+from PIL import Image
+
+
+@dataclass
+class Screenshot:
+    """Represents a captured screenshot."""
+
+    base64_data: str
+    width: int
+    height: int
+    is_sensitive: bool = False
+
+
+def get_screenshot(device_id: str | None = None, timeout: int = 10) -> Screenshot:
+    """
+    Capture a screenshot from the connected Android device.
+
+    Args:
+        device_id: Optional ADB device ID for multi-device setups.
+        timeout: Timeout in seconds for screenshot operations.
+
+    Returns:
+        Screenshot object containing base64 data and dimensions.
+
+    Note:
+        If the screenshot fails (e.g., on sensitive screens like payment pages),
+        a black fallback image is returned with is_sensitive=True.
+    """
+    temp_path = os.path.join(tempfile.gettempdir(), f"screenshot_{uuid.uuid4()}.png")
+    adb_prefix = _get_adb_prefix(device_id)
+
+    try:
+        # Execute screenshot command
+        result = subprocess.run(
+            adb_prefix + ["shell", "screencap", "-p", "/sdcard/tmp.png"],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+
+        # Check for screenshot failure (sensitive screen)
+        output = result.stdout + result.stderr
+        if "Status: -1" in output or "Failed" in output:
+            return _create_fallback_screenshot(is_sensitive=True)
+
+        # Pull screenshot to local temp path
+        subprocess.run(
+            adb_prefix + ["pull", "/sdcard/tmp.png", temp_path],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+
+        if not os.path.exists(temp_path):
+            return _create_fallback_screenshot(is_sensitive=False)
+
+        # Read and encode image
+        img = Image.open(temp_path)
+        width, height = img.size
+
+        buffered = BytesIO()
+        img.save(buffered, format="PNG")
+        base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+        # Cleanup
+        os.remove(temp_path)
+
+        return Screenshot(
+            base64_data=base64_data, width=width, height=height, is_sensitive=False
+        )
+
+    except Exception as e:
+        print(f"Screenshot error: {e}")
+        return _create_fallback_screenshot(is_sensitive=False)
+
+
+def _get_adb_prefix(device_id: str | None) -> list:
+    """Get ADB command prefix with optional device specifier."""
+    if device_id:
+        return ["adb", "-s", device_id]
+    return ["adb"]
+
+
+def _create_fallback_screenshot(is_sensitive: bool) -> Screenshot:
+    """Create a black fallback image when screenshot fails."""
+    default_width, default_height = 1080, 2400
+
+    black_img = Image.new("RGB", (default_width, default_height), color="black")
+    buffered = BytesIO()
+    black_img.save(buffered, format="PNG")
+    base64_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+    return Screenshot(
+        base64_data=base64_data,
+        width=default_width,
+        height=default_height,
+        is_sensitive=is_sensitive,
+    )

phone_agent/agent.py

@@ -0,0 +1,253 @@
+"""Main PhoneAgent class for orchestrating phone automation."""
+
+import json
+import traceback
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from phone_agent.actions import ActionHandler
+from phone_agent.actions.handler import do, finish, parse_action
+from phone_agent.adb import get_current_app, get_screenshot
+from phone_agent.config import get_messages, get_system_prompt
+from phone_agent.model import ModelClient, ModelConfig
+from phone_agent.model.client import MessageBuilder
+
+
+@dataclass
+class AgentConfig:
+    """Configuration for the PhoneAgent."""
+
+    max_steps: int = 100
+    device_id: str | None = None
+    lang: str = "cn"
+    system_prompt: str | None = None
+    verbose: bool = True
+
+    def __post_init__(self):
+        if self.system_prompt is None:
+            self.system_prompt = get_system_prompt(self.lang)
+
+
+@dataclass
+class StepResult:
+    """Result of a single agent step."""
+
+    success: bool
+    finished: bool
+    action: dict[str, Any] | None
+    thinking: str
+    message: str | None = None
+
+
+class PhoneAgent:
+    """
+    AI-powered agent for automating Android phone interactions.
+
+    The agent uses a vision-language model to understand screen content
+    and decide on actions to complete user tasks.
+
+    Args:
+        model_config: Configuration for the AI model.
+        agent_config: Configuration for the agent behavior.
+        confirmation_callback: Optional callback for sensitive action confirmation.
+        takeover_callback: Optional callback for takeover requests.
+
+    Example:
+        >>> from phone_agent import PhoneAgent
+        >>> from phone_agent.model import ModelConfig
+        >>>
+        >>> model_config = ModelConfig(base_url="http://localhost:8000/v1")
+        >>> agent = PhoneAgent(model_config)
+        >>> agent.run("Open WeChat and send a message to John")
+    """
+
+    def __init__(
+        self,
+        model_config: ModelConfig | None = None,
+        agent_config: AgentConfig | None = None,
+        confirmation_callback: Callable[[str], bool] | None = None,
+        takeover_callback: Callable[[str], None] | None = None,
+    ):
+        self.model_config = model_config or ModelConfig()
+        self.agent_config = agent_config or AgentConfig()
+
+        self.model_client = ModelClient(self.model_config)
+        self.action_handler = ActionHandler(
+            device_id=self.agent_config.device_id,
+            confirmation_callback=confirmation_callback,
+            takeover_callback=takeover_callback,
+        )
+
+        self._context: list[dict[str, Any]] = []
+        self._step_count = 0
+
+    def run(self, task: str) -> str:
+        """
+        Run the agent to complete a task.
+
+        Args:
+            task: Natural language description of the task.
+
+        Returns:
+            Final message from the agent.
+        """
+        self._context = []
+        self._step_count = 0
+
+        # First step with user prompt
+        result = self._execute_step(task, is_first=True)
+
+        if result.finished:
+            return result.message or "Task completed"
+
+        # Continue until finished or max steps reached
+        while self._step_count < self.agent_config.max_steps:
+            result = self._execute_step(is_first=False)
+
+            if result.finished:
+                return result.message or "Task completed"
+
+        return "Max steps reached"
+
+    def step(self, task: str | None = None) -> StepResult:
+        """
+        Execute a single step of the agent.
+
+        Useful for manual control or debugging.
+
+        Args:
+            task: Task description (only needed for first step).
+
+        Returns:
+            StepResult with step details.
+        """
+        is_first = len(self._context) == 0
+
+        if is_first and not task:
+            raise ValueError("Task is required for the first step")
+
+        return self._execute_step(task, is_first)
+
+    def reset(self) -> None:
+        """Reset the agent state for a new task."""
+        self._context = []
+        self._step_count = 0
+
+    def _execute_step(
+        self, user_prompt: str | None = None, is_first: bool = False
+    ) -> StepResult:
+        """Execute a single step of the agent loop."""
+        self._step_count += 1
+
+        # Capture current screen state
+        screenshot = get_screenshot(self.agent_config.device_id)
+        current_app = get_current_app(self.agent_config.device_id)
+
+        # Build messages
+        if is_first:
+            self._context.append(
+                MessageBuilder.create_system_message(self.agent_config.system_prompt)
+            )
+
+            screen_info = MessageBuilder.build_screen_info(current_app)
+            text_content = f"{user_prompt}\n\n{screen_info}"
+
+            self._context.append(
+                MessageBuilder.create_user_message(
+                    text=text_content, image_base64=screenshot.base64_data
+                )
+            )
+        else:
+            screen_info = MessageBuilder.build_screen_info(current_app)
+            text_content = f"** Screen Info **\n\n{screen_info}"
+
+            self._context.append(
+                MessageBuilder.create_user_message(
+                    text=text_content, image_base64=screenshot.base64_data
+                )
+            )
+
+        # Get model response
+        try:
+            response = self.model_client.request(self._context)
+        except Exception as e:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            return StepResult(
+                success=False,
+                finished=True,
+                action=None,
+                thinking="",
+                message=f"Model error: {e}",
+            )
+
+        # Parse action from response
+        try:
+            action = parse_action(response.action)
+        except ValueError:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            action = finish(message=response.action)
+
+        if self.agent_config.verbose:
+            # Print thinking process
+            msgs = get_messages(self.agent_config.lang)
+            print("\n" + "=" * 50)
+            print(f"💭 {msgs['thinking']}:")
+            print("-" * 50)
+            print(response.thinking)
+            print("-" * 50)
+            print(f"🎯 {msgs['action']}:")
+            print(json.dumps(action, ensure_ascii=False, indent=2))
+            print("=" * 50 + "\n")
+
+        # Remove image from context to save space
+        self._context[-1] = MessageBuilder.remove_images_from_message(self._context[-1])
+
+        # Execute action
+        try:
+            result = self.action_handler.execute(
+                action, screenshot.width, screenshot.height
+            )
+        except Exception as e:
+            if self.agent_config.verbose:
+                traceback.print_exc()
+            result = self.action_handler.execute(
+                finish(message=str(e)), screenshot.width, screenshot.height
+            )
+
+        # Add assistant response to context
+        self._context.append(
+            MessageBuilder.create_assistant_message(
+                f"<think>{response.thinking}</think><answer>{response.action}</answer>"
+            )
+        )
+
+        # Check if finished
+        finished = action.get("_metadata") == "finish" or result.should_finish
+
+        if finished and self.agent_config.verbose:
+            msgs = get_messages(self.agent_config.lang)
+            print("\n" + "🎉 " + "=" * 48)
+            print(
+                f"✅ {msgs['task_completed']}: {result.message or action.get('message', msgs['done'])}"
+            )
+            print("=" * 50 + "\n")
+
+        return StepResult(
+            success=result.success,
+            finished=finished,
+            action=action,
+            thinking=response.thinking,
+            message=result.message or action.get("message"),
+        )
+
+    @property
+    def context(self) -> list[dict[str, Any]]:
+        """Get the current conversation context."""
+        return self._context.copy()
+
+    @property
+    def step_count(self) -> int:
+        """Get the current step count."""
+        return self._step_count

phone_agent/config/__init__.py

@@ -0,0 +1,35 @@
+"""Configuration module for Phone Agent."""
+
+from phone_agent.config.apps import APP_PACKAGES
+from phone_agent.config.i18n import get_message, get_messages
+from phone_agent.config.prompts_en import SYSTEM_PROMPT as SYSTEM_PROMPT_EN
+from phone_agent.config.prompts_zh import SYSTEM_PROMPT as SYSTEM_PROMPT_ZH
+
+
+def get_system_prompt(lang: str = "cn") -> str:
+    """
+    Get system prompt by language.
+
+    Args:
+        lang: Language code, 'cn' for Chinese, 'en' for English.
+
+    Returns:
+        System prompt string.
+    """
+    if lang == "en":
+        return SYSTEM_PROMPT_EN
+    return SYSTEM_PROMPT_ZH
+
+
+# Default to Chinese for backward compatibility
+SYSTEM_PROMPT = SYSTEM_PROMPT_ZH
+
+__all__ = [
+    "APP_PACKAGES",
+    "SYSTEM_PROMPT",
+    "SYSTEM_PROMPT_ZH",
+    "SYSTEM_PROMPT_EN",
+    "get_system_prompt",
+    "get_messages",
+    "get_message",
+]

phone_agent/config/apps.py

@@ -0,0 +1,227 @@
+"""App name to package name mapping for supported applications."""
+
+APP_PACKAGES: dict[str, str] = {
+    # Social & Messaging
+    "微信": "com.tencent.mm",
+    "QQ": "com.tencent.mobileqq",
+    "微博": "com.sina.weibo",
+    # E-commerce
+    "淘宝": "com.taobao.taobao",
+    "京东": "com.jingdong.app.mall",
+    "拼多多": "com.xunmeng.pinduoduo",
+    "淘宝闪购": "com.taobao.taobao",
+    "京东秒送": "com.jingdong.app.mall",
+    # Lifestyle & Social
+    "小红书": "com.xingin.xhs",
+    "豆瓣": "com.douban.frodo",
+    "知乎": "com.zhihu.android",
+    # Maps & Navigation
+    "高德地图": "com.autonavi.minimap",
+    "百度地图": "com.baidu.BaiduMap",
+    # Food & Services
+    "美团": "com.sankuai.meituan",
+    "大众点评": "com.dianping.v1",
+    "饿了么": "me.ele",
+    "肯德基": "com.yek.android.kfc.activitys",
+    # Travel
+    "携程": "ctrip.android.view",
+    "铁路12306": "com.MobileTicket",
+    "12306": "com.MobileTicket",
+    "去哪儿": "com.Qunar",
+    "去哪儿旅行": "com.Qunar",
+    "滴滴出行": "com.sdu.did.psnger",
+    # Video & Entertainment
+    "bilibili": "tv.danmaku.bili",
+    "抖音": "com.ss.android.ugc.aweme",
+    "快手": "com.smile.gifmaker",
+    "腾讯视频": "com.tencent.qqlive",
+    "爱奇艺": "com.qiyi.video",
+    "优酷视频": "com.youku.phone",
+    "芒果TV": "com.hunantv.imgo.activity",
+    "红果短剧": "com.phoenix.read",
+    # Music & Audio
+    "网易云音乐": "com.netease.cloudmusic",
+    "QQ音乐": "com.tencent.qqmusic",
+    "汽水音乐": "com.luna.music",
+    "喜马拉雅": "com.ximalaya.ting.android",
+    # Reading
+    "番茄小说": "com.dragon.read",
+    "番茄免费小说": "com.dragon.read",
+    "七猫免费小说": "com.kmxs.reader",
+    # Productivity
+    "飞书": "com.ss.android.lark",
+    "QQ邮箱": "com.tencent.androidqqmail",
+    # AI & Tools
+    "豆包": "com.larus.nova",
+    # Health & Fitness
+    "keep": "com.gotokeep.keep",
+    "美柚": "com.lingan.seeyou",
+    # News & Information
+    "腾讯新闻": "com.tencent.news",
+    "今日头条": "com.ss.android.article.news",
+    # Real Estate
+    "贝壳找房": "com.lianjia.beike",
+    "安居客": "com.anjuke.android.app",
+    # Finance
+    "同花顺": "com.hexin.plat.android",
+    # Games
+    "星穹铁道": "com.miHoYo.hkrpg",
+    "崩坏：星穹铁道": "com.miHoYo.hkrpg",
+    "恋与深空": "com.papegames.lysk.cn",
+    "AndroidSystemSettings": "com.android.settings",
+    "Android System Settings": "com.android.settings",
+    "Android  System Settings": "com.android.settings",
+    "Android-System-Settings": "com.android.settings",
+    "Settings": "com.android.settings",
+    "AudioRecorder": "com.android.soundrecorder",
+    "audiorecorder": "com.android.soundrecorder",
+    "Bluecoins": "com.rammigsoftware.bluecoins",
+    "bluecoins": "com.rammigsoftware.bluecoins",
+    "Broccoli": "com.flauschcode.broccoli",
+    "broccoli": "com.flauschcode.broccoli",
+    "Booking.com": "com.booking",
+    "Booking": "com.booking",
+    "booking.com": "com.booking",
+    "booking": "com.booking",
+    "BOOKING.COM": "com.booking",
+    "Chrome": "com.android.chrome",
+    "chrome": "com.android.chrome",
+    "Google Chrome": "com.android.chrome",
+    "Clock": "com.android.deskclock",
+    "clock": "com.android.deskclock",
+    "Contacts": "com.android.contacts",
+    "contacts": "com.android.contacts",
+    "Duolingo": "com.duolingo",
+    "duolingo": "com.duolingo",
+    "Expedia": "com.expedia.bookings",
+    "expedia": "com.expedia.bookings",
+    "Files": "com.android.fileexplorer",
+    "files": "com.android.fileexplorer",
+    "File Manager": "com.android.fileexplorer",
+    "file manager": "com.android.fileexplorer",
+    "gmail": "com.google.android.gm",
+    "Gmail": "com.google.android.gm",
+    "GoogleMail": "com.google.android.gm",
+    "Google Mail": "com.google.android.gm",
+    "GoogleFiles": "com.google.android.apps.nbu.files",
+    "googlefiles": "com.google.android.apps.nbu.files",
+    "FilesbyGoogle": "com.google.android.apps.nbu.files",
+    "GoogleCalendar": "com.google.android.calendar",
+    "Google-Calendar": "com.google.android.calendar",
+    "Google Calendar": "com.google.android.calendar",
+    "google-calendar": "com.google.android.calendar",
+    "google calendar": "com.google.android.calendar",
+    "GoogleChat": "com.google.android.apps.dynamite",
+    "Google Chat": "com.google.android.apps.dynamite",
+    "Google-Chat": "com.google.android.apps.dynamite",
+    "GoogleClock": "com.google.android.deskclock",
+    "Google Clock": "com.google.android.deskclock",
+    "Google-Clock": "com.google.android.deskclock",
+    "GoogleContacts": "com.google.android.contacts",
+    "Google-Contacts": "com.google.android.contacts",
+    "Google Contacts": "com.google.android.contacts",
+    "google-contacts": "com.google.android.contacts",
+    "google contacts": "com.google.android.contacts",
+    "GoogleDocs": "com.google.android.apps.docs.editors.docs",
+    "Google Docs": "com.google.android.apps.docs.editors.docs",
+    "googledocs": "com.google.android.apps.docs.editors.docs",
+    "google docs": "com.google.android.apps.docs.editors.docs",
+    "Google Drive": "com.google.android.apps.docs",
+    "Google-Drive": "com.google.android.apps.docs",
+    "google drive": "com.google.android.apps.docs",
+    "google-drive": "com.google.android.apps.docs",
+    "GoogleDrive": "com.google.android.apps.docs",
+    "Googledrive": "com.google.android.apps.docs",
+    "googledrive": "com.google.android.apps.docs",
+    "GoogleFit": "com.google.android.apps.fitness",
+    "googlefit": "com.google.android.apps.fitness",
+    "GoogleKeep": "com.google.android.keep",
+    "googlekeep": "com.google.android.keep",
+    "GoogleMaps": "com.google.android.apps.maps",
+    "Google Maps": "com.google.android.apps.maps",
+    "googlemaps": "com.google.android.apps.maps",
+    "google maps": "com.google.android.apps.maps",
+    "Google Play Books": "com.google.android.apps.books",
+    "Google-Play-Books": "com.google.android.apps.books",
+    "google play books": "com.google.android.apps.books",
+    "google-play-books": "com.google.android.apps.books",
+    "GooglePlayBooks": "com.google.android.apps.books",
+    "googleplaybooks": "com.google.android.apps.books",
+    "GooglePlayStore": "com.android.vending",
+    "Google Play Store": "com.android.vending",
+    "Google-Play-Store": "com.android.vending",
+    "GoogleSlides": "com.google.android.apps.docs.editors.slides",
+    "Google Slides": "com.google.android.apps.docs.editors.slides",
+    "Google-Slides": "com.google.android.apps.docs.editors.slides",
+    "GoogleTasks": "com.google.android.apps.tasks",
+    "Google Tasks": "com.google.android.apps.tasks",
+    "Google-Tasks": "com.google.android.apps.tasks",
+    "Joplin": "net.cozic.joplin",
+    "joplin": "net.cozic.joplin",
+    "McDonald": "com.mcdonalds.app",
+    "mcdonald": "com.mcdonalds.app",
+    "Osmand": "net.osmand",
+    "osmand": "net.osmand",
+    "PiMusicPlayer": "com.Project100Pi.themusicplayer",
+    "pimusicplayer": "com.Project100Pi.themusicplayer",
+    "Quora": "com.quora.android",
+    "quora": "com.quora.android",
+    "Reddit": "com.reddit.frontpage",
+    "reddit": "com.reddit.frontpage",
+    "RetroMusic": "code.name.monkey.retromusic",
+    "retromusic": "code.name.monkey.retromusic",
+    "SimpleCalendarPro": "com.scientificcalculatorplus.simplecalculator.basiccalculator.mathcalc",
+    "SimpleSMSMessenger": "com.simplemobiletools.smsmessenger",
+    "Telegram": "org.telegram.messenger",
+    "temu": "com.einnovation.temu",
+    "Temu": "com.einnovation.temu",
+    "Tiktok": "com.zhiliaoapp.musically",
+    "tiktok": "com.zhiliaoapp.musically",
+    "Twitter": "com.twitter.android",
+    "twitter": "com.twitter.android",
+    "X": "com.twitter.android",
+    "VLC": "org.videolan.vlc",
+    "WeChat": "com.tencent.mm",
+    "wechat": "com.tencent.mm",
+    "Whatsapp": "com.whatsapp",
+    "WhatsApp": "com.whatsapp",
+}
+
+
+def get_package_name(app_name: str) -> str | None:
+    """
+    Get the package name for an app.
+
+    Args:
+        app_name: The display name of the app.
+
+    Returns:
+        The Android package name, or None if not found.
+    """
+    return APP_PACKAGES.get(app_name)
+
+
+def get_app_name(package_name: str) -> str | None:
+    """
+    Get the app name from a package name.
+
+    Args:
+        package_name: The Android package name.
+
+    Returns:
+        The display name of the app, or None if not found.
+    """
+    for name, package in APP_PACKAGES.items():
+        if package == package_name:
+            return name
+    return None
+
+
+def list_supported_apps() -> list[str]:
+    """
+    Get a list of all supported app names.
+
+    Returns:
+        List of app names.
+    """
+    return list(APP_PACKAGES.keys())