screenforge 0.4.0__py3-none-any.whl

common/tool_protocol.py

@@ -0,0 +1,196 @@
+import json
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field, ValidationError, model_validator
+
+from common.capabilities import (
+    ACTIONS_REQUIRING_EXTRA_VALUE,
+    GLOBAL_ACTIONS,
+    SUPPORTED_ACTIONS,
+    SUPPORTED_PLATFORMS,
+    get_capabilities_payload,
+)
+from common.runtime_modes import MODE_DOCTOR, MODE_DRY_RUN, MODE_PLAN_ONLY, MODE_RUN
+
+SUPPORTED_TOOL_MODES = (MODE_RUN, MODE_DOCTOR, MODE_PLAN_ONLY, MODE_DRY_RUN)
+
+
+class ToolRequestError(ValueError):
+    pass
+
+
+class WorkflowToolControl(BaseModel):
+    path: str
+    vars: dict[str, str] = Field(default_factory=dict)
+
+
+class ActionToolControl(BaseModel):
+    action: str
+    action_name: str = ""
+    locator_type: str = ""
+    locator_value: str = ""
+    extra_value: str = ""
+
+    @model_validator(mode="after")
+    def validate_action(self):
+        if self.action not in SUPPORTED_ACTIONS:
+            raise ValueError(f"Unsupported action: {self.action}")
+        # Use the shared GLOBAL_ACTIONS set, not a hardcoded literal: a global
+        # action (goto/swipe/press/assert_url) needs no locator. A literal here
+        # drifts from the canonical set (e.g. it wrongly demanded a locator for
+        # assert_url, which reads page.url).
+        if self.action not in GLOBAL_ACTIONS:
+            if not str(self.locator_type).strip() or not str(self.locator_value).strip():
+                raise ValueError("Element actions require locator_type and locator_value")
+        if self.action in ACTIONS_REQUIRING_EXTRA_VALUE and not str(self.extra_value).strip():
+            raise ValueError(f"Action '{self.action}' requires extra_value")
+        return self
+
+
+class ToolRequest(BaseModel):
+    operation: Literal[
+        "capabilities",
+        "execute",
+        "load_run",
+        "inspect_ui",
+        "load_case_memory",
+    ]
+    mode: Literal["run", "doctor", "plan_only", "dry_run"] = MODE_RUN
+    platform: str = ""
+    env: str = "dev"
+    vision: bool = False
+    context: str = ""
+    output: str = ""
+    resume_run_id: str = ""
+    run_id: str = ""
+    goal: str = ""
+    query: str = ""
+    source_ref: str = ""
+    control_kind: str = ""
+    limit: int = 20
+    workflow: WorkflowToolControl | None = None
+    action: ActionToolControl | None = None
+
+    @model_validator(mode="after")
+    def validate_request(self):
+        if self.operation == "capabilities":
+            return self
+
+        if _normalize_text := str(self.goal).strip():
+            if self.operation == "execute":
+                raise ValueError(
+                    "Execute tool request does not accept goal; the calling Agent should interpret natural language and pass workflow or action"
+                )
+            raise ValueError(f"{self.operation} tool request cannot include goal")
+
+        if self.operation in {"execute", "inspect_ui"} and self.platform not in SUPPORTED_PLATFORMS:
+            raise ValueError(f"Unsupported platform: {self.platform}")
+
+        control_count = int(self.workflow is not None) + int(self.action is not None)
+        if self.operation == "load_run":
+            if control_count:
+                raise ValueError("load_run tool request cannot include workflow or action")
+            if not str(self.run_id).strip():
+                raise ValueError("load_run tool request requires run_id")
+            return self
+        if self.operation == "load_case_memory":
+            if control_count:
+                raise ValueError("load_case_memory tool request cannot include workflow or action")
+            return self
+        if self.operation == "inspect_ui":
+            if control_count:
+                raise ValueError("inspect_ui tool request cannot include workflow or action")
+            return self
+
+        if self.mode == MODE_DOCTOR:
+            if control_count:
+                raise ValueError("doctor tool request cannot include workflow or action")
+            return self
+
+        if control_count != 1:
+            raise ValueError(
+                "Execute tool request must provide exactly one control: workflow or action"
+            )
+        return self
+
+
+def _validate_tool_request_payload(payload: dict, source_label: str) -> ToolRequest:
+    try:
+        return ToolRequest.model_validate(payload)
+    except ValidationError as exc:
+        raise ToolRequestError(f"{source_label} validation failed: {exc}") from exc
+
+
+def load_tool_request(file_path: str | Path) -> ToolRequest:
+    path = Path(file_path).expanduser().resolve()
+    if not path.exists():
+        raise ToolRequestError(f"Tool request file not found: {path}")
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ToolRequestError(f"Tool request JSON parse error: {exc}") from exc
+
+    return _validate_tool_request_payload(payload, "tool request")
+
+
+def load_tool_request_from_stdin(raw_text: str) -> ToolRequest:
+    if not str(raw_text).strip():
+        raise ToolRequestError("Tool stdin is empty")
+
+    try:
+        payload = json.loads(raw_text)
+    except json.JSONDecodeError as exc:
+        raise ToolRequestError(f"Tool stdin JSON parse error: {exc}") from exc
+
+    return _validate_tool_request_payload(payload, "tool stdin")
+
+
+def build_cli_arg_overrides(request: ToolRequest) -> dict:
+    overrides = {
+        "goal": "",
+        "context": request.context,
+        "env": request.env,
+        "platform": request.platform,
+        "vision": request.vision,
+        "json": False,
+        "doctor": request.mode == MODE_DOCTOR,
+        "plan_only": request.mode == MODE_PLAN_ONLY,
+        "dry_run": request.mode == MODE_DRY_RUN,
+        "resume_run_id": request.resume_run_id,
+        "workflow": "",
+        "workflow_var": [],
+        "action": "",
+        "action_name": "",
+        "locator_type": "",
+        "locator_value": "",
+        "extra_value": "",
+        "output": request.output,
+        "capabilities": False,
+        "tool_request": "",
+        "tool_stdin": False,
+        "mcp_server": False,
+    }
+
+    if request.workflow:
+        overrides["workflow"] = request.workflow.path
+        overrides["workflow_var"] = [
+            f"{key}={value}" for key, value in request.workflow.vars.items()
+        ]
+    elif request.action:
+        overrides["action"] = request.action.action
+        overrides["action_name"] = request.action.action_name
+        overrides["locator_type"] = request.action.locator_type
+        overrides["locator_value"] = request.action.locator_value
+        overrides["extra_value"] = request.action.extra_value
+
+    return overrides
+
+
+def build_capabilities_response() -> dict:
+    return {
+        "ok": True,
+        "operation": "capabilities",
+        "capabilities": get_capabilities_payload(),
+    }

common/visual_fallback.py

@@ -0,0 +1,71 @@
+"""Visual fallback: locate elements via VLM screenshot when DOM lookup fails."""
+
+import json
+import re
+
+from common.logs import log
+
+
+def visual_locate(screenshot_bytes: bytes, description: str) -> tuple[int, int] | None:
+    import base64
+
+    import config.config as config
+
+    try:
+        from openai import OpenAI
+    except ImportError:
+        log.error("[Visual Fallback] openai package not installed")
+        return None
+
+    screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+
+    prompt = (
+        f"你是一个 UI 元素定位专家。用户在页面上找不到元素: {description}\n"
+        "请在截图中找到最匹配的元素, 返回该元素中心点的像素坐标。\n"
+        "只返回 JSON, 格式: {\"x\": 数字, \"y\": 数字}\n"
+        "如果找不到, 返回: {\"x\": -1, \"y\": -1}"
+    )
+
+    try:
+        client = OpenAI(
+            api_key=config.VISION_API_KEY,
+            base_url=config.VISION_BASE_URL,
+        )
+        response = client.chat.completions.create(
+            model=config.VISION_MODEL_NAME,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": prompt},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+                        },
+                    ],
+                }
+            ],
+            max_tokens=100,
+            temperature=0,
+        )
+
+        raw = response.choices[0].message.content.strip()
+        json_match = re.search(r"\{[^}]+\}", raw)
+        if not json_match:
+            log.warning(f"[Visual Fallback] VLM response not parseable: {raw[:200]}")
+            return None
+
+        coords = json.loads(json_match.group())
+        x = int(coords.get("x", -1))
+        y = int(coords.get("y", -1))
+
+        if x < 0 or y < 0:
+            log.warning("[Visual Fallback] VLM could not locate target element")
+            return None
+
+        log.info(f"[Visual Fallback] VLM located element at ({x}, {y})")
+        return (x, y)
+
+    except Exception as e:
+        log.error(f"[Visual Fallback] VLM call failed: {e}")
+        return None

common/workflow_schema.py

@@ -0,0 +1,150 @@
+import re
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, Field, model_validator
+
+from common.capabilities import GLOBAL_ACTIONS, SUPPORTED_ACTIONS
+
+SUPPORTED_WORKFLOW_ACTIONS = SUPPORTED_ACTIONS
+GLOBAL_WORKFLOW_ACTIONS = GLOBAL_ACTIONS
+WORKFLOW_VAR_PATTERN = re.compile(r"{{\s*([a-zA-Z0-9_\-]+)\s*}}")
+
+
+class WorkflowLoadError(ValueError):
+    pass
+
+
+class WorkflowStep(BaseModel):
+    name: str = ""
+    action: Literal[
+        "goto",
+        "click",
+        "long_click",
+        "hover",
+        "input",
+        "swipe",
+        "press",
+        "scroll_into_view",
+        "select",
+        "upload",
+        "double_click",
+        "right_click",
+        "drag",
+        "wait_for",
+        "assert_exist",
+        "assert_not_exist",
+        "assert_text_equals",
+        "assert_text_contains",
+        "assert_value",
+        "assert_url",
+    ]
+    locator_type: str = "global"
+    locator_value: str = "global"
+    extra_value: str = ""
+    enabled: bool = True
+
+    @model_validator(mode="after")
+    def validate_locator(self):
+        if not self.enabled:
+            return self
+
+        if self.action in GLOBAL_WORKFLOW_ACTIONS:
+            if not str(self.locator_type).strip():
+                self.locator_type = "global"
+            if not str(self.locator_value).strip():
+                self.locator_value = "global"
+            return self
+
+        if not str(self.locator_type).strip() or not str(self.locator_value).strip():
+            raise ValueError("Element workflow steps require locator_type and locator_value")
+
+        return self
+
+
+class WorkflowDefinition(BaseModel):
+    version: int = 1
+    name: str = ""
+    platform: str = ""
+    env: str = ""
+    vars: dict[str, str] = Field(default_factory=dict)
+    steps: list[WorkflowStep] = Field(min_length=1)
+
+
+def load_workflow_file(file_path: str | Path) -> WorkflowDefinition:
+    path = Path(file_path).expanduser().resolve()
+    if not path.exists():
+        raise WorkflowLoadError(f"Workflow file not found: {path}")
+
+    try:
+        payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as exc:
+        raise WorkflowLoadError(f"Workflow YAML parse error: {exc}") from exc
+
+    if not isinstance(payload, dict):
+        raise WorkflowLoadError("Workflow file top-level must be an object")
+
+    try:
+        return WorkflowDefinition.model_validate(payload)
+    except Exception as exc:
+        raise WorkflowLoadError(f"Workflow validation failed: {exc}") from exc
+
+
+def parse_workflow_var_overrides(items: list[str] | None) -> dict[str, str]:
+    resolved = {}
+    for item in items or []:
+        if "=" not in str(item):
+            raise WorkflowLoadError("Workflow variable override format must be KEY=VALUE")
+        key, value = str(item).split("=", 1)
+        key = key.strip()
+        if not key:
+            raise WorkflowLoadError("Workflow variable name cannot be empty")
+        resolved[key] = value
+    return resolved
+
+
+def _render_template(value: str, variables: dict[str, str]) -> str:
+    text = str(value)
+
+    def replace(match: re.Match[str]) -> str:
+        key = match.group(1)
+        if key not in variables:
+            raise WorkflowLoadError(f"Workflow references undefined variable: {key}")
+        return str(variables[key])
+
+    return WORKFLOW_VAR_PATTERN.sub(replace, text)
+
+
+def resolve_workflow_definition(
+    workflow: WorkflowDefinition,
+    overrides: dict[str, str] | None = None,
+) -> WorkflowDefinition:
+    variables = {key: str(value) for key, value in workflow.vars.items()}
+    variables.update({key: str(value) for key, value in (overrides or {}).items()})
+
+    resolved_steps = []
+    for step in workflow.steps:
+        resolved_steps.append(
+            WorkflowStep.model_validate(
+                {
+                    "name": _render_template(step.name, variables),
+                    "action": step.action,
+                    "locator_type": _render_template(step.locator_type, variables),
+                    "locator_value": _render_template(step.locator_value, variables),
+                    "extra_value": _render_template(step.extra_value, variables),
+                    "enabled": step.enabled,
+                }
+            )
+        )
+
+    return WorkflowDefinition.model_validate(
+        {
+            "version": workflow.version,
+            "name": _render_template(workflow.name, variables),
+            "platform": _render_template(workflow.platform, variables),
+            "env": _render_template(workflow.env, variables),
+            "vars": variables,
+            "steps": [step.model_dump() for step in resolved_steps],
+        }
+    )

config/config.py

@@ -0,0 +1,167 @@
+import os
+from pathlib import Path
+
+from common.logs import log
+from config.env_loader import resolve_dotenv_path, safe_load_dotenv
+
+# ==========================================
+# 1. 基础路径与 .env 自动加载 (工程化核心)
+# ==========================================
+# 动态获取项目根目录 (config.py 的上一级目录)
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+# 尝试寻找并加载根目录或上层主仓库中的 .env/.ENV 文件到系统环境变量中
+# override=False 表示如果宿主系统已经配置了该变量(如在 CI/CD 流水线中)，则以系统优先
+env_path = resolve_dotenv_path(BASE_DIR)
+safe_load_dotenv(dotenv_path=env_path, override=False)
+
+# ==========================================
+# 2. 文本大模型配置 (用于处理纯 XML 树，高频、廉价、快速)
+# ==========================================
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
+OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o")
+
+# ==========================================
+# 3. 多模态视觉大模型配置 (用于处理屏幕截图，低频、复杂场景辅助)
+# ==========================================
+# 默认 fallback 到文本模型的配置，实现优雅降级；若配置了则实现异构解耦
+VISION_API_KEY = os.getenv("VISION_API_KEY", OPENAI_API_KEY)
+VISION_BASE_URL = os.getenv("VISION_BASE_URL", OPENAI_BASE_URL)
+VISION_MODEL_NAME = os.getenv("VISION_MODEL_NAME", MODEL_NAME)
+
+# ==========================================
+# 4. 自动化自愈配置 (Self-Healing)
+# ==========================================
+# 开启 AI 自动修复失败用例功能
+AUTO_HEAL_ENABLED = str(os.getenv("AUTO_HEAL_ENABLED", "True")).lower() in ('true', '1', 'yes')
+# 连续失败多少次后触发自愈
+AUTO_HEAL_TRIGGER_THRESHOLD = int(os.getenv("AUTO_HEAL_TRIGGER_THRESHOLD", 2))
+# 自愈置信度阈值 — 低于此值的修复方案将被丢弃
+AUTO_HEAL_MIN_CONFIDENCE = float(os.getenv("AUTO_HEAL_MIN_CONFIDENCE", 0.7))
+
+# ==========================================
+# 5. 自动化测试框架配置
+# ==========================================
+# 使用绝对路径，彻底杜绝不同命令路径下生成文件夹位置错乱的问题
+OUTPUT_SCRIPT_FILE = str(BASE_DIR / "test_cases" / "test_auto_generated.py")
+
+# 全局隐式等待时间，强转为 float 确保安全
+DEFAULT_TIMEOUT = float(os.getenv("DEFAULT_TIMEOUT", 30.0))
+
+# ==========================================
+# 6. App 环境配置
+# ==========================================
+APP_ENV_CONFIG = {
+    "dev": {
+        "android": "",
+        "ios": "",
+        "web": "",
+    },
+}
+
+# ==========================================
+# 7. 本地语义缓存配置
+# ==========================================
+# 强转布尔值，兼容 .env 中的 True/true/1/yes
+CACHE_ENABLED = str(os.getenv("CACHE_ENABLED", "True")).lower() in ('true', '1', 't', 'yes')
+CACHE_DIR = str(BASE_DIR / '.cache')
+CACHE_TTL_DAYS = int(os.getenv("CACHE_TTL_DAYS", 7))
+CACHE_MAX_SIZE_MB = int(os.getenv("CACHE_MAX_SIZE_MB", 100))
+CACHE_COMPRESSION = str(os.getenv("CACHE_COMPRESSION", "False")).lower() in ('true', '1', 't', 'yes')
+
+CACHE_SIMILARITY_THRESHOLD = float(os.getenv("CACHE_SIMILARITY_THRESHOLD", "0.90"))
+CACHE_EXACT_MATCH_THRESHOLD = float(os.getenv("CACHE_EXACT_MATCH_THRESHOLD", "0.98"))
+
+
+# ==========================================
+# 8. Web CDP 连接配置
+# ==========================================
+WEB_CDP_URL = os.getenv("WEB_CDP_URL", "http://localhost:9222")
+
+# ==========================================
+# 8a. Android 设备连接配置
+# ==========================================
+ANDROID_SERIAL = os.getenv("ANDROID_SERIAL", "")
+ANDROID_CONNECT_TIMEOUT = float(os.getenv("ANDROID_CONNECT_TIMEOUT", "10.0"))
+
+# ==========================================
+# 8b. iOS 设备连接配置
+# ==========================================
+WDA_URL = os.getenv("WDA_URL", "http://localhost:8100")
+IOS_DEVICE_UDID = os.getenv("IOS_DEVICE_UDID", "")
+
+# ==========================================
+# 9. Agent 运行产物目录
+# ==========================================
+RUN_REPORT_BASE_DIR = BASE_DIR / "report" / "runs"
+
+# ==========================================
+# 10. 跨运行测试记忆目录
+# ==========================================
+CASE_MEMORY_PATH = Path(os.getenv("CASE_MEMORY_PATH", str(BASE_DIR / "memory" / "case_memory.json")))
+
+# ==========================================
+# 11. Pytest 真机回放测试控制
+# ==========================================
+TEST_PLATFORM = os.getenv("TEST_PLATFORM", "android").lower()
+RUN_LIVE_PLATFORM_TESTS = str(os.getenv("RUN_LIVE_PLATFORM_TESTS", "False")).lower() in (
+    "true",
+    "1",
+    "t",
+    "yes",
+)
+
+
+def validate_config() -> bool:
+    """Validate required configuration. Returns False and logs errors on failure."""
+    errors: list[tuple[str, str, str]] = []  # (code, what, fix)
+
+    if not OPENAI_API_KEY:
+        errors.append((
+            "E001",
+            "OPENAI_API_KEY is not set.",
+            "Fix: export OPENAI_API_KEY=sk-... or add it to your .env file",
+        ))
+    if DEFAULT_TIMEOUT <= 0:
+        errors.append((
+            "E002",
+            f"DEFAULT_TIMEOUT must be > 0 (current: {DEFAULT_TIMEOUT}).",
+            "Fix: export DEFAULT_TIMEOUT=30",
+        ))
+    if not (0 <= CACHE_SIMILARITY_THRESHOLD <= 1):
+        errors.append((
+            "E003",
+            f"CACHE_SIMILARITY_THRESHOLD must be 0-1 (current: {CACHE_SIMILARITY_THRESHOLD}).",
+            "Fix: export CACHE_SIMILARITY_THRESHOLD=0.90",
+        ))
+    if not (0 <= CACHE_EXACT_MATCH_THRESHOLD <= 1):
+        errors.append((
+            "E004",
+            f"CACHE_EXACT_MATCH_THRESHOLD must be 0-1 (current: {CACHE_EXACT_MATCH_THRESHOLD}).",
+            "Fix: export CACHE_EXACT_MATCH_THRESHOLD=0.98",
+        ))
+    if not WEB_CDP_URL.startswith(("http://", "https://")):
+        errors.append((
+            "E005",
+            f"WEB_CDP_URL must start with http:// or https:// (current: {WEB_CDP_URL}).",
+            "Fix: export WEB_CDP_URL=http://localhost:9222",
+        ))
+    if not (0 <= AUTO_HEAL_MIN_CONFIDENCE <= 1):
+        errors.append((
+            "E006",
+            f"AUTO_HEAL_MIN_CONFIDENCE must be 0-1 (current: {AUTO_HEAL_MIN_CONFIDENCE}).",
+            "Fix: export AUTO_HEAL_MIN_CONFIDENCE=0.7",
+        ))
+    if AUTO_HEAL_TRIGGER_THRESHOLD < 1:
+        errors.append((
+            "E007",
+            f"AUTO_HEAL_TRIGGER_THRESHOLD must be >= 1 (current: {AUTO_HEAL_TRIGGER_THRESHOLD}).",
+            "Fix: export AUTO_HEAL_TRIGGER_THRESHOLD=2",
+        ))
+
+    if errors:
+        for code, what, fix in errors:
+            log.error(f"[{code}] {what} {fix}")
+        return False
+    return True

config/env_loader.py

@@ -0,0 +1,76 @@
+import os
+import shlex
+from pathlib import Path
+
+
+def resolve_dotenv_path(project_root: Path):
+    project_root = Path(project_root).resolve()
+    for candidate_root in [project_root, *project_root.parents]:
+        try:
+            entries = {path.name.lower(): path for path in candidate_root.iterdir()}
+        except OSError:
+            continue
+        for file_name in (".env", ".ENV"):
+            candidate = entries.get(file_name.lower())
+            if candidate and candidate.is_file():
+                return candidate
+    return project_root / ".env"
+
+
+def _parse_env_line(line: str):
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+
+    if line.startswith("export "):
+        line = line[len("export ") :].strip()
+
+    if "=" not in line:
+        return None
+
+    key, value = line.split("=", 1)
+    key = key.strip()
+    value = value.strip()
+    if not key:
+        return None
+
+    if value and value[0] in {"'", '"'}:
+        try:
+            parsed = shlex.split(f"v={value}", posix=True)
+            value = parsed[0].split("=", 1)[1] if parsed else ""
+        except ValueError:
+            if len(value) >= 2 and value[0] == value[-1]:
+                value = value[1:-1]
+    else:
+        value = value.split(" #", 1)[0].rstrip()
+
+    return key, value
+
+
+def _fallback_load_dotenv(dotenv_path: Path, override: bool = False) -> bool:
+    dotenv_path = Path(dotenv_path)
+    if not dotenv_path.exists():
+        return False
+
+    loaded = False
+    for raw_line in dotenv_path.read_text(encoding="utf-8").splitlines():
+        parsed = _parse_env_line(raw_line)
+        if not parsed:
+            continue
+
+        key, value = parsed
+        loaded = True
+        if key in os.environ and not override:
+            continue
+        os.environ[key] = value
+
+    return loaded
+
+
+def safe_load_dotenv(dotenv_path: Path, override: bool = False) -> bool:
+    try:
+        from dotenv import load_dotenv
+
+        return bool(load_dotenv(dotenv_path=dotenv_path, override=override))
+    except ModuleNotFoundError:
+        return _fallback_load_dotenv(dotenv_path, override=override)