screenforge 0.4.0__py3-none-any.whl

screenforge-0.4.0.dist-info/METADATA

@@ -0,0 +1,43 @@
+Metadata-Version: 2.4
+Name: screenforge
+Version: 0.4.0
+Summary: AI-driven cross-platform UI automation engine with test script generation
+License: MIT
+Project-URL: Homepage, https://github.com/jhinzzz/ScreenForge
+Project-URL: Repository, https://github.com/jhinzzz/ScreenForge
+Project-URL: Issues, https://github.com/jhinzzz/ScreenForge/issues
+Requires-Python: >=3.11
+License-File: LICENSE
+Requires-Dist: playwright>=1.50
+Requires-Dist: openai>=2.0
+Requires-Dist: allure-pytest>=2.15
+Requires-Dist: loguru>=0.7
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: pillow>=12.0
+Requires-Dist: lxml>=5.0
+Requires-Dist: requests>=2.30
+Requires-Dist: rich>=14.0
+Requires-Dist: typer>=0.24
+Requires-Dist: retry2>=0.9
+Requires-Dist: opencv-python>=4.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: filelock>=3.12
+Provides-Extra: android
+Requires-Dist: uiautomator2>=3.5; extra == "android"
+Requires-Dist: adbutils>=2.12; extra == "android"
+Provides-Extra: ios
+Requires-Dist: facebook-wda>=1.0; extra == "ios"
+Provides-Extra: ml
+Requires-Dist: torch>=2.0; extra == "ml"
+Requires-Dist: transformers>=5.0; extra == "ml"
+Requires-Dist: sentence-transformers>=5.0; extra == "ml"
+Requires-Dist: scikit-learn>=1.8; extra == "ml"
+Provides-Extra: playground
+Requires-Dist: fastapi>=0.115; extra == "playground"
+Requires-Dist: uvicorn>=0.34; extra == "playground"
+Requires-Dist: websockets>=14.0; extra == "playground"
+Provides-Extra: all
+Requires-Dist: screenforge[android,ios,ml,playground]; extra == "all"
+Dynamic: license-file

screenforge-0.4.0.dist-info/RECORD

@@ -0,0 +1,64 @@
+cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cli/_version.py,sha256=42STGor_9nKYXumfeV5tiyD_M8VdcddX7CEexmibPBk,22
+cli/dispatch.py,sha256=2sOjAvfrn0mwOwUIgETKlGbX2Pw_iBVhp9eKGIQPS3Y,8458
+cli/doctor.py,sha256=uWHxWXl1wive_8XDXpS8j9iK1AxYkQMpZZ9A1uJfX2U,18383
+cli/parser.py,sha256=OtIeDfYMPBy28P5D_A7dz47Whak7UEFlbkWbghtQV5A,10170
+cli/reporter.py,sha256=74TdEl-RcRR6T8RSGAKEvJf7rE1gRrGWXHn_Z_8c9A0,8399
+cli/session.py,sha256=MVu7J_HVhd8MzMRHRtSI3TrlGjcA9t9vTMAMZM6RjI4,4271
+cli/shared.py,sha256=h4I_QkGYqABSAEVo6lPWAjzLKD9HYXj-tXAkecyBdik,15237
+cli/shorthand.py,sha256=Axjc1RsfRxvJBDEEkid0WOSw-bNGNi49kACTj0EnH0Q,3099
+cli/tool_protocol_handlers.py,sha256=7lU_ogNScQNIHjq2EwejK_XdN7ObiyfTmukkXlhQUxY,14885
+cli/modes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cli/modes/action.py,sha256=0XFHPu3iva6ssvm_zVmRfr6rdQ6FgN3gttypboWM1DM,10150
+cli/modes/default.py,sha256=4zmcn3bShqeiNGTqjJsNV38horYfla0ZvKX0hs_d0iw,11211
+cli/modes/demo.py,sha256=Xc79dl9IED60xJcwqj1kC2CBqGSVqrn6j3qrJFFf-XA,5228
+cli/modes/dry_run.py,sha256=ta-0CN1IDWpe9V_QQKqYLtL7VeAFATSlPZ_vfYlkfcU,8706
+cli/modes/init.py,sha256=4Pfd55Ck8EluQGnQ_OHvt_Zin4iWEyXq5PSrNfUE7yM,4226
+cli/modes/plan.py,sha256=-l15staToEpUR9V23gludWA3Ao7oShqe2p_bFSnfNVQ,5095
+cli/modes/workflow.py,sha256=V-uE7LFFtzOfQ_kSMrvH_UVorwdngY-1U9ruyk3O_es,12649
+common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+common/ai.py,sha256=6vYmBPA-0_aCWSJckjIF0QI1SrlcXGlBylUlNXeR73Q,13459
+common/ai_autonomous.py,sha256=zlOMZwWDm6Udw809WDTkr9QC1Re1ESK9iTp9ehIAUdk,12279
+common/ai_heal.py,sha256=apsAizcJPuu2h3gBc0KxB1jEq7OVpX0kI5NN8S4ko7M,7501
+common/capabilities.py,sha256=PXWRdCTMZpiBVxZ9wv8G_mk895Qg_UE2cIJP7WXdX9g,4066
+common/case_memory.py,sha256=3Eo99vS3FPb4kRcUp8SXpHzlnXkXJX1xr1NkQeJt7Bg,11153
+common/error_codes.py,sha256=UAUmEd4bfw62OfdfVZTmCv0v1uJZkBdMK-dkpQqiZwA,2157
+common/exceptions.py,sha256=FXZBAWPkRPuYxtHOZSVKgk-H01zqxm2HHEhpKurhVKg,214
+common/executor.py,sha256=nu7WGzk5prqfU9mFUbZTfczHbzxmgdP0QL9iQm-4bU4,70501
+common/failure_diagnosis.py,sha256=zAor5pPtlYroJ8txdZrQt5HTmjDRyjSaiSN1Qb3dW0k,4758
+common/history_manager.py,sha256=BfiRslOyyhltDRRIg_SurM2pbU5wWfrL3EEvNkFEtWE,3002
+common/logs.py,sha256=bfrbekylNwJ01zeEO2t_pVTwyRKF8-9osKljIRBiUc0,4060
+common/mcp_server.py,sha256=iHTrly4098RR3E3VGfQIigYOkeAZ0-M0x0XZRYcHFzs,16173
+common/preflight.py,sha256=DVeYNjvjVRQtwAfmcl_ebKptz9RG8KeZLPu4VV94hfE,17449
+common/progress.py,sha256=Od9_5VQN1DNza4zbr-B6zp6fWr7-ywKJFxXTDQ94CfE,926
+common/run_reporter.py,sha256=ZK2NRWGNMTumJozFMgSTEpI6EHp3lrtpHxlqEDhP2wc,15870
+common/run_resume.py,sha256=QGn_YRVjTOZD1tdZzlkymdx0g-MLXjGj-E8m4OM8eVI,5400
+common/runtime_modes.py,sha256=n6o9e-A3YUxoa3_f-pzfBhACgVzv_1Ranm7q3VjgmEE,794
+common/tool_protocol.py,sha256=_GVd1GVyLAt8NpfaBA2xND7_ACkl29bb1yMbvYf6J8M,6782
+common/visual_fallback.py,sha256=dBvNOianmrOilWTGrrgmDpd7G3JJF5X7Nm7qi5-psCQ,2267
+common/workflow_schema.py,sha256=-1zCBB0Tea9Rjg1-AM6TKJkwARWNJvDXTojjNs_eSUs,4698
+common/adapters/__init__.py,sha256=EFdBr_meqoMB9Nea_mNuFlqGDHT7hi35b9A3dLFHQMo,624
+common/adapters/android_adapter.py,sha256=wkUFXtfopdDZVAeoZGJ4j7nPfKUaDxUICJB4_BEjeE0,9699
+common/adapters/base_adapter.py,sha256=QYsz9fqDyBtVhcqx_gTrVoIJCiV_h7ZlIhgr86o0oFk,450
+common/adapters/ios_adapter.py,sha256=ONs9-jZN2Z3tsF0ft_A572RuB0Zmn5Gqw65N61sarZc,9671
+common/adapters/web_adapter.py,sha256=SkPoZ4j4VEoD62MNe6XfwWLV86BvrYhclKKkCeWnxkY,10396
+common/cache/__init__.py,sha256=6gWtxmR6uzvB8SCz-qirbF_dKpBGSvd2DUvB-_XXTKA,437
+common/cache/cache_hash.py,sha256=YcdFgfYOl9KraLNBi-YyLzzYjdwwUzuw1pclb53MqMg,1907
+common/cache/cache_manager.py,sha256=70oaCNFZRiV8eiwSeeVneruy6I63M9iYYXbTr3HJv5Q,12084
+common/cache/cache_stats.py,sha256=vuWdmWiv1QS7qY-n1_ao_YDAK1bUoYOnYwwOndmCoF0,4510
+common/cache/cache_storage.py,sha256=FO26Nih2DSsif9g_o_uJK1WExm34KQ-yxfpde39nufQ,2395
+common/cache/embedding_loader.py,sha256=qjiXUNMYu9-oATHsnJZaW25-k3Un4AkKuLXaC8U9-IA,5424
+config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+config/config.py,sha256=u2RkvTYtDQCPMIPVBiPKgLic0gSyW15pxbkMPWnMcNs,6778
+config/env_loader.py,sha256=1IXZXuH62yeKXQ3l1-ZWc2I1Am1AShJ4vW6OTVibD_M,2137
+screenforge-0.4.0.dist-info/licenses/LICENSE,sha256=yCiuqsAjnCqF1spnYbvar2ouEI7CdnU_A1lVRp-hv-4,1064
+utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+utils/screenshot_annotator.py,sha256=Wuf1ogk_qTcmaTD8L6cEISQhBH2tAOB69wUQcuw4kSY,1884
+utils/utils_ios.py,sha256=YuTzwyOJAZFqFVq6sRlf2f_yOmax9I2YA75ae_m_fpk,9136
+utils/utils_web.py,sha256=m7bwZVfgGr6rOYVrMA9dN-UanV_nan5gnh06tU4RoFs,17565
+utils/utils_xml.py,sha256=q26alq33OJZD7e--AlW6r2awY81zP091HQesoCEo2xQ,9569
+screenforge-0.4.0.dist-info/METADATA,sha256=tS5imJNDLxOFeF3qr2ud1MaPT0kpL1PD5cWUhNcoGns,1564
+screenforge-0.4.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+screenforge-0.4.0.dist-info/entry_points.txt,sha256=SNdpMdZH6iU6Y9Vo2lAph1LdDxDPcko8XX42aBgAzEY,50
+screenforge-0.4.0.dist-info/top_level.txt,sha256=nS7U81ECciGCg0T8oLxLdD_5miVGgnxaWyfPUTgRvRY,24
+screenforge-0.4.0.dist-info/RECORD,,

screenforge-0.4.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

screenforge-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+screenforge = cli.dispatch:main

screenforge-0.4.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 jhinzzz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

screenforge-0.4.0.dist-info/top_level.txt

@@ -0,0 +1,4 @@
+cli
+common
+config
+utils

utils/screenshot_annotator.py

@@ -0,0 +1,60 @@
+"""截图标注器: 在截图上绘制红色矩形 + ref 标签, 帮助 Agent 直观定位元素。"""
+
+import io
+
+from PIL import Image, ImageDraw, ImageFont
+
+
+def annotate_screenshot(png_bytes: bytes, ui_elements: list[dict]) -> bytes:
+    """在截图上为可点击元素绘制红色边框和 ref 标签。
+
+    Args:
+        png_bytes: 原始截图的 PNG 字节
+        ui_elements: compress_web_dom 返回的元素列表, 每个元素需含 ref/x/y/w/h
+
+    Returns:
+        标注后的 PNG 字节
+    """
+    img = Image.open(io.BytesIO(png_bytes)).convert("RGBA")
+    overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
+    draw = ImageDraw.Draw(overlay)
+
+    try:
+        font = ImageFont.truetype("/System/Library/Fonts/Helvetica.ttc", 14)
+    except Exception:
+        font = ImageFont.load_default()
+
+    for el in ui_elements:
+        if not el.get("clickable"):
+            continue
+        ref = el.get("ref", "")
+        x = el.get("x", 0)
+        y = el.get("y", 0)
+        w = el.get("w", 0)
+        h = el.get("h", 0)
+        if w <= 0 or h <= 0:
+            continue
+
+        # 红色半透明矩形边框
+        draw.rectangle(
+            [x, y, x + w, y + h],
+            outline=(255, 0, 0, 200),
+            width=2,
+        )
+
+        # ref 标签背景 + 文字
+        if ref:
+            label = ref
+            bbox = font.getbbox(label)
+            tw = bbox[2] - bbox[0] + 6
+            th = bbox[3] - bbox[1] + 4
+            # 标签放在元素左上角上方, 避免遮挡内容
+            lx = x
+            ly = max(y - th - 2, 0)
+            draw.rectangle([lx, ly, lx + tw, ly + th], fill=(255, 0, 0, 220))
+            draw.text((lx + 3, ly + 1), label, fill=(255, 255, 255, 255), font=font)
+
+    result = Image.alpha_composite(img, overlay).convert("RGB")
+    buf = io.BytesIO()
+    result.save(buf, format="PNG")
+    return buf.getvalue()

utils/utils_ios.py

@@ -0,0 +1,195 @@
+"""iOS WDA XML compression and dimensionality reduction."""
+
+import json
+import xml.etree.ElementTree as ET
+
+_SKIP_TYPES = frozenset(("Window", "Application"))
+_KEYBOARD_TYPES = frozenset(("Key",))
+_NOISE_LABEL_MAX_LEN = 1
+
+# How strongly a node type represents "the actionable control" of a row. WDA
+# repeats a row's label on a nested StaticText AND on the control that carries it
+# (a Switch/Button, or the Cell itself); the StaticText is an inert shadow. When
+# several elements in one row share an exact label, the highest-priority type wins
+# and strictly-lower ones are suppressed — collapsing the inert StaticText shadow
+# while keeping the actionable control. (Equal-priority ties — e.g. two same-label
+# Buttons in one row — are both kept; that's two real controls, not a shadow.)
+_INTERACTIVE_PRIORITY = {"Switch": 3, "Button": 2, "Cell": 1}
+
+
+def _ios_type(node) -> str:
+    """Element type without the verbose XCUIElementType prefix."""
+    return node.attrib.get("type", "").replace("XCUIElementType", "")
+
+
+def _interactive_priority(node_type: str) -> int:
+    """Higher = more 'the actionable control'. A label-shadow StaticText is 0 and
+    loses to the Switch/Button/Cell carrying the same label."""
+    return _INTERACTIVE_PRIORITY.get(node_type, 0)
+
+
+def _row_members(cell):
+    """The Cell plus its descendants that belong to THIS row — stopping the descent
+    at any nested Cell (a separate row owns its own labels). Without this boundary,
+    an outer Cell's label group would swallow an inner row's distinct label and
+    suppress it, erasing the inner row from the output."""
+    yield cell
+    for child in cell:
+        if _ios_type(child) == "Cell":
+            continue  # nested row — its labels belong to it, not to `cell`
+        yield from _row_members(child)
+
+
+def _compute_label_shadows(root) -> set:
+    """Find redundant label-shadow nodes to suppress (returns a set of id(node)).
+
+    WDA models every Settings/list row as a `Cell`, and repeats the row's label on
+    a nested `StaticText` AND on the row's interactive control. A flat walk emits
+    the same label 2-3x — a Button/Cell/Switch plus a StaticText twin (a Switch row
+    yields Cell + StaticText + Switch, all labelled the same). That bloats the tree
+    and makes `d(label=...)` ambiguous (it matches the tap target AND its inert text
+    twin). Within each row we group labelled members (the Cell + its non-nested-Cell
+    descendants) by EXACT label and suppress every element whose interactive priority
+    is strictly below the group's max — keeping the actionable control per label and
+    dropping the inert StaticText shadow. Equal-priority ties are preserved (two real
+    same-label controls are not a shadow).
+
+    Identity keys are safe because `root` is held alive across both passes inside
+    compress_ios_xml; do not stream/re-parse between the passes.
+
+    Honesty boundaries (verified on a live simulator):
+      - EXACT-label match only. A real subtitle whose text merely contains the
+        control's label (e.g. the Apple-account row, whose Button carries a combined
+        label and whose StaticTexts are distinct title/subtitle) is NOT a shadow and
+        survives.
+      - Scope is the row (a Cell, NOT descending into a nested Cell — see
+        _row_members). A StaticText with no Cell ancestor (a standalone caption like
+        '关'/Off) is never grouped, so it survives; and an inner row's distinct label
+        is never eaten by the outer row that contains it.
+      - A label-group with no interactive element (all StaticText) is left untouched —
+        never suppress a row down to nothing; the row's tap target is preserved.
+      - VISIBILITY-AWARE. Pass 2 emits only visible nodes, so the priority winner is
+        chosen among VISIBLE nodes and only VISIBLE nodes are suppressed. Otherwise an
+        invisible higher-priority twin (WDA marks subviews invisible routinely) would
+        suppress the visible sibling and then be dropped itself — erasing a real,
+        on-screen row entirely (label + tap target + any value). Suppressing only
+        visible nodes is sufficient: invisible ones never reach the output anyway.
+    """
+    suppress: set = set()
+    for cell in root.iter():
+        if _ios_type(cell) != "Cell":
+            continue
+        groups: dict = {}
+        for node in _row_members(cell):  # this row only — not nested-Cell rows
+            label = node.attrib.get("label", "").strip()
+            if label:
+                groups.setdefault(label, []).append(node)
+        for nodes in groups.values():
+            # Only nodes the emit loop will actually keep (visible) can be a shadow
+            # or a winner — judging by all nodes lets an invisible winner erase a
+            # visible row (the winner is then dropped by Pass 2's visibility filter).
+            visible_nodes = [n for n in nodes if n.attrib.get("visible") == "true"]
+            if len(visible_nodes) < 2:
+                continue  # 0/1 visible element with this label — nothing to dedup
+            top = max(_interactive_priority(_ios_type(n)) for n in visible_nodes)
+            if top < 1:
+                continue  # no visible interactive element — keep them all
+            for node in visible_nodes:
+                if _interactive_priority(_ios_type(node)) < top:
+                    suppress.add(id(node))
+    return suppress
+
+
+def compress_ios_xml(raw_xml: str) -> str:
+    try:
+        root = ET.fromstring(raw_xml)
+    except ET.ParseError as e:
+        raw_preview = raw_xml[:200] if raw_xml else "(empty)"
+        print(f"[Warning] iOS XML parse failed: {e}, first 200 chars: {raw_preview}")
+        return '{"ui_elements": []}'
+
+    elements = []
+    seen_keys = set()
+    has_keyboard = False
+    # Pass 1: within each row (Cell), find label-shadow StaticTexts that merely
+    # repeat the row control's label, so Pass 2 emits ONE targetable element per
+    # row instead of a Button/Cell/Switch + its inert text twin. `root` stays alive
+    # across both passes (id()-keyed), so don't re-parse between them.
+    shadow_ids = _compute_label_shadows(root)
+
+    for node in root.iter():
+        if id(node) in shadow_ids:
+            continue  # redundant label-shadow — the row's actionable control carries it.
+
+        attrib = node.attrib
+        label = attrib.get("label", "").strip()
+        name = attrib.get("name", "").strip()
+        value = attrib.get("value", "").strip()
+        node_type = attrib.get("type", "").replace("XCUIElementType", "")
+        enabled = attrib.get("enabled") == "true"
+        visible = attrib.get("visible") == "true"
+        accessible = attrib.get("accessible") == "true"
+
+        if not visible:
+            continue
+
+        if not (label or name or value or accessible):
+            continue
+
+        if node_type in _SKIP_TYPES:
+            if not label:
+                continue
+
+        if node_type == "Other" and not label:
+            continue
+
+        if node_type in _KEYBOARD_TYPES:
+            has_keyboard = True
+            continue
+
+        if node_type == "WebView" and not label:
+            continue
+
+        if label and ("滚动条" in label or "scroll bar" in label.lower()):
+            continue
+
+        if node_type == "StaticText" and label and len(label) <= _NOISE_LABEL_MAX_LEN:
+            if label.isdigit() or label in (".", ","):
+                continue
+
+        # Flat exact-dedup (pre-existing): collapse repeated (type, label, name),
+        # except Cells (each Cell is its own row). Note its interaction with the
+        # shadow pass above: two SEPARATE rows whose interactive controls share an
+        # identical (type, label, name) collapse to a single element here, and their
+        # StaticText shadows were already suppressed — so identically-named rows are
+        # not independently targetable. Benign for real WDA, where each row's control
+        # carries a unique name (e.g. com.apple.settings.general); it only affects
+        # synthetic identical-name rows, which d(label=...) could not disambiguate
+        # even before de-shadowing. Pinned by test_duplicate_name_rows_* in
+        # tests/test_utils_ios.py.
+        dedup_key = (node_type, label, name)
+        if dedup_key in seen_keys and node_type != "Cell":
+            continue
+        seen_keys.add(dedup_key)
+
+        el_info = {"type": node_type}
+        if label:
+            el_info["label"] = label
+        if name and name != label:
+            el_info["name"] = name
+        if value:
+            el_info["value"] = value
+        if accessible:
+            el_info["accessible"] = True
+        # Unified cross-platform key: a disabled control is `disabled: true`,
+        # matching Android (utils_xml.py) and Web (utils_web.py), so the LLM brain
+        # sees one vocabulary for "can't interact" on every platform.
+        if not enabled:
+            el_info["disabled"] = True
+
+        elements.append(el_info)
+
+    if has_keyboard:
+        elements.append({"type": "Keyboard", "label": "keyboard_visible"})
+
+    return json.dumps({"ui_elements": elements}, ensure_ascii=False)