a11y-computer-use 0.1.1__tar.gz

a11y_computer_use-0.1.1/.editorconfig

@@ -0,0 +1,22 @@
+# EditorConfig: https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.py]
+indent_style = space
+indent_size = 4
+
+[*.{yml,yaml,json,toml}]
+indent_style = space
+indent_size = 2
+
+# Markdown: 2-space list indentation; keep trailing double-space line breaks.
+[*.md]
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = false

a11y_computer_use-0.1.1/.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# Every pull request requests a review from the maintainer.
+# Mukhsin0508 is the configured git user for this repository (git config user.name).
+* @Mukhsin0508

a11y_computer_use-0.1.1/.github/ISSUE_TEMPLATE/app_compatibility.yml

@@ -0,0 +1,154 @@
+name: App compatibility report
+description: An app whose accessibility tree is missing, partial or hostile. Say which roles showed up and whether the vision fallback worked.
+title: "[app-compat] <app> on <platform>"
+labels: ["app-compat"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This template builds the per-app coverage table. Two signals matter most: what `desktop_snapshot` returned (a full snapshot with no clickable or editable element ends with a line beginning `note: no interactive elements were found`), and whether `screenshot` plus coordinate `click` got the task done when refs could not.
+
+        Values in a snapshot can contain private text. Redact before you paste. Secure fields never carry a value.
+
+  - type: input
+    id: app_name
+    attributes:
+      label: App name and version
+      placeholder: Telegram 11.2 / Slack 4.40 / LibreOffice Writer 24.8
+    validations:
+      required: true
+
+  - type: input
+    id: app_id
+    attributes:
+      label: App identifier
+      description: What the permission gate keys on. macOS bundle id, Windows process image name, Linux process comm name, or the CDP target (tab) id of the bound tab for the browser backend (the `id` field of the target in `http://127.0.0.1:9222/json`; paste the page URL as well).
+      placeholder: ru.keepcoder.Telegram / slack.exe / soffice.bin / <CDP target id> (https://app.example.com)
+    validations:
+      required: true
+
+  - type: dropdown
+    id: backend
+    attributes:
+      label: Backend
+      options:
+        - macOS (AXUIElement)
+        - Windows (UI Automation)
+        - Linux X11 or XWayland (AT-SPI2)
+        - Linux native Wayland (AT-SPI2)
+        - Browser (Chromium over CDP)
+    validations:
+      required: true
+
+  - type: dropdown
+    id: toolkit
+    attributes:
+      label: UI toolkit, if you know it
+      options:
+        - Native (AppKit, SwiftUI, Win32, WinUI, GTK, Qt)
+        - Chromium or Electron
+        - Custom-drawn (game engine, Flutter, a canvas-drawn client)
+        - Java (Swing, JavaFX)
+        - Not sure
+
+  - type: dropdown
+    id: scope_tried
+    attributes:
+      label: Snapshot scopes tried
+      description: The handoff note suggests `scope="app"` when `scope="window"` came back empty.
+      options:
+        - window only
+        - app only
+        - both window and app
+    validations:
+      required: true
+
+  - type: dropdown
+    id: tree_state
+    attributes:
+      label: What did desktop_snapshot return?
+      options:
+        - Nothing actionable, the handoff note appeared (no clickable or editable element)
+        - A shell only (window and groups, no buttons or fields)
+        - Partial (some controls present, others missing)
+        - Controls present but untitled (no title, description or value, so refs are unusable)
+        - A full tree, but actions fail (press returns False, stale_ref, wrong bounds)
+        - The snapshot timed out or raised an error
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: role_coverage
+    attributes:
+      label: Role coverage
+      description: Tick each kind of element that appears in the snapshot with a usable title or value. Role names are the snapshot's lowercase forms.
+      options:
+        - label: Buttons, menu items and links (button, menuitem, link) with titles
+        - label: Text inputs (textfield, textarea, searchfield) marked edit
+        - label: Password fields as securetextfield
+        - label: Toggles (checkbox, radiobutton) with a checked or unchecked flag
+        - label: Lists, tables and outlines (list, table, outline, grid, row, cell)
+        - label: Tab groups and toolbars
+        - label: Web content (webarea) with children
+        - label: Static text and images with titles
+
+  - type: textarea
+    id: snapshot
+    attributes:
+      label: Snapshot excerpt
+      description: The header line `[snap-N] <app> (<scope>)` and enough element lines to show what is missing. Include the trailing note if there was one.
+      render: text
+    validations:
+      required: true
+
+  - type: dropdown
+    id: web_a11y
+    attributes:
+      label: Chromium or Electron apps only, did the accessibility force-enable help?
+      description: On macOS a11y-computer-use sets `AXManualAccessibility` and `AXEnhancedUserInterface` on the app element once per process; on Linux it sets `org.a11y.Status` on the session bus. Set `A11Y_COMPUTER_USE_NO_WEB_A11Y=1` to compare with it off; `examples/web_a11y_demo.py` prints a before and after on macOS.
+      options:
+        - Not applicable, the app is not Chromium or Electron
+        - Yes, the tree populated after the first snapshot
+        - Partly, some web content appeared but not the whole page
+        - No, the tree stayed empty with the force-enable on
+        - Not compared
+
+  - type: dropdown
+    id: vision_fallback
+    attributes:
+      label: Did the vision fallback work?
+      description: The vision path is `screenshot` (downscaled to 1280 px on the long edge by default), `zoom` for a native-resolution crop, then `click` by x and y in display-qualified physical pixels. On macOS it needs the Screen Recording grant; on native Wayland it needs `grim`; on Windows `screenshot` and `zoom` are not implemented yet.
+      options:
+        - Yes, screenshot plus coordinate click completed the task
+        - Partly, the screenshot worked but coordinate clicks landed off target or the app ignored synthetic input
+        - No, screenshot failed (permission_denied_screen, unsupported, or NotImplementedError on Windows)
+        - Not tried
+    validations:
+      required: true
+
+  - type: textarea
+    id: actions
+    attributes:
+      label: Which actions worked and which did not
+      description: For example `click(ref)` through the a11y press versus a synthetic click with `A11Y_COMPUTER_USE_AX_CLICKS=0`, `set_value`, `type`, `scroll` with `into_view=true`. Paste the error strings.
+
+  - type: textarea
+    id: native_inspector
+    attributes:
+      label: What a native accessibility inspector shows
+      description: Accessibility Inspector (Xcode) on macOS, Accessibility Insights on Windows, Accerciser on Linux, `chrome://accessibility` for Chromium. Does the native inspector see more than a11y-computer-use did? A screenshot of the inspector helps.
+
+  - type: textarea
+    id: doctor
+    attributes:
+      label: Output of a11y-computer-use doctor
+      render: text
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      options:
+        - label: The pastes above contain no passwords, tokens or private text.
+          required: true
+        - label: I searched the open and closed issues for this app.

a11y_computer_use-0.1.1/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,188 @@
+name: Bug report
+description: A tool call, CLI command or Runtime method did the wrong thing. Include the doctor report, the snapshot excerpt and the audit line.
+title: "[bug] "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for the report. Two things before you start.
+
+        Redact secrets. The audit log keeps typed text and key chords in the clear unless the field was secure; clipboard writes and secure fields are already written as `[REDACTED]`. Read every paste before you submit.
+
+        Known gaps are not bugs. Raw coordinate input (click, drag, scroll, key chords) returns `unsupported` on native Wayland by design; ref-based press, `set_value` and typing into a focused field still work there (docs/linux-port.md). A `NotImplementedError` raised from `a11y_computer_use/drivers/windows.py` names the native API that primitive will use and marks a known gap. `window` supports `list` and `raise` only, and `app quit` is not exposed; both raise `ValueError`. For any of these, open a feature request instead.
+
+  - type: dropdown
+    id: backend
+    attributes:
+      label: Backend
+      description: The driver the server ran on. The default is the current OS; `A11Y_COMPUTER_USE_DRIVER=browser` selects the browser backend explicitly.
+      options:
+        - macOS (AXUIElement, CGEvent)
+        - Windows (UI Automation, SendInput)
+        - Linux X11 or XWayland (AT-SPI2, XTEST)
+        - Linux native Wayland (AT-SPI2 only, WAYLAND_DISPLAY set and no DISPLAY)
+        - Browser (Chromium over CDP)
+    validations:
+      required: true
+
+  - type: input
+    id: os_version
+    attributes:
+      label: OS and version
+      description: For the browser backend, also give the Chromium version and whether it ran headless.
+      placeholder: macOS 15.5 / Windows 11 / Ubuntu 24.04 with GNOME on Wayland / Chrome 128 headless
+    validations:
+      required: true
+
+  - type: input
+    id: app
+    attributes:
+      label: App under control
+      description: The identifier the permission gate keys on. macOS uses the bundle id, Windows the process image name, Linux the process comm name, the browser backend the CDP tab id (paste the page URL as well).
+      placeholder: com.apple.finder / notepad.exe / gedit / https://example.com
+    validations:
+      required: true
+
+  - type: dropdown
+    id: tool
+    attributes:
+      label: Tool or command
+      description: The MCP tool, CLI subcommand or Python entry point that misbehaved.
+      options:
+        - desktop_snapshot
+        - find
+        - screenshot
+        - zoom
+        - click
+        - type
+        - key
+        - scroll
+        - drag
+        - wait_for
+        - act
+        - set_value
+        - scroll_to_find
+        - app
+        - window
+        - clipboard
+        - console (browser backend only)
+        - network (browser backend only)
+        - a11y-computer-use mcp (server startup or handshake)
+        - a11y-computer-use doctor
+        - a11y-computer-use snapshot
+        - a11y-computer-use run-once
+        - a11y-computer-use bench
+        - a11y-computer-use agent (the reference agent loop)
+        - Python API (server.Runtime, safety, drivers)
+        - Not sure
+    validations:
+      required: true
+
+  - type: dropdown
+    id: error_code
+    attributes:
+      label: Error code
+      description: >-
+        The code at the start of the error string, if there was one. Structured errors arrive as
+        `<code>: <message> | detail: {...} | hint: ...`; safety refusals as `<verdict>: <reason>`.
+      options:
+        - stale_ref
+        - permission_denied_accessibility
+        - permission_denied_screen
+        - secure_field
+        - focus_changed
+        - app_not_found
+        - timeout
+        - confirmation_declined
+        - unsupported
+        - needs_permission (a safety verdict, the app has no grant)
+        - deny (a safety verdict, deny list or insufficient tier)
+        - NotImplementedError
+        - ValueError
+        - Another traceback (paste it below)
+        - No error, the behaviour was wrong
+
+  - type: input
+    id: version
+    attributes:
+      label: a11y-computer-use version and commit
+      description: a11y-computer-use is not published on PyPI, so name the commit you installed from (`git rev-parse --short HEAD`) and the extras you installed.
+      placeholder: 0.0.1 at <short sha>, pip install -e '.[dev,browser]'
+    validations:
+      required: true
+
+  - type: textarea
+    id: doctor
+    attributes:
+      label: Output of a11y-computer-use doctor
+      description: Run `a11y-computer-use doctor` (from a clone, `.venv/bin/a11y-computer-use doctor`) in the same host context that runs the server. On macOS the grants attach to the responsible app, and the report names it. Paste the whole output. If doctor itself fails on your platform, paste that traceback instead.
+      render: text
+    validations:
+      required: true
+
+  - type: textarea
+    id: snapshot
+    attributes:
+      label: Snapshot excerpt
+      description: >-
+        The relevant lines of the `desktop_snapshot` or `find` output. Keep the header line
+        `[snap-N] <app> (<scope>)` and the element lines around the target ref. If the snapshot ended
+        with a line beginning `note: no interactive elements were found`, use the app compatibility
+        template instead.
+      render: text
+
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to reproduce
+      description: The tool calls in order, with their arguments. For `act`, paste the steps list. For the CLI, paste the exact command line.
+      placeholder: |
+        1. desktop_snapshot(app="com.apple.TextEdit", scope="window")
+        2. click(ref="e7", verify=true)
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behaviour
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual
+    attributes:
+      label: Actual behaviour
+      description: Paste the full error string or traceback, and the `effect:` diff if you used `verify=true`.
+      render: text
+    validations:
+      required: true
+
+  - type: textarea
+    id: audit
+    attributes:
+      label: Audit log entry
+      description: The JSONL line(s) for the failing action. The log is at `~/.a11y-computer-use/audit/<YYYY-MM-DD>.jsonl`, one file per UTC day, and every gated call is written, including refusals. Redact `params.text` and `params.chord` if they contain anything private.
+      render: json
+
+  - type: textarea
+    id: environment
+    attributes:
+      label: Environment variables and host
+      description: Any `A11Y_COMPUTER_USE_*` variables in effect and the MCP host that started the server (Claude Code via `claude mcp add`, a Node client, an in-process `server.Runtime`, `a11y-computer-use run-once`, `a11y-computer-use agent`). `A11Y_COMPUTER_USE_AX_CLICKS` and `A11Y_COMPUTER_USE_CONFIRM` are read once when `a11y_computer_use.server` is imported, so they must be set in the host process before it spawns the server.
+      placeholder: |
+        A11Y_COMPUTER_USE_DRIVER=browser
+        A11Y_COMPUTER_USE_CDP_ENDPOINT=http://127.0.0.1:9222
+        host = Claude Code
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      options:
+        - label: I searched the open and closed issues for this problem.
+          required: true
+        - label: The pastes above contain no passwords, tokens or private text.
+          required: true
+        - label: I reproduced this on the current head of the branch I installed from.

a11y_computer_use-0.1.1/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Questions and ideas
+    url: https://github.com/Perception-Dynamics-Inc/a11y-computer-use/issues
+    about: Search existing issues first. Ask how to embed a11y-computer-use, share an integration, or discuss the roadmap by opening an issue that starts from PLAN.md.
+  - name: Security vulnerabilities
+    url: https://github.com/Perception-Dynamics-Inc/a11y-computer-use/security/policy
+    about: Do not open a public issue for a vulnerability. SECURITY.md describes how to report one privately.

a11y_computer_use-0.1.1/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,78 @@
+name: Feature request
+description: Propose a new tool, a driver primitive, a safety behavior or an integration surface.
+title: "[feature] "
+labels: ["enhancement"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        a11y-computer-use is an embeddable Driver plus an MCP surface: pruned accessibility snapshots, element refs, and a safety layer. PLAN.md section 4 lists what the project deliberately does not build and section 9 lists the roadmap phases. Skim both first so the proposal lands in the right place.
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What are you trying to make an agent do that a11y-computer-use cannot do today? Name the app and the platform.
+    validations:
+      required: true
+
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      description: Pick every part this touches.
+      multiple: true
+      options:
+        - Observation (desktop_snapshot, find, diff mode, refs and re-resolution)
+        - Action (click, type, key, scroll, drag, act, set_value, scroll_to_find, wait_for)
+        - Vision path (screenshot, zoom, Set-of-Mark)
+        - Safety (tiers, grants, confirmation gate, audit log, secure fields)
+        - macOS driver
+        - Windows driver
+        - Linux driver (AT-SPI2, X11, Wayland)
+        - Browser driver (CDP, console, network)
+        - MCP surface and host integration
+        - CLI (mcp, doctor, snapshot, run-once, bench, agent)
+        - Python API (server.Runtime, safety, drivers)
+        - Docs and examples
+        - Packaging and release
+        - Something else
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed change
+      description: For a new MCP tool, give the signature and the safety tier you expect it to gate at (`read`, `click` or `full`). For a driver primitive, name the native API on each backend you know. For a safety change, say what the audit entry should record.
+    validations:
+      required: true
+
+  - type: textarea
+    id: workarounds
+    attributes:
+      label: Workarounds you tried
+      description: Which existing tools you combined (for example `find` followed by `act`, or `screenshot` followed by a coordinate `click`) and where they fell short.
+
+  - type: dropdown
+    id: backends
+    attributes:
+      label: Backends this should cover
+      multiple: true
+      options:
+        - macOS
+        - Windows
+        - Linux (X11 or XWayland)
+        - Linux (native Wayland)
+        - Browser (CDP)
+        - Platform-free core (observe, act, safety, server)
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      options:
+        - label: I checked PLAN.md and the open issues for this idea.
+          required: true
+        - label: I can help implement it.
+        - label: I can test it on the backends selected above.

a11y_computer_use-0.1.1/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,50 @@
+## Summary
+
+What changed and why. Link the issue if there is one.
+
+## Platforms touched
+
+- [ ] Platform-free core (observe, act, safety, schema, server, cli)
+- [ ] macOS driver (a11y_computer_use/drivers/macos.py and the observe, act, capture modules it delegates to)
+- [ ] Windows driver (a11y_computer_use/drivers/windows.py, _uia, _win_input, _win_system)
+- [ ] Linux driver (a11y_computer_use/drivers/linux.py, _atspi, _linux_input, _linux_system)
+- [ ] Browser driver (a11y_computer_use/drivers/browser.py, _cdp, _cdp_ax)
+- [ ] CI workflow (.github/workflows/ci.yml)
+- [ ] Docs or examples only
+
+## Tests run
+
+Paste the summary line of each run and name the skips you saw.
+
+```text
+.venv/bin/pytest -q
+<summary line, for example: N passed, M skipped>
+```
+
+Live runs, where the change touches them:
+
+- [ ] macOS live tests ran with the Accessibility and Screen Recording grants held by the test process. Without the grants they skip with "Accessibility TCC grant missing"; `a11y-computer-use doctor` shows the grant state.
+- [ ] Windows live UIA tests: `pytest tests/test_windows_live.py -q -s` on Windows.
+- [ ] Linux live AT-SPI2 tests: `pytest tests/test_linux_live.py` under `xvfb-run` and `dbus-run-session`, the way the linux job in ci.yml runs them.
+- [ ] Browser live tests: `pytest tests/test_browser.py -k live` and `pytest tests/test_arena.py -k live` against a Chromium started with `--remote-debugging-port=9222`, with `A11Y_COMPUTER_USE_CDP_ENDPOINT` pointing at it.
+- [ ] Not applicable, the change has no live path.
+
+If a tool was added or removed: `tests/test_server.py` pins the tool list in `EXPECTED_TOOLS`, and `tests/test_browser.py` asserts that `console` and `network` exist only on the browser backend. Update both, and every doc that states the count.
+
+## Docs updated
+
+- [ ] README.md
+- [ ] docs/ (name the file)
+- [ ] Docstrings and the MCP tool descriptions in a11y_computer_use/server.py
+- [ ] docs/decisions/plan-2026-07.md, where the roadmap status changed
+- [ ] No doc change needed (say why)
+
+## Claims
+
+- [ ] Every sentence in this PR, its commits and the docs it touches states what is implemented and verified. Anything gated, unsupported, skipped in CI or tested only through a fake transport is labelled that way in the same sentence.
+- [ ] Every number I wrote (tool counts, test counts, token figures, timings) was measured on this branch, and the command that produced it is in this description.
+
+## Commits
+
+- [ ] Commit messages carry no `Co-Authored-By` or "Generated with" trailers. The message ends at the body.
+- [ ] The branch is rebased on the target branch and CI is green, or the failing job is explained above.

a11y_computer_use-0.1.1/.github/dependabot.yml

@@ -0,0 +1,23 @@
+# Dependabot keeps the Python dependencies declared in pyproject.toml and the
+# actions used by .github/workflows current. Both ecosystems are checked weekly.
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: deps
+    ignore:
+      # a11y_computer_use/server.py imports FastMCP from mcp.server.fastmcp, which
+      # mcp 2.0 removed; pyproject.toml pins mcp<2 for that reason. Skip the
+      # major bump until the import is migrated.
+      - dependency-name: mcp
+        update-types: ["version-update:semver-major"]
+
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: ci