off-camber-open-frame 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {off_camber_open_frame-0.2.0/src/off_camber_open_frame.egg-info → off_camber_open_frame-0.2.2}/PKG-INFO +8 -4
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/README.md +4 -3
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/pyproject.toml +5 -1
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2/src/off_camber_open_frame.egg-info}/PKG-INFO +8 -4
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/SOURCES.txt +9 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/requires.txt +4 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/act/core.py +42 -7
- off_camber_open_frame-0.2.2/src/openframe/cancel.py +42 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/capture/macos.py +73 -18
- off_camber_open_frame-0.2.2/src/openframe/capture/sidecar.py +49 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/cli.py +64 -9
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/doctor.py +1 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/adapter.py +251 -25
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/server.py +37 -4
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/__init__.py +3 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/macos.py +27 -13
- off_camber_open_frame-0.2.2/src/openframe/recognize/defaults.py +45 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/locator.py +11 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/match.py +3 -1
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/tesseract.py +42 -16
- off_camber_open_frame-0.2.2/src/openframe/recognize/template/__init__.py +5 -0
- off_camber_open_frame-0.2.2/src/openframe/recognize/template/matcher.py +204 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/runner.py +544 -48
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/session.py +7 -4
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/types.py +2 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/window.py +22 -11
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_a11y_recognizer.py +61 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_actuator.py +103 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_capture.py +24 -12
- off_camber_open_frame-0.2.2/tests/test_capture_geometry.py +158 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_cli.py +16 -22
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_examples.py +36 -0
- off_camber_open_frame-0.2.2/tests/test_geometry_checks.py +48 -0
- off_camber_open_frame-0.2.2/tests/test_market_validation_tools.py +288 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_match.py +3 -0
- off_camber_open_frame-0.2.2/tests/test_mcp_adapter.py +468 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_mcp_server.py +5 -1
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_recognize_locator.py +34 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_runner.py +394 -55
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_selector_parity.py +4 -5
- off_camber_open_frame-0.2.2/tests/test_template_recognizer.py +179 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_tesseract_recognizer.py +87 -10
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_window_state.py +60 -6
- off_camber_open_frame-0.2.0/tests/test_mcp_adapter.py +0 -173
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/LICENSE +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/setup.cfg +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/dependency_links.txt +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/entry_points.txt +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/top_level.txt +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/act/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/base.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/cost.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/anthropic_provider.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/runner.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/capture/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/flow.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/base.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/coords.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/debug.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/verify/__init__.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/verify/base.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/verify/core.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_agent_anthropic_provider.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_agent_cost.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_agent_runner.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_calibration_fixture.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_doctor.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_flow.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_imports.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_recognize_coords.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_recognize_debug.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_session_api.py +0 -0
- {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: off-camber-open-frame
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Open-source recognition and action for anything on a display.
|
|
5
5
|
Author: Off-Camber
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -32,6 +32,9 @@ Provides-Extra: agent
|
|
|
32
32
|
Requires-Dist: anthropic; extra == "agent"
|
|
33
33
|
Provides-Extra: mcp
|
|
34
34
|
Requires-Dist: mcp<2,>=1.0; extra == "mcp"
|
|
35
|
+
Provides-Extra: template
|
|
36
|
+
Requires-Dist: Pillow; extra == "template"
|
|
37
|
+
Requires-Dist: numpy; extra == "template"
|
|
35
38
|
Dynamic: license-file
|
|
36
39
|
|
|
37
40
|
<p align="center">
|
|
@@ -62,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
|
|
|
62
65
|
|
|
63
66
|
## Status
|
|
64
67
|
|
|
65
|
-
**Beta** (`v0.2.
|
|
68
|
+
**Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
|
|
66
69
|
[`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
|
|
67
70
|
Evidence for this release comes from CI plus a local macOS live gate (including
|
|
68
71
|
MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
@@ -79,7 +82,7 @@ MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
|
79
82
|
### 1. Install
|
|
80
83
|
|
|
81
84
|
```bash
|
|
82
|
-
pip install "off-camber-open-frame[mcp,ocr,act,flow]"
|
|
85
|
+
pip install "off-camber-open-frame[mcp,ocr,act,flow,template]"
|
|
83
86
|
brew install tesseract
|
|
84
87
|
```
|
|
85
88
|
|
|
@@ -123,7 +126,7 @@ python -m openframe.cli doctor
|
|
|
123
126
|
```bash
|
|
124
127
|
python3.11 -m venv .venv
|
|
125
128
|
source .venv/bin/activate
|
|
126
|
-
pip install -e ".[dev,ocr,act,flow,mcp]"
|
|
129
|
+
pip install -e ".[dev,ocr,act,flow,mcp,template]"
|
|
127
130
|
open-frame doctor
|
|
128
131
|
open-frame mcp list-tools --json
|
|
129
132
|
```
|
|
@@ -138,6 +141,7 @@ brew install python@3.11
|
|
|
138
141
|
|
|
139
142
|
- [MCP server setup](docs/MCP_SERVER.md) — Claude Desktop / Cursor config.
|
|
140
143
|
- [Flow setup](docs/FLOW_SETUP.md) — define and run YAML flows.
|
|
144
|
+
- [Template setup](docs/TEMPLATE_SETUP.md) — optional icon/image matching.
|
|
141
145
|
- [API](docs/API.md) — use `Session` and MCP-oriented integration guidance.
|
|
142
146
|
- [Act setup](docs/ACT_SETUP.md) and [Verify setup](docs/VERIFY_SETUP.md) — run safely with evidence.
|
|
143
147
|
- [Full docs index](docs/README.md) — contributor and planning docs.
|
|
@@ -26,7 +26,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
|
|
|
26
26
|
|
|
27
27
|
## Status
|
|
28
28
|
|
|
29
|
-
**Beta** (`v0.2.
|
|
29
|
+
**Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
|
|
30
30
|
[`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
|
|
31
31
|
Evidence for this release comes from CI plus a local macOS live gate (including
|
|
32
32
|
MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
@@ -43,7 +43,7 @@ MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
|
43
43
|
### 1. Install
|
|
44
44
|
|
|
45
45
|
```bash
|
|
46
|
-
pip install "off-camber-open-frame[mcp,ocr,act,flow]"
|
|
46
|
+
pip install "off-camber-open-frame[mcp,ocr,act,flow,template]"
|
|
47
47
|
brew install tesseract
|
|
48
48
|
```
|
|
49
49
|
|
|
@@ -87,7 +87,7 @@ python -m openframe.cli doctor
|
|
|
87
87
|
```bash
|
|
88
88
|
python3.11 -m venv .venv
|
|
89
89
|
source .venv/bin/activate
|
|
90
|
-
pip install -e ".[dev,ocr,act,flow,mcp]"
|
|
90
|
+
pip install -e ".[dev,ocr,act,flow,mcp,template]"
|
|
91
91
|
open-frame doctor
|
|
92
92
|
open-frame mcp list-tools --json
|
|
93
93
|
```
|
|
@@ -102,6 +102,7 @@ brew install python@3.11
|
|
|
102
102
|
|
|
103
103
|
- [MCP server setup](docs/MCP_SERVER.md) — Claude Desktop / Cursor config.
|
|
104
104
|
- [Flow setup](docs/FLOW_SETUP.md) — define and run YAML flows.
|
|
105
|
+
- [Template setup](docs/TEMPLATE_SETUP.md) — optional icon/image matching.
|
|
105
106
|
- [API](docs/API.md) — use `Session` and MCP-oriented integration guidance.
|
|
106
107
|
- [Act setup](docs/ACT_SETUP.md) and [Verify setup](docs/VERIFY_SETUP.md) — run safely with evidence.
|
|
107
108
|
- [Full docs index](docs/README.md) — contributor and planning docs.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "off-camber-open-frame"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.2"
|
|
8
8
|
description = "Open-source recognition and action for anything on a display."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -51,6 +51,10 @@ agent = [
|
|
|
51
51
|
mcp = [
|
|
52
52
|
"mcp>=1.0,<2",
|
|
53
53
|
]
|
|
54
|
+
template = [
|
|
55
|
+
"Pillow",
|
|
56
|
+
"numpy",
|
|
57
|
+
]
|
|
54
58
|
|
|
55
59
|
[tool.setuptools]
|
|
56
60
|
package-dir = { "" = "src" }
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: off-camber-open-frame
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Open-source recognition and action for anything on a display.
|
|
5
5
|
Author: Off-Camber
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -32,6 +32,9 @@ Provides-Extra: agent
|
|
|
32
32
|
Requires-Dist: anthropic; extra == "agent"
|
|
33
33
|
Provides-Extra: mcp
|
|
34
34
|
Requires-Dist: mcp<2,>=1.0; extra == "mcp"
|
|
35
|
+
Provides-Extra: template
|
|
36
|
+
Requires-Dist: Pillow; extra == "template"
|
|
37
|
+
Requires-Dist: numpy; extra == "template"
|
|
35
38
|
Dynamic: license-file
|
|
36
39
|
|
|
37
40
|
<p align="center">
|
|
@@ -62,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
|
|
|
62
65
|
|
|
63
66
|
## Status
|
|
64
67
|
|
|
65
|
-
**Beta** (`v0.2.
|
|
68
|
+
**Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
|
|
66
69
|
[`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
|
|
67
70
|
Evidence for this release comes from CI plus a local macOS live gate (including
|
|
68
71
|
MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
@@ -79,7 +82,7 @@ MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
|
79
82
|
### 1. Install
|
|
80
83
|
|
|
81
84
|
```bash
|
|
82
|
-
pip install "off-camber-open-frame[mcp,ocr,act,flow]"
|
|
85
|
+
pip install "off-camber-open-frame[mcp,ocr,act,flow,template]"
|
|
83
86
|
brew install tesseract
|
|
84
87
|
```
|
|
85
88
|
|
|
@@ -123,7 +126,7 @@ python -m openframe.cli doctor
|
|
|
123
126
|
```bash
|
|
124
127
|
python3.11 -m venv .venv
|
|
125
128
|
source .venv/bin/activate
|
|
126
|
-
pip install -e ".[dev,ocr,act,flow,mcp]"
|
|
129
|
+
pip install -e ".[dev,ocr,act,flow,mcp,template]"
|
|
127
130
|
open-frame doctor
|
|
128
131
|
open-frame mcp list-tools --json
|
|
129
132
|
```
|
|
@@ -138,6 +141,7 @@ brew install python@3.11
|
|
|
138
141
|
|
|
139
142
|
- [MCP server setup](docs/MCP_SERVER.md) — Claude Desktop / Cursor config.
|
|
140
143
|
- [Flow setup](docs/FLOW_SETUP.md) — define and run YAML flows.
|
|
144
|
+
- [Template setup](docs/TEMPLATE_SETUP.md) — optional icon/image matching.
|
|
141
145
|
- [API](docs/API.md) — use `Session` and MCP-oriented integration guidance.
|
|
142
146
|
- [Act setup](docs/ACT_SETUP.md) and [Verify setup](docs/VERIFY_SETUP.md) — run safely with evidence.
|
|
143
147
|
- [Full docs index](docs/README.md) — contributor and planning docs.
|
|
@@ -8,6 +8,7 @@ src/off_camber_open_frame.egg-info/entry_points.txt
|
|
|
8
8
|
src/off_camber_open_frame.egg-info/requires.txt
|
|
9
9
|
src/off_camber_open_frame.egg-info/top_level.txt
|
|
10
10
|
src/openframe/__init__.py
|
|
11
|
+
src/openframe/cancel.py
|
|
11
12
|
src/openframe/cli.py
|
|
12
13
|
src/openframe/doctor.py
|
|
13
14
|
src/openframe/flow.py
|
|
@@ -25,6 +26,7 @@ src/openframe/agent/providers/__init__.py
|
|
|
25
26
|
src/openframe/agent/providers/anthropic_provider.py
|
|
26
27
|
src/openframe/capture/__init__.py
|
|
27
28
|
src/openframe/capture/macos.py
|
|
29
|
+
src/openframe/capture/sidecar.py
|
|
28
30
|
src/openframe/integrations/__init__.py
|
|
29
31
|
src/openframe/integrations/mcp/__init__.py
|
|
30
32
|
src/openframe/integrations/mcp/adapter.py
|
|
@@ -33,12 +35,15 @@ src/openframe/recognize/__init__.py
|
|
|
33
35
|
src/openframe/recognize/base.py
|
|
34
36
|
src/openframe/recognize/coords.py
|
|
35
37
|
src/openframe/recognize/debug.py
|
|
38
|
+
src/openframe/recognize/defaults.py
|
|
36
39
|
src/openframe/recognize/locator.py
|
|
37
40
|
src/openframe/recognize/match.py
|
|
38
41
|
src/openframe/recognize/a11y/__init__.py
|
|
39
42
|
src/openframe/recognize/a11y/macos.py
|
|
40
43
|
src/openframe/recognize/ocr/__init__.py
|
|
41
44
|
src/openframe/recognize/ocr/tesseract.py
|
|
45
|
+
src/openframe/recognize/template/__init__.py
|
|
46
|
+
src/openframe/recognize/template/matcher.py
|
|
42
47
|
src/openframe/verify/__init__.py
|
|
43
48
|
src/openframe/verify/base.py
|
|
44
49
|
src/openframe/verify/core.py
|
|
@@ -49,11 +54,14 @@ tests/test_agent_cost.py
|
|
|
49
54
|
tests/test_agent_runner.py
|
|
50
55
|
tests/test_calibration_fixture.py
|
|
51
56
|
tests/test_capture.py
|
|
57
|
+
tests/test_capture_geometry.py
|
|
52
58
|
tests/test_cli.py
|
|
53
59
|
tests/test_doctor.py
|
|
54
60
|
tests/test_examples.py
|
|
55
61
|
tests/test_flow.py
|
|
62
|
+
tests/test_geometry_checks.py
|
|
56
63
|
tests/test_imports.py
|
|
64
|
+
tests/test_market_validation_tools.py
|
|
57
65
|
tests/test_match.py
|
|
58
66
|
tests/test_mcp_adapter.py
|
|
59
67
|
tests/test_mcp_server.py
|
|
@@ -63,6 +71,7 @@ tests/test_recognize_locator.py
|
|
|
63
71
|
tests/test_runner.py
|
|
64
72
|
tests/test_selector_parity.py
|
|
65
73
|
tests/test_session_api.py
|
|
74
|
+
tests/test_template_recognizer.py
|
|
66
75
|
tests/test_tesseract_recognizer.py
|
|
67
76
|
tests/test_verify.py
|
|
68
77
|
tests/test_window_state.py
|
|
@@ -6,6 +6,7 @@ import time
|
|
|
6
6
|
from collections.abc import Callable
|
|
7
7
|
from typing import Literal
|
|
8
8
|
|
|
9
|
+
from openframe.cancel import raise_if_tool_cancelled
|
|
9
10
|
from openframe.recognize.coords import target_uses_physical_pixels
|
|
10
11
|
from openframe.types import Frame, Target
|
|
11
12
|
|
|
@@ -29,13 +30,15 @@ class Actuator:
|
|
|
29
30
|
*,
|
|
30
31
|
anchor: ClickAnchor = "center",
|
|
31
32
|
scale_factor: float = 1.0,
|
|
33
|
+
origin: tuple[int, int] = (0, 0),
|
|
32
34
|
) -> tuple[int, int]:
|
|
33
35
|
"""Map target bounds to a click point.
|
|
34
36
|
|
|
35
|
-
Recognizers report
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
Recognizers report image matches in captured-image pixels. Mouse
|
|
38
|
+
actuation works in logical screen points. ``scale_factor`` converts
|
|
39
|
+
those pixels, then ``origin`` (the capture's logical top-left) places
|
|
40
|
+
the point on the screen. Accessibility targets are already logical
|
|
41
|
+
screen points, so ``origin`` is not added for them.
|
|
39
42
|
"""
|
|
40
43
|
if scale_factor <= 0:
|
|
41
44
|
raise ValueError("scale_factor must be greater than zero.")
|
|
@@ -54,7 +57,8 @@ class Actuator:
|
|
|
54
57
|
raise ValueError(f"Unsupported click anchor: {anchor}")
|
|
55
58
|
|
|
56
59
|
if target_uses_physical_pixels(target):
|
|
57
|
-
|
|
60
|
+
origin_x, origin_y = origin
|
|
61
|
+
return (round(px / scale_factor) + origin_x, round(py / scale_factor) + origin_y)
|
|
58
62
|
return (px, py)
|
|
59
63
|
|
|
60
64
|
def click_target(
|
|
@@ -64,14 +68,18 @@ class Actuator:
|
|
|
64
68
|
anchor: ClickAnchor = "center",
|
|
65
69
|
kind: ClickKind = "click",
|
|
66
70
|
scale_factor: float = 1.0,
|
|
71
|
+
origin: tuple[int, int] = (0, 0),
|
|
67
72
|
) -> tuple[int, int]:
|
|
68
73
|
"""Click a target at a mapped point in logical click space."""
|
|
69
|
-
x, y = self.point_for_target(
|
|
74
|
+
x, y = self.point_for_target(
|
|
75
|
+
target, anchor=anchor, scale_factor=scale_factor, origin=origin
|
|
76
|
+
)
|
|
70
77
|
self.click_point(x, y, kind=kind)
|
|
71
78
|
return (x, y)
|
|
72
79
|
|
|
73
80
|
def click_point(self, x: int, y: int, *, kind: ClickKind = "click") -> None:
|
|
74
81
|
"""Click at absolute screen coordinates."""
|
|
82
|
+
raise_if_tool_cancelled()
|
|
75
83
|
if self.dry_run:
|
|
76
84
|
return
|
|
77
85
|
|
|
@@ -89,6 +97,7 @@ class Actuator:
|
|
|
89
97
|
|
|
90
98
|
def type_text(self, text: str, *, interval: float = 0.0) -> None:
|
|
91
99
|
"""Type plain text at current focus."""
|
|
100
|
+
raise_if_tool_cancelled()
|
|
92
101
|
if self.dry_run:
|
|
93
102
|
return
|
|
94
103
|
pyautogui = _load_pyautogui()
|
|
@@ -96,6 +105,7 @@ class Actuator:
|
|
|
96
105
|
|
|
97
106
|
def key_combo(self, *keys: str) -> None:
|
|
98
107
|
"""Press a key combination, e.g. ('command', 'v')."""
|
|
108
|
+
raise_if_tool_cancelled()
|
|
99
109
|
if self.dry_run:
|
|
100
110
|
return
|
|
101
111
|
if not keys:
|
|
@@ -105,6 +115,7 @@ class Actuator:
|
|
|
105
115
|
|
|
106
116
|
def press_key(self, key: str) -> None:
|
|
107
117
|
"""Press a single key."""
|
|
118
|
+
raise_if_tool_cancelled()
|
|
108
119
|
if self.dry_run:
|
|
109
120
|
return
|
|
110
121
|
if not key:
|
|
@@ -113,7 +124,11 @@ class Actuator:
|
|
|
113
124
|
pyautogui.press(key)
|
|
114
125
|
|
|
115
126
|
def scroll(self, clicks: int, *, x: int | None = None, y: int | None = None) -> None:
|
|
116
|
-
"""Scroll at a point (or current cursor position).
|
|
127
|
+
"""Scroll at a point (or current cursor position).
|
|
128
|
+
|
|
129
|
+
Positive ``clicks`` scroll up; negative scroll down (pyautogui convention).
|
|
130
|
+
"""
|
|
131
|
+
raise_if_tool_cancelled()
|
|
117
132
|
if self.dry_run:
|
|
118
133
|
return
|
|
119
134
|
pyautogui = _load_pyautogui()
|
|
@@ -122,6 +137,26 @@ class Actuator:
|
|
|
122
137
|
return
|
|
123
138
|
pyautogui.scroll(clicks)
|
|
124
139
|
|
|
140
|
+
def drag(
|
|
141
|
+
self,
|
|
142
|
+
start_x: int,
|
|
143
|
+
start_y: int,
|
|
144
|
+
end_x: int,
|
|
145
|
+
end_y: int,
|
|
146
|
+
*,
|
|
147
|
+
duration: float = 0.2,
|
|
148
|
+
button: Literal["left", "middle", "right"] = "left",
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Drag from one screen point to another in logical coordinates."""
|
|
151
|
+
raise_if_tool_cancelled()
|
|
152
|
+
if duration < 0:
|
|
153
|
+
raise ValueError("duration must be >= 0")
|
|
154
|
+
if self.dry_run:
|
|
155
|
+
return
|
|
156
|
+
pyautogui = _load_pyautogui()
|
|
157
|
+
pyautogui.moveTo(start_x, start_y)
|
|
158
|
+
pyautogui.dragTo(end_x, end_y, duration=duration, button=button)
|
|
159
|
+
|
|
125
160
|
def wait_ms(self, milliseconds: int) -> None:
|
|
126
161
|
"""Wait for a fixed duration."""
|
|
127
162
|
if milliseconds < 0:
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Cooperative cancellation for a timed-out MCP tool call.
|
|
2
|
+
|
|
3
|
+
The MCP adapter binds one :class:`threading.Event` for the worker thread. On
|
|
4
|
+
timeout it sets the event. Actuation and the flow runner check it and refuse
|
|
5
|
+
to start another click, keystroke, scroll, drag, or flow step.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import threading
|
|
11
|
+
from contextvars import ContextVar, Token
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ToolCancelled(RuntimeError):
|
|
15
|
+
"""The current tool call was cancelled after its wall-clock timeout."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
_cancellation: ContextVar[threading.Event | None] = ContextVar(
|
|
19
|
+
"openframe_tool_cancellation", default=None
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def bind_tool_cancellation(event: threading.Event) -> Token[threading.Event | None]:
|
|
24
|
+
"""Bind ``event`` for the current thread. Reset with the returned token."""
|
|
25
|
+
return _cancellation.set(event)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def reset_tool_cancellation(token: Token[threading.Event | None]) -> None:
|
|
29
|
+
"""Restore the cancellation state captured by :func:`bind_tool_cancellation`."""
|
|
30
|
+
_cancellation.reset(token)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def tool_cancelled() -> bool:
|
|
34
|
+
"""Return True when the current tool call has been cancelled."""
|
|
35
|
+
event = _cancellation.get()
|
|
36
|
+
return event is not None and event.is_set()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def raise_if_tool_cancelled() -> None:
|
|
40
|
+
"""Raise when the current tool call has been cancelled."""
|
|
41
|
+
if tool_cancelled():
|
|
42
|
+
raise ToolCancelled("Tool call was cancelled after timeout; refusing to actuate.")
|
|
@@ -11,6 +11,7 @@ import tempfile
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
|
+
from openframe.capture.sidecar import write_frame_sidecar
|
|
14
15
|
from openframe.types import Frame
|
|
15
16
|
|
|
16
17
|
|
|
@@ -18,6 +19,9 @@ class CaptureError(RuntimeError):
|
|
|
18
19
|
"""Raised when a capture operation fails."""
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
_SUBPROCESS_TIMEOUT_SEC = 30
|
|
23
|
+
|
|
24
|
+
|
|
21
25
|
def screen(out_path: str | Path | None = None) -> Frame:
|
|
22
26
|
"""Capture the primary display to PNG and return frame metadata."""
|
|
23
27
|
_ensure_macos()
|
|
@@ -26,7 +30,10 @@ def screen(out_path: str | Path | None = None) -> Frame:
|
|
|
26
30
|
_run_command(["screencapture", "-x", "-D", "1", str(image_path)])
|
|
27
31
|
|
|
28
32
|
width, height = _read_png_dimensions(image_path)
|
|
29
|
-
|
|
33
|
+
left, top, logical_width, _logical_height = _main_display_bounds()
|
|
34
|
+
scale_factor = _scale_from_logical_width(
|
|
35
|
+
physical_width=width, logical_width=logical_width
|
|
36
|
+
)
|
|
30
37
|
metadata = {
|
|
31
38
|
"backend": "screencapture",
|
|
32
39
|
"display_index": 1,
|
|
@@ -34,14 +41,18 @@ def screen(out_path: str | Path | None = None) -> Frame:
|
|
|
34
41
|
"physical_height": height,
|
|
35
42
|
}
|
|
36
43
|
|
|
37
|
-
|
|
44
|
+
frame = Frame(
|
|
38
45
|
width=width,
|
|
39
46
|
height=height,
|
|
40
47
|
scale_factor=scale_factor,
|
|
41
48
|
source="screen:1",
|
|
42
49
|
image_path=str(image_path),
|
|
43
50
|
metadata=metadata,
|
|
51
|
+
origin_x=left,
|
|
52
|
+
origin_y=top,
|
|
44
53
|
)
|
|
54
|
+
write_frame_sidecar(frame)
|
|
55
|
+
return frame
|
|
45
56
|
|
|
46
57
|
|
|
47
58
|
def window(
|
|
@@ -54,11 +65,22 @@ def window(
|
|
|
54
65
|
_ensure_macos()
|
|
55
66
|
|
|
56
67
|
selected = _select_window_info(title=title, window_id=window_id)
|
|
68
|
+
bounds = selected.get("bounds") if isinstance(selected.get("bounds"), dict) else {}
|
|
69
|
+
logical_width = int(bounds.get("width") or 0)
|
|
70
|
+
logical_height = int(bounds.get("height") or 0)
|
|
71
|
+
if logical_width <= 0 or logical_height <= 0:
|
|
72
|
+
raise CaptureError("Window bounds are missing, so the capture scale is unknown.")
|
|
73
|
+
origin_x = int(bounds.get("x") or 0)
|
|
74
|
+
origin_y = int(bounds.get("y") or 0)
|
|
75
|
+
|
|
57
76
|
image_path = _resolve_output_path(out_path)
|
|
58
|
-
|
|
77
|
+
# -o drops the window shadow so the PNG maps onto the window bounds.
|
|
78
|
+
_run_command(["screencapture", "-x", "-o", "-l", str(selected["id"]), str(image_path)])
|
|
59
79
|
|
|
60
80
|
width, height = _read_png_dimensions(image_path)
|
|
61
|
-
scale_factor =
|
|
81
|
+
scale_factor = _scale_from_logical_width(
|
|
82
|
+
physical_width=width, logical_width=logical_width
|
|
83
|
+
)
|
|
62
84
|
metadata = {
|
|
63
85
|
"backend": "screencapture",
|
|
64
86
|
"window_id": selected["id"],
|
|
@@ -66,16 +88,22 @@ def window(
|
|
|
66
88
|
"window_owner": selected["owner"],
|
|
67
89
|
"physical_width": width,
|
|
68
90
|
"physical_height": height,
|
|
91
|
+
"logical_width": logical_width,
|
|
92
|
+
"logical_height": logical_height,
|
|
69
93
|
}
|
|
70
94
|
|
|
71
|
-
|
|
95
|
+
frame = Frame(
|
|
72
96
|
width=width,
|
|
73
97
|
height=height,
|
|
74
98
|
scale_factor=scale_factor,
|
|
75
99
|
source=f"window:{selected['id']}",
|
|
76
100
|
image_path=str(image_path),
|
|
77
101
|
metadata=metadata,
|
|
102
|
+
origin_x=origin_x,
|
|
103
|
+
origin_y=origin_y,
|
|
78
104
|
)
|
|
105
|
+
write_frame_sidecar(frame)
|
|
106
|
+
return frame
|
|
79
107
|
|
|
80
108
|
|
|
81
109
|
def region(
|
|
@@ -96,7 +124,7 @@ def region(
|
|
|
96
124
|
_run_command(["screencapture", "-x", "-R", region_spec, str(image_path)])
|
|
97
125
|
|
|
98
126
|
physical_width, physical_height = _read_png_dimensions(image_path)
|
|
99
|
-
scale_factor =
|
|
127
|
+
scale_factor = _scale_from_logical_width(physical_width=physical_width, logical_width=width)
|
|
100
128
|
metadata = {
|
|
101
129
|
"backend": "screencapture",
|
|
102
130
|
"region": {"x": x, "y": y, "width": width, "height": height},
|
|
@@ -104,14 +132,18 @@ def region(
|
|
|
104
132
|
"physical_height": physical_height,
|
|
105
133
|
}
|
|
106
134
|
|
|
107
|
-
|
|
135
|
+
frame = Frame(
|
|
108
136
|
width=physical_width,
|
|
109
137
|
height=physical_height,
|
|
110
138
|
scale_factor=scale_factor,
|
|
111
139
|
source=f"region:{x},{y},{width},{height}",
|
|
112
140
|
image_path=str(image_path),
|
|
113
141
|
metadata=metadata,
|
|
142
|
+
origin_x=x,
|
|
143
|
+
origin_y=y,
|
|
114
144
|
)
|
|
145
|
+
write_frame_sidecar(frame)
|
|
146
|
+
return frame
|
|
115
147
|
|
|
116
148
|
|
|
117
149
|
def list_windows() -> list[dict[str, Any]]:
|
|
@@ -170,7 +202,18 @@ def _resolve_output_path(out_path: str | Path | None) -> Path:
|
|
|
170
202
|
|
|
171
203
|
|
|
172
204
|
def _run_command(command: list[str]) -> subprocess.CompletedProcess[str]:
|
|
173
|
-
|
|
205
|
+
try:
|
|
206
|
+
completed = subprocess.run(
|
|
207
|
+
command,
|
|
208
|
+
check=False,
|
|
209
|
+
capture_output=True,
|
|
210
|
+
text=True,
|
|
211
|
+
timeout=_SUBPROCESS_TIMEOUT_SEC,
|
|
212
|
+
)
|
|
213
|
+
except subprocess.TimeoutExpired as exc:
|
|
214
|
+
raise CaptureError(
|
|
215
|
+
f"Capture command timed out after {_SUBPROCESS_TIMEOUT_SEC:g}s: {' '.join(command)}"
|
|
216
|
+
) from exc
|
|
174
217
|
if completed.returncode != 0:
|
|
175
218
|
stderr = completed.stderr.strip() or "unknown error"
|
|
176
219
|
raise CaptureError(f"Capture command failed: {' '.join(command)} ({stderr})")
|
|
@@ -272,7 +315,8 @@ def _read_png_dimensions(path: Path) -> tuple[int, int]:
|
|
|
272
315
|
return width, height
|
|
273
316
|
|
|
274
317
|
|
|
275
|
-
def
|
|
318
|
+
def _main_display_bounds() -> tuple[int, int, int, int]:
|
|
319
|
+
"""Return the main display as left, top, width, height in logical points."""
|
|
276
320
|
try:
|
|
277
321
|
completed = _run_command(
|
|
278
322
|
[
|
|
@@ -281,24 +325,35 @@ def _detect_scale_factor(physical_width: int) -> float:
|
|
|
281
325
|
'tell application "Finder" to get bounds of window of desktop',
|
|
282
326
|
]
|
|
283
327
|
)
|
|
284
|
-
except CaptureError:
|
|
285
|
-
|
|
328
|
+
except CaptureError as exc:
|
|
329
|
+
raise CaptureError(
|
|
330
|
+
"Cannot read the main display size, so the capture scale is unknown."
|
|
331
|
+
) from exc
|
|
286
332
|
|
|
287
333
|
values = [segment.strip() for segment in completed.stdout.strip().split(",")]
|
|
288
334
|
if len(values) != 4:
|
|
289
|
-
|
|
335
|
+
raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
|
|
290
336
|
|
|
291
337
|
try:
|
|
292
|
-
left,
|
|
293
|
-
except ValueError:
|
|
294
|
-
|
|
338
|
+
left, top, right, bottom = (int(value) for value in values)
|
|
339
|
+
except ValueError as exc:
|
|
340
|
+
raise CaptureError(
|
|
341
|
+
"Cannot read the main display size, so the capture scale is unknown."
|
|
342
|
+
) from exc
|
|
295
343
|
|
|
296
344
|
logical_width = right - left
|
|
297
|
-
|
|
298
|
-
|
|
345
|
+
logical_height = bottom - top
|
|
346
|
+
if logical_width <= 0 or logical_height <= 0:
|
|
347
|
+
raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
|
|
348
|
+
return left, top, logical_width, logical_height
|
|
349
|
+
|
|
299
350
|
|
|
351
|
+
def _scale_from_logical_width(*, physical_width: int, logical_width: int) -> float:
|
|
352
|
+
"""Return physical pixels per logical point for sizes that are already known."""
|
|
353
|
+
if physical_width <= 0 or logical_width <= 0:
|
|
354
|
+
raise CaptureError("Cannot compute scale: width must be positive.")
|
|
300
355
|
ratio = physical_width / logical_width
|
|
301
356
|
for expected in (1.0, 2.0, 3.0):
|
|
302
|
-
if abs(ratio - expected)
|
|
357
|
+
if abs(ratio - expected) <= 0.02:
|
|
303
358
|
return expected
|
|
304
359
|
return round(ratio, 2)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Metadata written beside a capture so a later click can map image pixels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from openframe.types import Frame
|
|
9
|
+
|
|
10
|
+
SIDECAR_SUFFIX = ".openframe.json"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def sidecar_path(image_path: Path) -> Path:
|
|
14
|
+
"""Return the metadata path for a capture PNG."""
|
|
15
|
+
return Path(str(image_path) + SIDECAR_SUFFIX)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def write_frame_sidecar(frame: Frame) -> None:
|
|
19
|
+
"""Write scale and origin next to ``frame.image_path``."""
|
|
20
|
+
if not frame.image_path:
|
|
21
|
+
return
|
|
22
|
+
payload = {
|
|
23
|
+
"width": frame.width,
|
|
24
|
+
"height": frame.height,
|
|
25
|
+
"scale_factor": frame.scale_factor,
|
|
26
|
+
"origin_x": frame.origin_x,
|
|
27
|
+
"origin_y": frame.origin_y,
|
|
28
|
+
"source": frame.source,
|
|
29
|
+
}
|
|
30
|
+
path = sidecar_path(Path(frame.image_path))
|
|
31
|
+
path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_frame_sidecar(image_path: Path) -> Frame | None:
|
|
35
|
+
"""Load a frame from its sidecar, or None when the sidecar is absent."""
|
|
36
|
+
path = sidecar_path(image_path)
|
|
37
|
+
if not path.is_file():
|
|
38
|
+
return None
|
|
39
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
40
|
+
return Frame(
|
|
41
|
+
width=int(payload["width"]),
|
|
42
|
+
height=int(payload["height"]),
|
|
43
|
+
scale_factor=float(payload["scale_factor"]),
|
|
44
|
+
source=str(payload.get("source") or f"file:{image_path.name}"),
|
|
45
|
+
image_path=str(image_path),
|
|
46
|
+
origin_x=int(payload.get("origin_x", 0)),
|
|
47
|
+
origin_y=int(payload.get("origin_y", 0)),
|
|
48
|
+
metadata={"input": "file", "sidecar": str(path)},
|
|
49
|
+
)
|