off-camber-open-frame 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {off_camber_open_frame-0.2.0/src/off_camber_open_frame.egg-info → off_camber_open_frame-0.2.2}/PKG-INFO +8 -4
  2. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/README.md +4 -3
  3. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/pyproject.toml +5 -1
  4. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2/src/off_camber_open_frame.egg-info}/PKG-INFO +8 -4
  5. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/SOURCES.txt +9 -0
  6. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/requires.txt +4 -0
  7. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/act/core.py +42 -7
  8. off_camber_open_frame-0.2.2/src/openframe/cancel.py +42 -0
  9. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/capture/macos.py +73 -18
  10. off_camber_open_frame-0.2.2/src/openframe/capture/sidecar.py +49 -0
  11. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/cli.py +64 -9
  12. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/doctor.py +1 -0
  13. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/adapter.py +251 -25
  14. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/server.py +37 -4
  15. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/__init__.py +3 -0
  16. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/macos.py +27 -13
  17. off_camber_open_frame-0.2.2/src/openframe/recognize/defaults.py +45 -0
  18. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/locator.py +11 -0
  19. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/match.py +3 -1
  20. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/tesseract.py +42 -16
  21. off_camber_open_frame-0.2.2/src/openframe/recognize/template/__init__.py +5 -0
  22. off_camber_open_frame-0.2.2/src/openframe/recognize/template/matcher.py +204 -0
  23. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/runner.py +544 -48
  24. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/session.py +7 -4
  25. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/types.py +2 -0
  26. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/window.py +22 -11
  27. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_a11y_recognizer.py +61 -0
  28. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_actuator.py +103 -0
  29. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_capture.py +24 -12
  30. off_camber_open_frame-0.2.2/tests/test_capture_geometry.py +158 -0
  31. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_cli.py +16 -22
  32. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_examples.py +36 -0
  33. off_camber_open_frame-0.2.2/tests/test_geometry_checks.py +48 -0
  34. off_camber_open_frame-0.2.2/tests/test_market_validation_tools.py +288 -0
  35. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_match.py +3 -0
  36. off_camber_open_frame-0.2.2/tests/test_mcp_adapter.py +468 -0
  37. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_mcp_server.py +5 -1
  38. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_recognize_locator.py +34 -0
  39. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_runner.py +394 -55
  40. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_selector_parity.py +4 -5
  41. off_camber_open_frame-0.2.2/tests/test_template_recognizer.py +179 -0
  42. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_tesseract_recognizer.py +87 -10
  43. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_window_state.py +60 -6
  44. off_camber_open_frame-0.2.0/tests/test_mcp_adapter.py +0 -173
  45. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/LICENSE +0 -0
  46. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/setup.cfg +0 -0
  47. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/dependency_links.txt +0 -0
  48. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/entry_points.txt +0 -0
  49. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/top_level.txt +0 -0
  50. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/__init__.py +0 -0
  51. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/act/__init__.py +0 -0
  52. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/__init__.py +0 -0
  53. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/base.py +0 -0
  54. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/cost.py +0 -0
  55. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/__init__.py +0 -0
  56. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/anthropic_provider.py +0 -0
  57. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/agent/runner.py +0 -0
  58. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/capture/__init__.py +0 -0
  59. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/flow.py +0 -0
  60. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/__init__.py +0 -0
  61. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/__init__.py +0 -0
  62. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/__init__.py +0 -0
  63. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/base.py +0 -0
  64. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/coords.py +0 -0
  65. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/debug.py +0 -0
  66. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/__init__.py +0 -0
  67. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/verify/__init__.py +0 -0
  68. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/verify/base.py +0 -0
  69. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/src/openframe/verify/core.py +0 -0
  70. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_agent_anthropic_provider.py +0 -0
  71. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_agent_cost.py +0 -0
  72. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_agent_runner.py +0 -0
  73. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_calibration_fixture.py +0 -0
  74. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_doctor.py +0 -0
  75. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_flow.py +0 -0
  76. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_imports.py +0 -0
  77. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_recognize_coords.py +0 -0
  78. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_recognize_debug.py +0 -0
  79. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_session_api.py +0 -0
  80. {off_camber_open_frame-0.2.0 → off_camber_open_frame-0.2.2}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: off-camber-open-frame
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Open-source recognition and action for anything on a display.
5
5
  Author: Off-Camber
6
6
  License-Expression: Apache-2.0
@@ -32,6 +32,9 @@ Provides-Extra: agent
32
32
  Requires-Dist: anthropic; extra == "agent"
33
33
  Provides-Extra: mcp
34
34
  Requires-Dist: mcp<2,>=1.0; extra == "mcp"
35
+ Provides-Extra: template
36
+ Requires-Dist: Pillow; extra == "template"
37
+ Requires-Dist: numpy; extra == "template"
35
38
  Dynamic: license-file
36
39
 
37
40
  <p align="center">
@@ -62,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
62
65
 
63
66
  ## Status
64
67
 
65
- **Beta** (`v0.2.0`) — macOS-first. Available on PyPI as
68
+ **Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
66
69
  [`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
67
70
  Evidence for this release comes from CI plus a local macOS live gate (including
68
71
  MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
@@ -79,7 +82,7 @@ MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
79
82
  ### 1. Install
80
83
 
81
84
  ```bash
82
- pip install "off-camber-open-frame[mcp,ocr,act,flow]"
85
+ pip install "off-camber-open-frame[mcp,ocr,act,flow,template]"
83
86
  brew install tesseract
84
87
  ```
85
88
 
@@ -123,7 +126,7 @@ python -m openframe.cli doctor
123
126
  ```bash
124
127
  python3.11 -m venv .venv
125
128
  source .venv/bin/activate
126
- pip install -e ".[dev,ocr,act,flow,mcp]"
129
+ pip install -e ".[dev,ocr,act,flow,mcp,template]"
127
130
  open-frame doctor
128
131
  open-frame mcp list-tools --json
129
132
  ```
@@ -138,6 +141,7 @@ brew install python@3.11
138
141
 
139
142
  - [MCP server setup](docs/MCP_SERVER.md) — Claude Desktop / Cursor config.
140
143
  - [Flow setup](docs/FLOW_SETUP.md) — define and run YAML flows.
144
+ - [Template setup](docs/TEMPLATE_SETUP.md) — optional icon/image matching.
141
145
  - [API](docs/API.md) — use `Session` and MCP-oriented integration guidance.
142
146
  - [Act setup](docs/ACT_SETUP.md) and [Verify setup](docs/VERIFY_SETUP.md) — run safely with evidence.
143
147
  - [Full docs index](docs/README.md) — contributor and planning docs.
@@ -26,7 +26,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
26
26
 
27
27
  ## Status
28
28
 
29
- **Beta** (`v0.2.0`) — macOS-first. Available on PyPI as
29
+ **Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
30
30
  [`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
31
31
  Evidence for this release comes from CI plus a local macOS live gate (including
32
32
  MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
@@ -43,7 +43,7 @@ MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
43
43
  ### 1. Install
44
44
 
45
45
  ```bash
46
- pip install "off-camber-open-frame[mcp,ocr,act,flow]"
46
+ pip install "off-camber-open-frame[mcp,ocr,act,flow,template]"
47
47
  brew install tesseract
48
48
  ```
49
49
 
@@ -87,7 +87,7 @@ python -m openframe.cli doctor
87
87
  ```bash
88
88
  python3.11 -m venv .venv
89
89
  source .venv/bin/activate
90
- pip install -e ".[dev,ocr,act,flow,mcp]"
90
+ pip install -e ".[dev,ocr,act,flow,mcp,template]"
91
91
  open-frame doctor
92
92
  open-frame mcp list-tools --json
93
93
  ```
@@ -102,6 +102,7 @@ brew install python@3.11
102
102
 
103
103
  - [MCP server setup](docs/MCP_SERVER.md) — Claude Desktop / Cursor config.
104
104
  - [Flow setup](docs/FLOW_SETUP.md) — define and run YAML flows.
105
+ - [Template setup](docs/TEMPLATE_SETUP.md) — optional icon/image matching.
105
106
  - [API](docs/API.md) — use `Session` and MCP-oriented integration guidance.
106
107
  - [Act setup](docs/ACT_SETUP.md) and [Verify setup](docs/VERIFY_SETUP.md) — run safely with evidence.
107
108
  - [Full docs index](docs/README.md) — contributor and planning docs.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "off-camber-open-frame"
7
- version = "0.2.0"
7
+ version = "0.2.2"
8
8
  description = "Open-source recognition and action for anything on a display."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -51,6 +51,10 @@ agent = [
51
51
  mcp = [
52
52
  "mcp>=1.0,<2",
53
53
  ]
54
+ template = [
55
+ "Pillow",
56
+ "numpy",
57
+ ]
54
58
 
55
59
  [tool.setuptools]
56
60
  package-dir = { "" = "src" }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: off-camber-open-frame
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: Open-source recognition and action for anything on a display.
5
5
  Author: Off-Camber
6
6
  License-Expression: Apache-2.0
@@ -32,6 +32,9 @@ Provides-Extra: agent
32
32
  Requires-Dist: anthropic; extra == "agent"
33
33
  Provides-Extra: mcp
34
34
  Requires-Dist: mcp<2,>=1.0; extra == "mcp"
35
+ Provides-Extra: template
36
+ Requires-Dist: Pillow; extra == "template"
37
+ Requires-Dist: numpy; extra == "template"
35
38
  Dynamic: license-file
36
39
 
37
40
  <p align="center">
@@ -62,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
62
65
 
63
66
  ## Status
64
67
 
65
- **Beta** (`v0.2.0`) — macOS-first. Available on PyPI as
68
+ **Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
66
69
  [`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
67
70
  Evidence for this release comes from CI plus a local macOS live gate (including
68
71
  MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
@@ -79,7 +82,7 @@ MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
79
82
  ### 1. Install
80
83
 
81
84
  ```bash
82
- pip install "off-camber-open-frame[mcp,ocr,act,flow]"
85
+ pip install "off-camber-open-frame[mcp,ocr,act,flow,template]"
83
86
  brew install tesseract
84
87
  ```
85
88
 
@@ -123,7 +126,7 @@ python -m openframe.cli doctor
123
126
  ```bash
124
127
  python3.11 -m venv .venv
125
128
  source .venv/bin/activate
126
- pip install -e ".[dev,ocr,act,flow,mcp]"
129
+ pip install -e ".[dev,ocr,act,flow,mcp,template]"
127
130
  open-frame doctor
128
131
  open-frame mcp list-tools --json
129
132
  ```
@@ -138,6 +141,7 @@ brew install python@3.11
138
141
 
139
142
  - [MCP server setup](docs/MCP_SERVER.md) — Claude Desktop / Cursor config.
140
143
  - [Flow setup](docs/FLOW_SETUP.md) — define and run YAML flows.
144
+ - [Template setup](docs/TEMPLATE_SETUP.md) — optional icon/image matching.
141
145
  - [API](docs/API.md) — use `Session` and MCP-oriented integration guidance.
142
146
  - [Act setup](docs/ACT_SETUP.md) and [Verify setup](docs/VERIFY_SETUP.md) — run safely with evidence.
143
147
  - [Full docs index](docs/README.md) — contributor and planning docs.
@@ -8,6 +8,7 @@ src/off_camber_open_frame.egg-info/entry_points.txt
8
8
  src/off_camber_open_frame.egg-info/requires.txt
9
9
  src/off_camber_open_frame.egg-info/top_level.txt
10
10
  src/openframe/__init__.py
11
+ src/openframe/cancel.py
11
12
  src/openframe/cli.py
12
13
  src/openframe/doctor.py
13
14
  src/openframe/flow.py
@@ -25,6 +26,7 @@ src/openframe/agent/providers/__init__.py
25
26
  src/openframe/agent/providers/anthropic_provider.py
26
27
  src/openframe/capture/__init__.py
27
28
  src/openframe/capture/macos.py
29
+ src/openframe/capture/sidecar.py
28
30
  src/openframe/integrations/__init__.py
29
31
  src/openframe/integrations/mcp/__init__.py
30
32
  src/openframe/integrations/mcp/adapter.py
@@ -33,12 +35,15 @@ src/openframe/recognize/__init__.py
33
35
  src/openframe/recognize/base.py
34
36
  src/openframe/recognize/coords.py
35
37
  src/openframe/recognize/debug.py
38
+ src/openframe/recognize/defaults.py
36
39
  src/openframe/recognize/locator.py
37
40
  src/openframe/recognize/match.py
38
41
  src/openframe/recognize/a11y/__init__.py
39
42
  src/openframe/recognize/a11y/macos.py
40
43
  src/openframe/recognize/ocr/__init__.py
41
44
  src/openframe/recognize/ocr/tesseract.py
45
+ src/openframe/recognize/template/__init__.py
46
+ src/openframe/recognize/template/matcher.py
42
47
  src/openframe/verify/__init__.py
43
48
  src/openframe/verify/base.py
44
49
  src/openframe/verify/core.py
@@ -49,11 +54,14 @@ tests/test_agent_cost.py
49
54
  tests/test_agent_runner.py
50
55
  tests/test_calibration_fixture.py
51
56
  tests/test_capture.py
57
+ tests/test_capture_geometry.py
52
58
  tests/test_cli.py
53
59
  tests/test_doctor.py
54
60
  tests/test_examples.py
55
61
  tests/test_flow.py
62
+ tests/test_geometry_checks.py
56
63
  tests/test_imports.py
64
+ tests/test_market_validation_tools.py
57
65
  tests/test_match.py
58
66
  tests/test_mcp_adapter.py
59
67
  tests/test_mcp_server.py
@@ -63,6 +71,7 @@ tests/test_recognize_locator.py
63
71
  tests/test_runner.py
64
72
  tests/test_selector_parity.py
65
73
  tests/test_session_api.py
74
+ tests/test_template_recognizer.py
66
75
  tests/test_tesseract_recognizer.py
67
76
  tests/test_verify.py
68
77
  tests/test_window_state.py
@@ -18,3 +18,7 @@ mcp<2,>=1.0
18
18
  [ocr]
19
19
  Pillow
20
20
  pytesseract
21
+
22
+ [template]
23
+ Pillow
24
+ numpy
@@ -6,6 +6,7 @@ import time
6
6
  from collections.abc import Callable
7
7
  from typing import Literal
8
8
 
9
+ from openframe.cancel import raise_if_tool_cancelled
9
10
  from openframe.recognize.coords import target_uses_physical_pixels
10
11
  from openframe.types import Frame, Target
11
12
 
@@ -29,13 +30,15 @@ class Actuator:
29
30
  *,
30
31
  anchor: ClickAnchor = "center",
31
32
  scale_factor: float = 1.0,
33
+ origin: tuple[int, int] = (0, 0),
32
34
  ) -> tuple[int, int]:
33
35
  """Map target bounds to a click point.
34
36
 
35
- Recognizers report target bounds in captured-image pixels (physical
36
- pixels on Retina displays). Mouse actuation works in logical screen
37
- points. ``scale_factor`` (display physical/logical ratio, e.g. 2.0 on
38
- Retina) converts the mapped point into logical click space.
37
+ Recognizers report image matches in captured-image pixels. Mouse
38
+ actuation works in logical screen points. ``scale_factor`` converts
39
+ those pixels, then ``origin`` (the capture's logical top-left) places
40
+ the point on the screen. Accessibility targets are already logical
41
+ screen points, so ``origin`` is not added for them.
39
42
  """
40
43
  if scale_factor <= 0:
41
44
  raise ValueError("scale_factor must be greater than zero.")
@@ -54,7 +57,8 @@ class Actuator:
54
57
  raise ValueError(f"Unsupported click anchor: {anchor}")
55
58
 
56
59
  if target_uses_physical_pixels(target):
57
- return (round(px / scale_factor), round(py / scale_factor))
60
+ origin_x, origin_y = origin
61
+ return (round(px / scale_factor) + origin_x, round(py / scale_factor) + origin_y)
58
62
  return (px, py)
59
63
 
60
64
  def click_target(
@@ -64,14 +68,18 @@ class Actuator:
64
68
  anchor: ClickAnchor = "center",
65
69
  kind: ClickKind = "click",
66
70
  scale_factor: float = 1.0,
71
+ origin: tuple[int, int] = (0, 0),
67
72
  ) -> tuple[int, int]:
68
73
  """Click a target at a mapped point in logical click space."""
69
- x, y = self.point_for_target(target, anchor=anchor, scale_factor=scale_factor)
74
+ x, y = self.point_for_target(
75
+ target, anchor=anchor, scale_factor=scale_factor, origin=origin
76
+ )
70
77
  self.click_point(x, y, kind=kind)
71
78
  return (x, y)
72
79
 
73
80
  def click_point(self, x: int, y: int, *, kind: ClickKind = "click") -> None:
74
81
  """Click at absolute screen coordinates."""
82
+ raise_if_tool_cancelled()
75
83
  if self.dry_run:
76
84
  return
77
85
 
@@ -89,6 +97,7 @@ class Actuator:
89
97
 
90
98
  def type_text(self, text: str, *, interval: float = 0.0) -> None:
91
99
  """Type plain text at current focus."""
100
+ raise_if_tool_cancelled()
92
101
  if self.dry_run:
93
102
  return
94
103
  pyautogui = _load_pyautogui()
@@ -96,6 +105,7 @@ class Actuator:
96
105
 
97
106
  def key_combo(self, *keys: str) -> None:
98
107
  """Press a key combination, e.g. ('command', 'v')."""
108
+ raise_if_tool_cancelled()
99
109
  if self.dry_run:
100
110
  return
101
111
  if not keys:
@@ -105,6 +115,7 @@ class Actuator:
105
115
 
106
116
  def press_key(self, key: str) -> None:
107
117
  """Press a single key."""
118
+ raise_if_tool_cancelled()
108
119
  if self.dry_run:
109
120
  return
110
121
  if not key:
@@ -113,7 +124,11 @@ class Actuator:
113
124
  pyautogui.press(key)
114
125
 
115
126
  def scroll(self, clicks: int, *, x: int | None = None, y: int | None = None) -> None:
116
- """Scroll at a point (or current cursor position)."""
127
+ """Scroll at a point (or current cursor position).
128
+
129
+ Positive ``clicks`` scroll up; negative scroll down (pyautogui convention).
130
+ """
131
+ raise_if_tool_cancelled()
117
132
  if self.dry_run:
118
133
  return
119
134
  pyautogui = _load_pyautogui()
@@ -122,6 +137,26 @@ class Actuator:
122
137
  return
123
138
  pyautogui.scroll(clicks)
124
139
 
140
+ def drag(
141
+ self,
142
+ start_x: int,
143
+ start_y: int,
144
+ end_x: int,
145
+ end_y: int,
146
+ *,
147
+ duration: float = 0.2,
148
+ button: Literal["left", "middle", "right"] = "left",
149
+ ) -> None:
150
+ """Drag from one screen point to another in logical coordinates."""
151
+ raise_if_tool_cancelled()
152
+ if duration < 0:
153
+ raise ValueError("duration must be >= 0")
154
+ if self.dry_run:
155
+ return
156
+ pyautogui = _load_pyautogui()
157
+ pyautogui.moveTo(start_x, start_y)
158
+ pyautogui.dragTo(end_x, end_y, duration=duration, button=button)
159
+
125
160
  def wait_ms(self, milliseconds: int) -> None:
126
161
  """Wait for a fixed duration."""
127
162
  if milliseconds < 0:
@@ -0,0 +1,42 @@
1
+ """Cooperative cancellation for a timed-out MCP tool call.
2
+
3
+ The MCP adapter binds one :class:`threading.Event` for the worker thread. On
4
+ timeout it sets the event. Actuation and the flow runner check it and refuse
5
+ to start another click, keystroke, scroll, drag, or flow step.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import threading
11
+ from contextvars import ContextVar, Token
12
+
13
+
14
+ class ToolCancelled(RuntimeError):
15
+ """The current tool call was cancelled after its wall-clock timeout."""
16
+
17
+
18
+ _cancellation: ContextVar[threading.Event | None] = ContextVar(
19
+ "openframe_tool_cancellation", default=None
20
+ )
21
+
22
+
23
+ def bind_tool_cancellation(event: threading.Event) -> Token[threading.Event | None]:
24
+ """Bind ``event`` for the current thread. Reset with the returned token."""
25
+ return _cancellation.set(event)
26
+
27
+
28
+ def reset_tool_cancellation(token: Token[threading.Event | None]) -> None:
29
+ """Restore the cancellation state captured by :func:`bind_tool_cancellation`."""
30
+ _cancellation.reset(token)
31
+
32
+
33
+ def tool_cancelled() -> bool:
34
+ """Return True when the current tool call has been cancelled."""
35
+ event = _cancellation.get()
36
+ return event is not None and event.is_set()
37
+
38
+
39
+ def raise_if_tool_cancelled() -> None:
40
+ """Raise when the current tool call has been cancelled."""
41
+ if tool_cancelled():
42
+ raise ToolCancelled("Tool call was cancelled after timeout; refusing to actuate.")
@@ -11,6 +11,7 @@ import tempfile
11
11
  from pathlib import Path
12
12
  from typing import Any
13
13
 
14
+ from openframe.capture.sidecar import write_frame_sidecar
14
15
  from openframe.types import Frame
15
16
 
16
17
 
@@ -18,6 +19,9 @@ class CaptureError(RuntimeError):
18
19
  """Raised when a capture operation fails."""
19
20
 
20
21
 
22
+ _SUBPROCESS_TIMEOUT_SEC = 30
23
+
24
+
21
25
  def screen(out_path: str | Path | None = None) -> Frame:
22
26
  """Capture the primary display to PNG and return frame metadata."""
23
27
  _ensure_macos()
@@ -26,7 +30,10 @@ def screen(out_path: str | Path | None = None) -> Frame:
26
30
  _run_command(["screencapture", "-x", "-D", "1", str(image_path)])
27
31
 
28
32
  width, height = _read_png_dimensions(image_path)
29
- scale_factor = _detect_scale_factor(width)
33
+ left, top, logical_width, _logical_height = _main_display_bounds()
34
+ scale_factor = _scale_from_logical_width(
35
+ physical_width=width, logical_width=logical_width
36
+ )
30
37
  metadata = {
31
38
  "backend": "screencapture",
32
39
  "display_index": 1,
@@ -34,14 +41,18 @@ def screen(out_path: str | Path | None = None) -> Frame:
34
41
  "physical_height": height,
35
42
  }
36
43
 
37
- return Frame(
44
+ frame = Frame(
38
45
  width=width,
39
46
  height=height,
40
47
  scale_factor=scale_factor,
41
48
  source="screen:1",
42
49
  image_path=str(image_path),
43
50
  metadata=metadata,
51
+ origin_x=left,
52
+ origin_y=top,
44
53
  )
54
+ write_frame_sidecar(frame)
55
+ return frame
45
56
 
46
57
 
47
58
  def window(
@@ -54,11 +65,22 @@ def window(
54
65
  _ensure_macos()
55
66
 
56
67
  selected = _select_window_info(title=title, window_id=window_id)
68
+ bounds = selected.get("bounds") if isinstance(selected.get("bounds"), dict) else {}
69
+ logical_width = int(bounds.get("width") or 0)
70
+ logical_height = int(bounds.get("height") or 0)
71
+ if logical_width <= 0 or logical_height <= 0:
72
+ raise CaptureError("Window bounds are missing, so the capture scale is unknown.")
73
+ origin_x = int(bounds.get("x") or 0)
74
+ origin_y = int(bounds.get("y") or 0)
75
+
57
76
  image_path = _resolve_output_path(out_path)
58
- _run_command(["screencapture", "-x", "-l", str(selected["id"]), str(image_path)])
77
+ # -o drops the window shadow so the PNG maps onto the window bounds.
78
+ _run_command(["screencapture", "-x", "-o", "-l", str(selected["id"]), str(image_path)])
59
79
 
60
80
  width, height = _read_png_dimensions(image_path)
61
- scale_factor = _detect_scale_factor(width)
81
+ scale_factor = _scale_from_logical_width(
82
+ physical_width=width, logical_width=logical_width
83
+ )
62
84
  metadata = {
63
85
  "backend": "screencapture",
64
86
  "window_id": selected["id"],
@@ -66,16 +88,22 @@ def window(
66
88
  "window_owner": selected["owner"],
67
89
  "physical_width": width,
68
90
  "physical_height": height,
91
+ "logical_width": logical_width,
92
+ "logical_height": logical_height,
69
93
  }
70
94
 
71
- return Frame(
95
+ frame = Frame(
72
96
  width=width,
73
97
  height=height,
74
98
  scale_factor=scale_factor,
75
99
  source=f"window:{selected['id']}",
76
100
  image_path=str(image_path),
77
101
  metadata=metadata,
102
+ origin_x=origin_x,
103
+ origin_y=origin_y,
78
104
  )
105
+ write_frame_sidecar(frame)
106
+ return frame
79
107
 
80
108
 
81
109
  def region(
@@ -96,7 +124,7 @@ def region(
96
124
  _run_command(["screencapture", "-x", "-R", region_spec, str(image_path)])
97
125
 
98
126
  physical_width, physical_height = _read_png_dimensions(image_path)
99
- scale_factor = _detect_scale_factor(physical_width)
127
+ scale_factor = _scale_from_logical_width(physical_width=physical_width, logical_width=width)
100
128
  metadata = {
101
129
  "backend": "screencapture",
102
130
  "region": {"x": x, "y": y, "width": width, "height": height},
@@ -104,14 +132,18 @@ def region(
104
132
  "physical_height": physical_height,
105
133
  }
106
134
 
107
- return Frame(
135
+ frame = Frame(
108
136
  width=physical_width,
109
137
  height=physical_height,
110
138
  scale_factor=scale_factor,
111
139
  source=f"region:{x},{y},{width},{height}",
112
140
  image_path=str(image_path),
113
141
  metadata=metadata,
142
+ origin_x=x,
143
+ origin_y=y,
114
144
  )
145
+ write_frame_sidecar(frame)
146
+ return frame
115
147
 
116
148
 
117
149
  def list_windows() -> list[dict[str, Any]]:
@@ -170,7 +202,18 @@ def _resolve_output_path(out_path: str | Path | None) -> Path:
170
202
 
171
203
 
172
204
  def _run_command(command: list[str]) -> subprocess.CompletedProcess[str]:
173
- completed = subprocess.run(command, check=False, capture_output=True, text=True)
205
+ try:
206
+ completed = subprocess.run(
207
+ command,
208
+ check=False,
209
+ capture_output=True,
210
+ text=True,
211
+ timeout=_SUBPROCESS_TIMEOUT_SEC,
212
+ )
213
+ except subprocess.TimeoutExpired as exc:
214
+ raise CaptureError(
215
+ f"Capture command timed out after {_SUBPROCESS_TIMEOUT_SEC:g}s: {' '.join(command)}"
216
+ ) from exc
174
217
  if completed.returncode != 0:
175
218
  stderr = completed.stderr.strip() or "unknown error"
176
219
  raise CaptureError(f"Capture command failed: {' '.join(command)} ({stderr})")
@@ -272,7 +315,8 @@ def _read_png_dimensions(path: Path) -> tuple[int, int]:
272
315
  return width, height
273
316
 
274
317
 
275
- def _detect_scale_factor(physical_width: int) -> float:
318
+ def _main_display_bounds() -> tuple[int, int, int, int]:
319
+ """Return the main display as left, top, width, height in logical points."""
276
320
  try:
277
321
  completed = _run_command(
278
322
  [
@@ -281,24 +325,35 @@ def _detect_scale_factor(physical_width: int) -> float:
281
325
  'tell application "Finder" to get bounds of window of desktop',
282
326
  ]
283
327
  )
284
- except CaptureError:
285
- return 1.0
328
+ except CaptureError as exc:
329
+ raise CaptureError(
330
+ "Cannot read the main display size, so the capture scale is unknown."
331
+ ) from exc
286
332
 
287
333
  values = [segment.strip() for segment in completed.stdout.strip().split(",")]
288
334
  if len(values) != 4:
289
- return 1.0
335
+ raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
290
336
 
291
337
  try:
292
- left, _top, right, _bottom = (int(value) for value in values)
293
- except ValueError:
294
- return 1.0
338
+ left, top, right, bottom = (int(value) for value in values)
339
+ except ValueError as exc:
340
+ raise CaptureError(
341
+ "Cannot read the main display size, so the capture scale is unknown."
342
+ ) from exc
295
343
 
296
344
  logical_width = right - left
297
- if logical_width <= 0:
298
- return 1.0
345
+ logical_height = bottom - top
346
+ if logical_width <= 0 or logical_height <= 0:
347
+ raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
348
+ return left, top, logical_width, logical_height
349
+
299
350
 
351
+ def _scale_from_logical_width(*, physical_width: int, logical_width: int) -> float:
352
+ """Return physical pixels per logical point for sizes that are already known."""
353
+ if physical_width <= 0 or logical_width <= 0:
354
+ raise CaptureError("Cannot compute scale: width must be positive.")
300
355
  ratio = physical_width / logical_width
301
356
  for expected in (1.0, 2.0, 3.0):
302
- if abs(ratio - expected) < 0.15:
357
+ if abs(ratio - expected) <= 0.02:
303
358
  return expected
304
359
  return round(ratio, 2)
@@ -0,0 +1,49 @@
1
+ """Metadata written beside a capture so a later click can map image pixels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ from openframe.types import Frame
9
+
10
+ SIDECAR_SUFFIX = ".openframe.json"
11
+
12
+
13
+ def sidecar_path(image_path: Path) -> Path:
14
+ """Return the metadata path for a capture PNG."""
15
+ return Path(str(image_path) + SIDECAR_SUFFIX)
16
+
17
+
18
+ def write_frame_sidecar(frame: Frame) -> None:
19
+ """Write scale and origin next to ``frame.image_path``."""
20
+ if not frame.image_path:
21
+ return
22
+ payload = {
23
+ "width": frame.width,
24
+ "height": frame.height,
25
+ "scale_factor": frame.scale_factor,
26
+ "origin_x": frame.origin_x,
27
+ "origin_y": frame.origin_y,
28
+ "source": frame.source,
29
+ }
30
+ path = sidecar_path(Path(frame.image_path))
31
+ path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
32
+
33
+
34
+ def load_frame_sidecar(image_path: Path) -> Frame | None:
35
+ """Load a frame from its sidecar, or None when the sidecar is absent."""
36
+ path = sidecar_path(image_path)
37
+ if not path.is_file():
38
+ return None
39
+ payload = json.loads(path.read_text(encoding="utf-8"))
40
+ return Frame(
41
+ width=int(payload["width"]),
42
+ height=int(payload["height"]),
43
+ scale_factor=float(payload["scale_factor"]),
44
+ source=str(payload.get("source") or f"file:{image_path.name}"),
45
+ image_path=str(image_path),
46
+ origin_x=int(payload.get("origin_x", 0)),
47
+ origin_y=int(payload.get("origin_y", 0)),
48
+ metadata={"input": "file", "sidecar": str(path)},
49
+ )