off-camber-open-frame 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {off_camber_open_frame-0.2.1/src/off_camber_open_frame.egg-info → off_camber_open_frame-0.2.2}/PKG-INFO +2 -2
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/README.md +1 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/pyproject.toml +1 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2/src/off_camber_open_frame.egg-info}/PKG-INFO +2 -2
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/SOURCES.txt +5 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/act/core.py +19 -6
- off_camber_open_frame-0.2.2/src/openframe/cancel.py +42 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/capture/macos.py +73 -18
- off_camber_open_frame-0.2.2/src/openframe/capture/sidecar.py +49 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/cli.py +21 -3
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/adapter.py +121 -33
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/macos.py +27 -13
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/locator.py +11 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/match.py +3 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/tesseract.py +42 -16
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/runner.py +216 -55
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/session.py +5 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/types.py +2 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/window.py +22 -11
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_a11y_recognizer.py +61 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_actuator.py +49 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_capture.py +24 -12
- off_camber_open_frame-0.2.2/tests/test_capture_geometry.py +158 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_cli.py +4 -4
- off_camber_open_frame-0.2.2/tests/test_geometry_checks.py +48 -0
- off_camber_open_frame-0.2.2/tests/test_market_validation_tools.py +288 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_match.py +3 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_mcp_adapter.py +223 -15
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_recognize_locator.py +34 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_runner.py +154 -4
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_selector_parity.py +2 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_template_recognizer.py +13 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_tesseract_recognizer.py +87 -10
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_window_state.py +57 -1
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/LICENSE +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/setup.cfg +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/dependency_links.txt +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/entry_points.txt +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/requires.txt +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/top_level.txt +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/act/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/base.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/cost.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/anthropic_provider.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/runner.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/capture/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/doctor.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/flow.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/server.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/base.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/coords.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/debug.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/defaults.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/template/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/template/matcher.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/verify/__init__.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/verify/base.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/verify/core.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_agent_anthropic_provider.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_agent_cost.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_agent_runner.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_calibration_fixture.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_doctor.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_examples.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_flow.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_imports.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_mcp_server.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_recognize_coords.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_recognize_debug.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_session_api.py +0 -0
- {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: off-camber-open-frame
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Open-source recognition and action for anything on a display.
|
|
5
5
|
Author: Off-Camber
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -65,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
|
|
|
65
65
|
|
|
66
66
|
## Status
|
|
67
67
|
|
|
68
|
-
**Beta** (`v0.2.
|
|
68
|
+
**Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
|
|
69
69
|
[`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
|
|
70
70
|
Evidence for this release comes from CI plus a local macOS live gate (including
|
|
71
71
|
MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
@@ -26,7 +26,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
|
|
|
26
26
|
|
|
27
27
|
## Status
|
|
28
28
|
|
|
29
|
-
**Beta** (`v0.2.
|
|
29
|
+
**Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
|
|
30
30
|
[`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
|
|
31
31
|
Evidence for this release comes from CI plus a local macOS live gate (including
|
|
32
32
|
MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: off-camber-open-frame
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Open-source recognition and action for anything on a display.
|
|
5
5
|
Author: Off-Camber
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -65,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
|
|
|
65
65
|
|
|
66
66
|
## Status
|
|
67
67
|
|
|
68
|
-
**Beta** (`v0.2.
|
|
68
|
+
**Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
|
|
69
69
|
[`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
|
|
70
70
|
Evidence for this release comes from CI plus a local macOS live gate (including
|
|
71
71
|
MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
|
|
@@ -8,6 +8,7 @@ src/off_camber_open_frame.egg-info/entry_points.txt
|
|
|
8
8
|
src/off_camber_open_frame.egg-info/requires.txt
|
|
9
9
|
src/off_camber_open_frame.egg-info/top_level.txt
|
|
10
10
|
src/openframe/__init__.py
|
|
11
|
+
src/openframe/cancel.py
|
|
11
12
|
src/openframe/cli.py
|
|
12
13
|
src/openframe/doctor.py
|
|
13
14
|
src/openframe/flow.py
|
|
@@ -25,6 +26,7 @@ src/openframe/agent/providers/__init__.py
|
|
|
25
26
|
src/openframe/agent/providers/anthropic_provider.py
|
|
26
27
|
src/openframe/capture/__init__.py
|
|
27
28
|
src/openframe/capture/macos.py
|
|
29
|
+
src/openframe/capture/sidecar.py
|
|
28
30
|
src/openframe/integrations/__init__.py
|
|
29
31
|
src/openframe/integrations/mcp/__init__.py
|
|
30
32
|
src/openframe/integrations/mcp/adapter.py
|
|
@@ -52,11 +54,14 @@ tests/test_agent_cost.py
|
|
|
52
54
|
tests/test_agent_runner.py
|
|
53
55
|
tests/test_calibration_fixture.py
|
|
54
56
|
tests/test_capture.py
|
|
57
|
+
tests/test_capture_geometry.py
|
|
55
58
|
tests/test_cli.py
|
|
56
59
|
tests/test_doctor.py
|
|
57
60
|
tests/test_examples.py
|
|
58
61
|
tests/test_flow.py
|
|
62
|
+
tests/test_geometry_checks.py
|
|
59
63
|
tests/test_imports.py
|
|
64
|
+
tests/test_market_validation_tools.py
|
|
60
65
|
tests/test_match.py
|
|
61
66
|
tests/test_mcp_adapter.py
|
|
62
67
|
tests/test_mcp_server.py
|
|
@@ -6,6 +6,7 @@ import time
|
|
|
6
6
|
from collections.abc import Callable
|
|
7
7
|
from typing import Literal
|
|
8
8
|
|
|
9
|
+
from openframe.cancel import raise_if_tool_cancelled
|
|
9
10
|
from openframe.recognize.coords import target_uses_physical_pixels
|
|
10
11
|
from openframe.types import Frame, Target
|
|
11
12
|
|
|
@@ -29,13 +30,15 @@ class Actuator:
|
|
|
29
30
|
*,
|
|
30
31
|
anchor: ClickAnchor = "center",
|
|
31
32
|
scale_factor: float = 1.0,
|
|
33
|
+
origin: tuple[int, int] = (0, 0),
|
|
32
34
|
) -> tuple[int, int]:
|
|
33
35
|
"""Map target bounds to a click point.
|
|
34
36
|
|
|
35
|
-
Recognizers report
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
Recognizers report image matches in captured-image pixels. Mouse
|
|
38
|
+
actuation works in logical screen points. ``scale_factor`` converts
|
|
39
|
+
those pixels, then ``origin`` (the capture's logical top-left) places
|
|
40
|
+
the point on the screen. Accessibility targets are already logical
|
|
41
|
+
screen points, so ``origin`` is not added for them.
|
|
39
42
|
"""
|
|
40
43
|
if scale_factor <= 0:
|
|
41
44
|
raise ValueError("scale_factor must be greater than zero.")
|
|
@@ -54,7 +57,8 @@ class Actuator:
|
|
|
54
57
|
raise ValueError(f"Unsupported click anchor: {anchor}")
|
|
55
58
|
|
|
56
59
|
if target_uses_physical_pixels(target):
|
|
57
|
-
|
|
60
|
+
origin_x, origin_y = origin
|
|
61
|
+
return (round(px / scale_factor) + origin_x, round(py / scale_factor) + origin_y)
|
|
58
62
|
return (px, py)
|
|
59
63
|
|
|
60
64
|
def click_target(
|
|
@@ -64,14 +68,18 @@ class Actuator:
|
|
|
64
68
|
anchor: ClickAnchor = "center",
|
|
65
69
|
kind: ClickKind = "click",
|
|
66
70
|
scale_factor: float = 1.0,
|
|
71
|
+
origin: tuple[int, int] = (0, 0),
|
|
67
72
|
) -> tuple[int, int]:
|
|
68
73
|
"""Click a target at a mapped point in logical click space."""
|
|
69
|
-
x, y = self.point_for_target(
|
|
74
|
+
x, y = self.point_for_target(
|
|
75
|
+
target, anchor=anchor, scale_factor=scale_factor, origin=origin
|
|
76
|
+
)
|
|
70
77
|
self.click_point(x, y, kind=kind)
|
|
71
78
|
return (x, y)
|
|
72
79
|
|
|
73
80
|
def click_point(self, x: int, y: int, *, kind: ClickKind = "click") -> None:
|
|
74
81
|
"""Click at absolute screen coordinates."""
|
|
82
|
+
raise_if_tool_cancelled()
|
|
75
83
|
if self.dry_run:
|
|
76
84
|
return
|
|
77
85
|
|
|
@@ -89,6 +97,7 @@ class Actuator:
|
|
|
89
97
|
|
|
90
98
|
def type_text(self, text: str, *, interval: float = 0.0) -> None:
|
|
91
99
|
"""Type plain text at current focus."""
|
|
100
|
+
raise_if_tool_cancelled()
|
|
92
101
|
if self.dry_run:
|
|
93
102
|
return
|
|
94
103
|
pyautogui = _load_pyautogui()
|
|
@@ -96,6 +105,7 @@ class Actuator:
|
|
|
96
105
|
|
|
97
106
|
def key_combo(self, *keys: str) -> None:
|
|
98
107
|
"""Press a key combination, e.g. ('command', 'v')."""
|
|
108
|
+
raise_if_tool_cancelled()
|
|
99
109
|
if self.dry_run:
|
|
100
110
|
return
|
|
101
111
|
if not keys:
|
|
@@ -105,6 +115,7 @@ class Actuator:
|
|
|
105
115
|
|
|
106
116
|
def press_key(self, key: str) -> None:
|
|
107
117
|
"""Press a single key."""
|
|
118
|
+
raise_if_tool_cancelled()
|
|
108
119
|
if self.dry_run:
|
|
109
120
|
return
|
|
110
121
|
if not key:
|
|
@@ -117,6 +128,7 @@ class Actuator:
|
|
|
117
128
|
|
|
118
129
|
Positive ``clicks`` scroll up; negative scroll down (pyautogui convention).
|
|
119
130
|
"""
|
|
131
|
+
raise_if_tool_cancelled()
|
|
120
132
|
if self.dry_run:
|
|
121
133
|
return
|
|
122
134
|
pyautogui = _load_pyautogui()
|
|
@@ -136,6 +148,7 @@ class Actuator:
|
|
|
136
148
|
button: Literal["left", "middle", "right"] = "left",
|
|
137
149
|
) -> None:
|
|
138
150
|
"""Drag from one screen point to another in logical coordinates."""
|
|
151
|
+
raise_if_tool_cancelled()
|
|
139
152
|
if duration < 0:
|
|
140
153
|
raise ValueError("duration must be >= 0")
|
|
141
154
|
if self.dry_run:
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Cooperative cancellation for a timed-out MCP tool call.
|
|
2
|
+
|
|
3
|
+
The MCP adapter binds one :class:`threading.Event` for the worker thread. On
|
|
4
|
+
timeout it sets the event. Actuation and the flow runner check it and refuse
|
|
5
|
+
to start another click, keystroke, scroll, drag, or flow step.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import threading
|
|
11
|
+
from contextvars import ContextVar, Token
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ToolCancelled(RuntimeError):
|
|
15
|
+
"""The current tool call was cancelled after its wall-clock timeout."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
_cancellation: ContextVar[threading.Event | None] = ContextVar(
|
|
19
|
+
"openframe_tool_cancellation", default=None
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def bind_tool_cancellation(event: threading.Event) -> Token[threading.Event | None]:
|
|
24
|
+
"""Bind ``event`` for the current thread. Reset with the returned token."""
|
|
25
|
+
return _cancellation.set(event)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def reset_tool_cancellation(token: Token[threading.Event | None]) -> None:
|
|
29
|
+
"""Restore the cancellation state captured by :func:`bind_tool_cancellation`."""
|
|
30
|
+
_cancellation.reset(token)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def tool_cancelled() -> bool:
|
|
34
|
+
"""Return True when the current tool call has been cancelled."""
|
|
35
|
+
event = _cancellation.get()
|
|
36
|
+
return event is not None and event.is_set()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def raise_if_tool_cancelled() -> None:
|
|
40
|
+
"""Raise when the current tool call has been cancelled."""
|
|
41
|
+
if tool_cancelled():
|
|
42
|
+
raise ToolCancelled("Tool call was cancelled after timeout; refusing to actuate.")
|
|
@@ -11,6 +11,7 @@ import tempfile
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
|
+
from openframe.capture.sidecar import write_frame_sidecar
|
|
14
15
|
from openframe.types import Frame
|
|
15
16
|
|
|
16
17
|
|
|
@@ -18,6 +19,9 @@ class CaptureError(RuntimeError):
|
|
|
18
19
|
"""Raised when a capture operation fails."""
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
_SUBPROCESS_TIMEOUT_SEC = 30
|
|
23
|
+
|
|
24
|
+
|
|
21
25
|
def screen(out_path: str | Path | None = None) -> Frame:
|
|
22
26
|
"""Capture the primary display to PNG and return frame metadata."""
|
|
23
27
|
_ensure_macos()
|
|
@@ -26,7 +30,10 @@ def screen(out_path: str | Path | None = None) -> Frame:
|
|
|
26
30
|
_run_command(["screencapture", "-x", "-D", "1", str(image_path)])
|
|
27
31
|
|
|
28
32
|
width, height = _read_png_dimensions(image_path)
|
|
29
|
-
|
|
33
|
+
left, top, logical_width, _logical_height = _main_display_bounds()
|
|
34
|
+
scale_factor = _scale_from_logical_width(
|
|
35
|
+
physical_width=width, logical_width=logical_width
|
|
36
|
+
)
|
|
30
37
|
metadata = {
|
|
31
38
|
"backend": "screencapture",
|
|
32
39
|
"display_index": 1,
|
|
@@ -34,14 +41,18 @@ def screen(out_path: str | Path | None = None) -> Frame:
|
|
|
34
41
|
"physical_height": height,
|
|
35
42
|
}
|
|
36
43
|
|
|
37
|
-
|
|
44
|
+
frame = Frame(
|
|
38
45
|
width=width,
|
|
39
46
|
height=height,
|
|
40
47
|
scale_factor=scale_factor,
|
|
41
48
|
source="screen:1",
|
|
42
49
|
image_path=str(image_path),
|
|
43
50
|
metadata=metadata,
|
|
51
|
+
origin_x=left,
|
|
52
|
+
origin_y=top,
|
|
44
53
|
)
|
|
54
|
+
write_frame_sidecar(frame)
|
|
55
|
+
return frame
|
|
45
56
|
|
|
46
57
|
|
|
47
58
|
def window(
|
|
@@ -54,11 +65,22 @@ def window(
|
|
|
54
65
|
_ensure_macos()
|
|
55
66
|
|
|
56
67
|
selected = _select_window_info(title=title, window_id=window_id)
|
|
68
|
+
bounds = selected.get("bounds") if isinstance(selected.get("bounds"), dict) else {}
|
|
69
|
+
logical_width = int(bounds.get("width") or 0)
|
|
70
|
+
logical_height = int(bounds.get("height") or 0)
|
|
71
|
+
if logical_width <= 0 or logical_height <= 0:
|
|
72
|
+
raise CaptureError("Window bounds are missing, so the capture scale is unknown.")
|
|
73
|
+
origin_x = int(bounds.get("x") or 0)
|
|
74
|
+
origin_y = int(bounds.get("y") or 0)
|
|
75
|
+
|
|
57
76
|
image_path = _resolve_output_path(out_path)
|
|
58
|
-
|
|
77
|
+
# -o drops the window shadow so the PNG maps onto the window bounds.
|
|
78
|
+
_run_command(["screencapture", "-x", "-o", "-l", str(selected["id"]), str(image_path)])
|
|
59
79
|
|
|
60
80
|
width, height = _read_png_dimensions(image_path)
|
|
61
|
-
scale_factor =
|
|
81
|
+
scale_factor = _scale_from_logical_width(
|
|
82
|
+
physical_width=width, logical_width=logical_width
|
|
83
|
+
)
|
|
62
84
|
metadata = {
|
|
63
85
|
"backend": "screencapture",
|
|
64
86
|
"window_id": selected["id"],
|
|
@@ -66,16 +88,22 @@ def window(
|
|
|
66
88
|
"window_owner": selected["owner"],
|
|
67
89
|
"physical_width": width,
|
|
68
90
|
"physical_height": height,
|
|
91
|
+
"logical_width": logical_width,
|
|
92
|
+
"logical_height": logical_height,
|
|
69
93
|
}
|
|
70
94
|
|
|
71
|
-
|
|
95
|
+
frame = Frame(
|
|
72
96
|
width=width,
|
|
73
97
|
height=height,
|
|
74
98
|
scale_factor=scale_factor,
|
|
75
99
|
source=f"window:{selected['id']}",
|
|
76
100
|
image_path=str(image_path),
|
|
77
101
|
metadata=metadata,
|
|
102
|
+
origin_x=origin_x,
|
|
103
|
+
origin_y=origin_y,
|
|
78
104
|
)
|
|
105
|
+
write_frame_sidecar(frame)
|
|
106
|
+
return frame
|
|
79
107
|
|
|
80
108
|
|
|
81
109
|
def region(
|
|
@@ -96,7 +124,7 @@ def region(
|
|
|
96
124
|
_run_command(["screencapture", "-x", "-R", region_spec, str(image_path)])
|
|
97
125
|
|
|
98
126
|
physical_width, physical_height = _read_png_dimensions(image_path)
|
|
99
|
-
scale_factor =
|
|
127
|
+
scale_factor = _scale_from_logical_width(physical_width=physical_width, logical_width=width)
|
|
100
128
|
metadata = {
|
|
101
129
|
"backend": "screencapture",
|
|
102
130
|
"region": {"x": x, "y": y, "width": width, "height": height},
|
|
@@ -104,14 +132,18 @@ def region(
|
|
|
104
132
|
"physical_height": physical_height,
|
|
105
133
|
}
|
|
106
134
|
|
|
107
|
-
|
|
135
|
+
frame = Frame(
|
|
108
136
|
width=physical_width,
|
|
109
137
|
height=physical_height,
|
|
110
138
|
scale_factor=scale_factor,
|
|
111
139
|
source=f"region:{x},{y},{width},{height}",
|
|
112
140
|
image_path=str(image_path),
|
|
113
141
|
metadata=metadata,
|
|
142
|
+
origin_x=x,
|
|
143
|
+
origin_y=y,
|
|
114
144
|
)
|
|
145
|
+
write_frame_sidecar(frame)
|
|
146
|
+
return frame
|
|
115
147
|
|
|
116
148
|
|
|
117
149
|
def list_windows() -> list[dict[str, Any]]:
|
|
@@ -170,7 +202,18 @@ def _resolve_output_path(out_path: str | Path | None) -> Path:
|
|
|
170
202
|
|
|
171
203
|
|
|
172
204
|
def _run_command(command: list[str]) -> subprocess.CompletedProcess[str]:
|
|
173
|
-
|
|
205
|
+
try:
|
|
206
|
+
completed = subprocess.run(
|
|
207
|
+
command,
|
|
208
|
+
check=False,
|
|
209
|
+
capture_output=True,
|
|
210
|
+
text=True,
|
|
211
|
+
timeout=_SUBPROCESS_TIMEOUT_SEC,
|
|
212
|
+
)
|
|
213
|
+
except subprocess.TimeoutExpired as exc:
|
|
214
|
+
raise CaptureError(
|
|
215
|
+
f"Capture command timed out after {_SUBPROCESS_TIMEOUT_SEC:g}s: {' '.join(command)}"
|
|
216
|
+
) from exc
|
|
174
217
|
if completed.returncode != 0:
|
|
175
218
|
stderr = completed.stderr.strip() or "unknown error"
|
|
176
219
|
raise CaptureError(f"Capture command failed: {' '.join(command)} ({stderr})")
|
|
@@ -272,7 +315,8 @@ def _read_png_dimensions(path: Path) -> tuple[int, int]:
|
|
|
272
315
|
return width, height
|
|
273
316
|
|
|
274
317
|
|
|
275
|
-
def
|
|
318
|
+
def _main_display_bounds() -> tuple[int, int, int, int]:
|
|
319
|
+
"""Return the main display as left, top, width, height in logical points."""
|
|
276
320
|
try:
|
|
277
321
|
completed = _run_command(
|
|
278
322
|
[
|
|
@@ -281,24 +325,35 @@ def _detect_scale_factor(physical_width: int) -> float:
|
|
|
281
325
|
'tell application "Finder" to get bounds of window of desktop',
|
|
282
326
|
]
|
|
283
327
|
)
|
|
284
|
-
except CaptureError:
|
|
285
|
-
|
|
328
|
+
except CaptureError as exc:
|
|
329
|
+
raise CaptureError(
|
|
330
|
+
"Cannot read the main display size, so the capture scale is unknown."
|
|
331
|
+
) from exc
|
|
286
332
|
|
|
287
333
|
values = [segment.strip() for segment in completed.stdout.strip().split(",")]
|
|
288
334
|
if len(values) != 4:
|
|
289
|
-
|
|
335
|
+
raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
|
|
290
336
|
|
|
291
337
|
try:
|
|
292
|
-
left,
|
|
293
|
-
except ValueError:
|
|
294
|
-
|
|
338
|
+
left, top, right, bottom = (int(value) for value in values)
|
|
339
|
+
except ValueError as exc:
|
|
340
|
+
raise CaptureError(
|
|
341
|
+
"Cannot read the main display size, so the capture scale is unknown."
|
|
342
|
+
) from exc
|
|
295
343
|
|
|
296
344
|
logical_width = right - left
|
|
297
|
-
|
|
298
|
-
|
|
345
|
+
logical_height = bottom - top
|
|
346
|
+
if logical_width <= 0 or logical_height <= 0:
|
|
347
|
+
raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
|
|
348
|
+
return left, top, logical_width, logical_height
|
|
349
|
+
|
|
299
350
|
|
|
351
|
+
def _scale_from_logical_width(*, physical_width: int, logical_width: int) -> float:
|
|
352
|
+
"""Return physical pixels per logical point for sizes that are already known."""
|
|
353
|
+
if physical_width <= 0 or logical_width <= 0:
|
|
354
|
+
raise CaptureError("Cannot compute scale: width must be positive.")
|
|
300
355
|
ratio = physical_width / logical_width
|
|
301
356
|
for expected in (1.0, 2.0, 3.0):
|
|
302
|
-
if abs(ratio - expected)
|
|
357
|
+
if abs(ratio - expected) <= 0.02:
|
|
303
358
|
return expected
|
|
304
359
|
return round(ratio, 2)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Metadata written beside a capture so a later click can map image pixels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from openframe.types import Frame
|
|
9
|
+
|
|
10
|
+
SIDECAR_SUFFIX = ".openframe.json"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def sidecar_path(image_path: Path) -> Path:
|
|
14
|
+
"""Return the metadata path for a capture PNG."""
|
|
15
|
+
return Path(str(image_path) + SIDECAR_SUFFIX)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def write_frame_sidecar(frame: Frame) -> None:
|
|
19
|
+
"""Write scale and origin next to ``frame.image_path``."""
|
|
20
|
+
if not frame.image_path:
|
|
21
|
+
return
|
|
22
|
+
payload = {
|
|
23
|
+
"width": frame.width,
|
|
24
|
+
"height": frame.height,
|
|
25
|
+
"scale_factor": frame.scale_factor,
|
|
26
|
+
"origin_x": frame.origin_x,
|
|
27
|
+
"origin_y": frame.origin_y,
|
|
28
|
+
"source": frame.source,
|
|
29
|
+
}
|
|
30
|
+
path = sidecar_path(Path(frame.image_path))
|
|
31
|
+
path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load_frame_sidecar(image_path: Path) -> Frame | None:
|
|
35
|
+
"""Load a frame from its sidecar, or None when the sidecar is absent."""
|
|
36
|
+
path = sidecar_path(image_path)
|
|
37
|
+
if not path.is_file():
|
|
38
|
+
return None
|
|
39
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
40
|
+
return Frame(
|
|
41
|
+
width=int(payload["width"]),
|
|
42
|
+
height=int(payload["height"]),
|
|
43
|
+
scale_factor=float(payload["scale_factor"]),
|
|
44
|
+
source=str(payload.get("source") or f"file:{image_path.name}"),
|
|
45
|
+
image_path=str(image_path),
|
|
46
|
+
origin_x=int(payload.get("origin_x", 0)),
|
|
47
|
+
origin_y=int(payload.get("origin_y", 0)),
|
|
48
|
+
metadata={"input": "file", "sidecar": str(path)},
|
|
49
|
+
)
|
|
@@ -10,6 +10,7 @@ from pathlib import Path
|
|
|
10
10
|
|
|
11
11
|
from openframe.act import ActError, Actuator
|
|
12
12
|
from openframe.capture import CaptureError, list_displays, list_windows, region, screen, window
|
|
13
|
+
from openframe.capture.sidecar import load_frame_sidecar
|
|
13
14
|
from openframe.doctor import doctor_ok, format_doctor_json, format_doctor_text, run_doctor
|
|
14
15
|
from openframe.flow import load_flow
|
|
15
16
|
from openframe.integrations.mcp import call_mcp_tool, list_mcp_tools
|
|
@@ -242,6 +243,9 @@ def main() -> int:
|
|
|
242
243
|
}
|
|
243
244
|
if overlay_path is not None:
|
|
244
245
|
payload["overlay_path"] = str(overlay_path)
|
|
246
|
+
recognizer_errors = getattr(locator, "recognizer_errors", None)
|
|
247
|
+
if recognizer_errors:
|
|
248
|
+
payload["recognizer_errors"] = list(recognizer_errors)
|
|
245
249
|
print(json.dumps(payload, indent=2))
|
|
246
250
|
return 0
|
|
247
251
|
for item in targets:
|
|
@@ -255,7 +259,9 @@ def main() -> int:
|
|
|
255
259
|
|
|
256
260
|
if args.command == "click":
|
|
257
261
|
try:
|
|
258
|
-
frame = _resolve_find_frame(
|
|
262
|
+
frame = _resolve_find_frame(
|
|
263
|
+
args.frame, require_sidecar=bool(args.frame) and not args.dry_run
|
|
264
|
+
)
|
|
259
265
|
locator = build_default_locator()
|
|
260
266
|
click_params: dict[str, object] = {}
|
|
261
267
|
if getattr(args, "template", None):
|
|
@@ -282,7 +288,11 @@ def main() -> int:
|
|
|
282
288
|
|
|
283
289
|
actuator = Actuator(dry_run=args.dry_run)
|
|
284
290
|
point = actuator.click_target(
|
|
285
|
-
selected,
|
|
291
|
+
selected,
|
|
292
|
+
anchor=args.anchor,
|
|
293
|
+
kind=args.kind,
|
|
294
|
+
scale_factor=frame.scale_factor,
|
|
295
|
+
origin=(frame.origin_x, frame.origin_y),
|
|
286
296
|
)
|
|
287
297
|
if args.dry_run and not args.verify:
|
|
288
298
|
after_frame = frame
|
|
@@ -422,11 +432,19 @@ def main() -> int:
|
|
|
422
432
|
return 2
|
|
423
433
|
|
|
424
434
|
|
|
425
|
-
def _resolve_find_frame(frame_path: Path | None) -> Frame:
|
|
435
|
+
def _resolve_find_frame(frame_path: Path | None, *, require_sidecar: bool = False) -> Frame:
|
|
426
436
|
if frame_path:
|
|
427
437
|
image_path = frame_path.expanduser().resolve()
|
|
428
438
|
if not image_path.exists():
|
|
429
439
|
raise ValueError(f"Frame path does not exist: {image_path}")
|
|
440
|
+
loaded = load_frame_sidecar(image_path)
|
|
441
|
+
if loaded is not None:
|
|
442
|
+
return loaded
|
|
443
|
+
if require_sidecar:
|
|
444
|
+
raise ValueError(
|
|
445
|
+
"frame path has no capture metadata sidecar; "
|
|
446
|
+
"refusing a live click that cannot map image coordinates."
|
|
447
|
+
)
|
|
430
448
|
return Frame(
|
|
431
449
|
width=0,
|
|
432
450
|
height=0,
|