off-camber-open-frame 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {off_camber_open_frame-0.2.1/src/off_camber_open_frame.egg-info → off_camber_open_frame-0.2.2}/PKG-INFO +2 -2
  2. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/README.md +1 -1
  3. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/pyproject.toml +1 -1
  4. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2/src/off_camber_open_frame.egg-info}/PKG-INFO +2 -2
  5. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/SOURCES.txt +5 -0
  6. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/act/core.py +19 -6
  7. off_camber_open_frame-0.2.2/src/openframe/cancel.py +42 -0
  8. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/capture/macos.py +73 -18
  9. off_camber_open_frame-0.2.2/src/openframe/capture/sidecar.py +49 -0
  10. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/cli.py +21 -3
  11. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/adapter.py +121 -33
  12. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/macos.py +27 -13
  13. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/locator.py +11 -0
  14. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/match.py +3 -1
  15. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/tesseract.py +42 -16
  16. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/runner.py +216 -55
  17. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/session.py +5 -1
  18. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/types.py +2 -0
  19. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/window.py +22 -11
  20. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_a11y_recognizer.py +61 -0
  21. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_actuator.py +49 -0
  22. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_capture.py +24 -12
  23. off_camber_open_frame-0.2.2/tests/test_capture_geometry.py +158 -0
  24. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_cli.py +4 -4
  25. off_camber_open_frame-0.2.2/tests/test_geometry_checks.py +48 -0
  26. off_camber_open_frame-0.2.2/tests/test_market_validation_tools.py +288 -0
  27. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_match.py +3 -0
  28. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_mcp_adapter.py +223 -15
  29. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_recognize_locator.py +34 -0
  30. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_runner.py +154 -4
  31. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_selector_parity.py +2 -1
  32. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_template_recognizer.py +13 -1
  33. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_tesseract_recognizer.py +87 -10
  34. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_window_state.py +57 -1
  35. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/LICENSE +0 -0
  36. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/setup.cfg +0 -0
  37. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/dependency_links.txt +0 -0
  38. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/entry_points.txt +0 -0
  39. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/requires.txt +0 -0
  40. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/off_camber_open_frame.egg-info/top_level.txt +0 -0
  41. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/__init__.py +0 -0
  42. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/act/__init__.py +0 -0
  43. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/__init__.py +0 -0
  44. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/base.py +0 -0
  45. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/cost.py +0 -0
  46. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/__init__.py +0 -0
  47. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/providers/anthropic_provider.py +0 -0
  48. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/agent/runner.py +0 -0
  49. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/capture/__init__.py +0 -0
  50. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/doctor.py +0 -0
  51. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/flow.py +0 -0
  52. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/__init__.py +0 -0
  53. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/__init__.py +0 -0
  54. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/integrations/mcp/server.py +0 -0
  55. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/__init__.py +0 -0
  56. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/a11y/__init__.py +0 -0
  57. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/base.py +0 -0
  58. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/coords.py +0 -0
  59. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/debug.py +0 -0
  60. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/defaults.py +0 -0
  61. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/ocr/__init__.py +0 -0
  62. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/template/__init__.py +0 -0
  63. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/recognize/template/matcher.py +0 -0
  64. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/verify/__init__.py +0 -0
  65. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/verify/base.py +0 -0
  66. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/src/openframe/verify/core.py +0 -0
  67. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_agent_anthropic_provider.py +0 -0
  68. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_agent_cost.py +0 -0
  69. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_agent_runner.py +0 -0
  70. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_calibration_fixture.py +0 -0
  71. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_doctor.py +0 -0
  72. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_examples.py +0 -0
  73. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_flow.py +0 -0
  74. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_imports.py +0 -0
  75. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_mcp_server.py +0 -0
  76. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_recognize_coords.py +0 -0
  77. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_recognize_debug.py +0 -0
  78. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_session_api.py +0 -0
  79. {off_camber_open_frame-0.2.1 → off_camber_open_frame-0.2.2}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: off-camber-open-frame
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Open-source recognition and action for anything on a display.
5
5
  Author: Off-Camber
6
6
  License-Expression: Apache-2.0
@@ -65,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
65
65
 
66
66
  ## Status
67
67
 
68
- **Beta** (`v0.2.1`) — macOS-first. Available on PyPI as
68
+ **Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
69
69
  [`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
70
70
  Evidence for this release comes from CI plus a local macOS live gate (including
71
71
  MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
@@ -26,7 +26,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
26
26
 
27
27
  ## Status
28
28
 
29
- **Beta** (`v0.2.1`) — macOS-first. Available on PyPI as
29
+ **Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
30
30
  [`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
31
31
  Evidence for this release comes from CI plus a local macOS live gate (including
32
32
  MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "off-camber-open-frame"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "Open-source recognition and action for anything on a display."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: off-camber-open-frame
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: Open-source recognition and action for anything on a display.
5
5
  Author: Off-Camber
6
6
  License-Expression: Apache-2.0
@@ -65,7 +65,7 @@ Open Frame is the deterministic execution layer behind AI agents, not the agent
65
65
 
66
66
  ## Status
67
67
 
68
- **Beta** (`v0.2.1`) — macOS-first. Available on PyPI as
68
+ **Beta** (`v0.2.2`) — macOS-first. Available on PyPI as
69
69
  [`off-camber-open-frame`](https://pypi.org/project/off-camber-open-frame/).
70
70
  Evidence for this release comes from CI plus a local macOS live gate (including
71
71
  MCP stdio smoke); see [release gates](docs/RELEASE_GATES.md). Changelog:
@@ -8,6 +8,7 @@ src/off_camber_open_frame.egg-info/entry_points.txt
8
8
  src/off_camber_open_frame.egg-info/requires.txt
9
9
  src/off_camber_open_frame.egg-info/top_level.txt
10
10
  src/openframe/__init__.py
11
+ src/openframe/cancel.py
11
12
  src/openframe/cli.py
12
13
  src/openframe/doctor.py
13
14
  src/openframe/flow.py
@@ -25,6 +26,7 @@ src/openframe/agent/providers/__init__.py
25
26
  src/openframe/agent/providers/anthropic_provider.py
26
27
  src/openframe/capture/__init__.py
27
28
  src/openframe/capture/macos.py
29
+ src/openframe/capture/sidecar.py
28
30
  src/openframe/integrations/__init__.py
29
31
  src/openframe/integrations/mcp/__init__.py
30
32
  src/openframe/integrations/mcp/adapter.py
@@ -52,11 +54,14 @@ tests/test_agent_cost.py
52
54
  tests/test_agent_runner.py
53
55
  tests/test_calibration_fixture.py
54
56
  tests/test_capture.py
57
+ tests/test_capture_geometry.py
55
58
  tests/test_cli.py
56
59
  tests/test_doctor.py
57
60
  tests/test_examples.py
58
61
  tests/test_flow.py
62
+ tests/test_geometry_checks.py
59
63
  tests/test_imports.py
64
+ tests/test_market_validation_tools.py
60
65
  tests/test_match.py
61
66
  tests/test_mcp_adapter.py
62
67
  tests/test_mcp_server.py
@@ -6,6 +6,7 @@ import time
6
6
  from collections.abc import Callable
7
7
  from typing import Literal
8
8
 
9
+ from openframe.cancel import raise_if_tool_cancelled
9
10
  from openframe.recognize.coords import target_uses_physical_pixels
10
11
  from openframe.types import Frame, Target
11
12
 
@@ -29,13 +30,15 @@ class Actuator:
29
30
  *,
30
31
  anchor: ClickAnchor = "center",
31
32
  scale_factor: float = 1.0,
33
+ origin: tuple[int, int] = (0, 0),
32
34
  ) -> tuple[int, int]:
33
35
  """Map target bounds to a click point.
34
36
 
35
- Recognizers report target bounds in captured-image pixels (physical
36
- pixels on Retina displays). Mouse actuation works in logical screen
37
- points. ``scale_factor`` (display physical/logical ratio, e.g. 2.0 on
38
- Retina) converts the mapped point into logical click space.
37
+ Recognizers report image matches in captured-image pixels. Mouse
38
+ actuation works in logical screen points. ``scale_factor`` converts
39
+ those pixels, then ``origin`` (the capture's logical top-left) places
40
+ the point on the screen. Accessibility targets are already logical
41
+ screen points, so ``origin`` is not added for them.
39
42
  """
40
43
  if scale_factor <= 0:
41
44
  raise ValueError("scale_factor must be greater than zero.")
@@ -54,7 +57,8 @@ class Actuator:
54
57
  raise ValueError(f"Unsupported click anchor: {anchor}")
55
58
 
56
59
  if target_uses_physical_pixels(target):
57
- return (round(px / scale_factor), round(py / scale_factor))
60
+ origin_x, origin_y = origin
61
+ return (round(px / scale_factor) + origin_x, round(py / scale_factor) + origin_y)
58
62
  return (px, py)
59
63
 
60
64
  def click_target(
@@ -64,14 +68,18 @@ class Actuator:
64
68
  anchor: ClickAnchor = "center",
65
69
  kind: ClickKind = "click",
66
70
  scale_factor: float = 1.0,
71
+ origin: tuple[int, int] = (0, 0),
67
72
  ) -> tuple[int, int]:
68
73
  """Click a target at a mapped point in logical click space."""
69
- x, y = self.point_for_target(target, anchor=anchor, scale_factor=scale_factor)
74
+ x, y = self.point_for_target(
75
+ target, anchor=anchor, scale_factor=scale_factor, origin=origin
76
+ )
70
77
  self.click_point(x, y, kind=kind)
71
78
  return (x, y)
72
79
 
73
80
  def click_point(self, x: int, y: int, *, kind: ClickKind = "click") -> None:
74
81
  """Click at absolute screen coordinates."""
82
+ raise_if_tool_cancelled()
75
83
  if self.dry_run:
76
84
  return
77
85
 
@@ -89,6 +97,7 @@ class Actuator:
89
97
 
90
98
  def type_text(self, text: str, *, interval: float = 0.0) -> None:
91
99
  """Type plain text at current focus."""
100
+ raise_if_tool_cancelled()
92
101
  if self.dry_run:
93
102
  return
94
103
  pyautogui = _load_pyautogui()
@@ -96,6 +105,7 @@ class Actuator:
96
105
 
97
106
  def key_combo(self, *keys: str) -> None:
98
107
  """Press a key combination, e.g. ('command', 'v')."""
108
+ raise_if_tool_cancelled()
99
109
  if self.dry_run:
100
110
  return
101
111
  if not keys:
@@ -105,6 +115,7 @@ class Actuator:
105
115
 
106
116
  def press_key(self, key: str) -> None:
107
117
  """Press a single key."""
118
+ raise_if_tool_cancelled()
108
119
  if self.dry_run:
109
120
  return
110
121
  if not key:
@@ -117,6 +128,7 @@ class Actuator:
117
128
 
118
129
  Positive ``clicks`` scroll up; negative scroll down (pyautogui convention).
119
130
  """
131
+ raise_if_tool_cancelled()
120
132
  if self.dry_run:
121
133
  return
122
134
  pyautogui = _load_pyautogui()
@@ -136,6 +148,7 @@ class Actuator:
136
148
  button: Literal["left", "middle", "right"] = "left",
137
149
  ) -> None:
138
150
  """Drag from one screen point to another in logical coordinates."""
151
+ raise_if_tool_cancelled()
139
152
  if duration < 0:
140
153
  raise ValueError("duration must be >= 0")
141
154
  if self.dry_run:
@@ -0,0 +1,42 @@
1
+ """Cooperative cancellation for a timed-out MCP tool call.
2
+
3
+ The MCP adapter binds one :class:`threading.Event` for the worker thread. On
4
+ timeout it sets the event. Actuation and the flow runner check it and refuse
5
+ to start another click, keystroke, scroll, drag, or flow step.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import threading
11
+ from contextvars import ContextVar, Token
12
+
13
+
14
+ class ToolCancelled(RuntimeError):
15
+ """The current tool call was cancelled after its wall-clock timeout."""
16
+
17
+
18
+ _cancellation: ContextVar[threading.Event | None] = ContextVar(
19
+ "openframe_tool_cancellation", default=None
20
+ )
21
+
22
+
23
+ def bind_tool_cancellation(event: threading.Event) -> Token[threading.Event | None]:
24
+ """Bind ``event`` for the current thread. Reset with the returned token."""
25
+ return _cancellation.set(event)
26
+
27
+
28
+ def reset_tool_cancellation(token: Token[threading.Event | None]) -> None:
29
+ """Restore the cancellation state captured by :func:`bind_tool_cancellation`."""
30
+ _cancellation.reset(token)
31
+
32
+
33
+ def tool_cancelled() -> bool:
34
+ """Return True when the current tool call has been cancelled."""
35
+ event = _cancellation.get()
36
+ return event is not None and event.is_set()
37
+
38
+
39
+ def raise_if_tool_cancelled() -> None:
40
+ """Raise when the current tool call has been cancelled."""
41
+ if tool_cancelled():
42
+ raise ToolCancelled("Tool call was cancelled after timeout; refusing to actuate.")
@@ -11,6 +11,7 @@ import tempfile
11
11
  from pathlib import Path
12
12
  from typing import Any
13
13
 
14
+ from openframe.capture.sidecar import write_frame_sidecar
14
15
  from openframe.types import Frame
15
16
 
16
17
 
@@ -18,6 +19,9 @@ class CaptureError(RuntimeError):
18
19
  """Raised when a capture operation fails."""
19
20
 
20
21
 
22
+ _SUBPROCESS_TIMEOUT_SEC = 30
23
+
24
+
21
25
  def screen(out_path: str | Path | None = None) -> Frame:
22
26
  """Capture the primary display to PNG and return frame metadata."""
23
27
  _ensure_macos()
@@ -26,7 +30,10 @@ def screen(out_path: str | Path | None = None) -> Frame:
26
30
  _run_command(["screencapture", "-x", "-D", "1", str(image_path)])
27
31
 
28
32
  width, height = _read_png_dimensions(image_path)
29
- scale_factor = _detect_scale_factor(width)
33
+ left, top, logical_width, _logical_height = _main_display_bounds()
34
+ scale_factor = _scale_from_logical_width(
35
+ physical_width=width, logical_width=logical_width
36
+ )
30
37
  metadata = {
31
38
  "backend": "screencapture",
32
39
  "display_index": 1,
@@ -34,14 +41,18 @@ def screen(out_path: str | Path | None = None) -> Frame:
34
41
  "physical_height": height,
35
42
  }
36
43
 
37
- return Frame(
44
+ frame = Frame(
38
45
  width=width,
39
46
  height=height,
40
47
  scale_factor=scale_factor,
41
48
  source="screen:1",
42
49
  image_path=str(image_path),
43
50
  metadata=metadata,
51
+ origin_x=left,
52
+ origin_y=top,
44
53
  )
54
+ write_frame_sidecar(frame)
55
+ return frame
45
56
 
46
57
 
47
58
  def window(
@@ -54,11 +65,22 @@ def window(
54
65
  _ensure_macos()
55
66
 
56
67
  selected = _select_window_info(title=title, window_id=window_id)
68
+ bounds = selected.get("bounds") if isinstance(selected.get("bounds"), dict) else {}
69
+ logical_width = int(bounds.get("width") or 0)
70
+ logical_height = int(bounds.get("height") or 0)
71
+ if logical_width <= 0 or logical_height <= 0:
72
+ raise CaptureError("Window bounds are missing, so the capture scale is unknown.")
73
+ origin_x = int(bounds.get("x") or 0)
74
+ origin_y = int(bounds.get("y") or 0)
75
+
57
76
  image_path = _resolve_output_path(out_path)
58
- _run_command(["screencapture", "-x", "-l", str(selected["id"]), str(image_path)])
77
+ # -o drops the window shadow so the PNG maps onto the window bounds.
78
+ _run_command(["screencapture", "-x", "-o", "-l", str(selected["id"]), str(image_path)])
59
79
 
60
80
  width, height = _read_png_dimensions(image_path)
61
- scale_factor = _detect_scale_factor(width)
81
+ scale_factor = _scale_from_logical_width(
82
+ physical_width=width, logical_width=logical_width
83
+ )
62
84
  metadata = {
63
85
  "backend": "screencapture",
64
86
  "window_id": selected["id"],
@@ -66,16 +88,22 @@ def window(
66
88
  "window_owner": selected["owner"],
67
89
  "physical_width": width,
68
90
  "physical_height": height,
91
+ "logical_width": logical_width,
92
+ "logical_height": logical_height,
69
93
  }
70
94
 
71
- return Frame(
95
+ frame = Frame(
72
96
  width=width,
73
97
  height=height,
74
98
  scale_factor=scale_factor,
75
99
  source=f"window:{selected['id']}",
76
100
  image_path=str(image_path),
77
101
  metadata=metadata,
102
+ origin_x=origin_x,
103
+ origin_y=origin_y,
78
104
  )
105
+ write_frame_sidecar(frame)
106
+ return frame
79
107
 
80
108
 
81
109
  def region(
@@ -96,7 +124,7 @@ def region(
96
124
  _run_command(["screencapture", "-x", "-R", region_spec, str(image_path)])
97
125
 
98
126
  physical_width, physical_height = _read_png_dimensions(image_path)
99
- scale_factor = _detect_scale_factor(physical_width)
127
+ scale_factor = _scale_from_logical_width(physical_width=physical_width, logical_width=width)
100
128
  metadata = {
101
129
  "backend": "screencapture",
102
130
  "region": {"x": x, "y": y, "width": width, "height": height},
@@ -104,14 +132,18 @@ def region(
104
132
  "physical_height": physical_height,
105
133
  }
106
134
 
107
- return Frame(
135
+ frame = Frame(
108
136
  width=physical_width,
109
137
  height=physical_height,
110
138
  scale_factor=scale_factor,
111
139
  source=f"region:{x},{y},{width},{height}",
112
140
  image_path=str(image_path),
113
141
  metadata=metadata,
142
+ origin_x=x,
143
+ origin_y=y,
114
144
  )
145
+ write_frame_sidecar(frame)
146
+ return frame
115
147
 
116
148
 
117
149
  def list_windows() -> list[dict[str, Any]]:
@@ -170,7 +202,18 @@ def _resolve_output_path(out_path: str | Path | None) -> Path:
170
202
 
171
203
 
172
204
  def _run_command(command: list[str]) -> subprocess.CompletedProcess[str]:
173
- completed = subprocess.run(command, check=False, capture_output=True, text=True)
205
+ try:
206
+ completed = subprocess.run(
207
+ command,
208
+ check=False,
209
+ capture_output=True,
210
+ text=True,
211
+ timeout=_SUBPROCESS_TIMEOUT_SEC,
212
+ )
213
+ except subprocess.TimeoutExpired as exc:
214
+ raise CaptureError(
215
+ f"Capture command timed out after {_SUBPROCESS_TIMEOUT_SEC:g}s: {' '.join(command)}"
216
+ ) from exc
174
217
  if completed.returncode != 0:
175
218
  stderr = completed.stderr.strip() or "unknown error"
176
219
  raise CaptureError(f"Capture command failed: {' '.join(command)} ({stderr})")
@@ -272,7 +315,8 @@ def _read_png_dimensions(path: Path) -> tuple[int, int]:
272
315
  return width, height
273
316
 
274
317
 
275
- def _detect_scale_factor(physical_width: int) -> float:
318
+ def _main_display_bounds() -> tuple[int, int, int, int]:
319
+ """Return the main display as left, top, width, height in logical points."""
276
320
  try:
277
321
  completed = _run_command(
278
322
  [
@@ -281,24 +325,35 @@ def _detect_scale_factor(physical_width: int) -> float:
281
325
  'tell application "Finder" to get bounds of window of desktop',
282
326
  ]
283
327
  )
284
- except CaptureError:
285
- return 1.0
328
+ except CaptureError as exc:
329
+ raise CaptureError(
330
+ "Cannot read the main display size, so the capture scale is unknown."
331
+ ) from exc
286
332
 
287
333
  values = [segment.strip() for segment in completed.stdout.strip().split(",")]
288
334
  if len(values) != 4:
289
- return 1.0
335
+ raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
290
336
 
291
337
  try:
292
- left, _top, right, _bottom = (int(value) for value in values)
293
- except ValueError:
294
- return 1.0
338
+ left, top, right, bottom = (int(value) for value in values)
339
+ except ValueError as exc:
340
+ raise CaptureError(
341
+ "Cannot read the main display size, so the capture scale is unknown."
342
+ ) from exc
295
343
 
296
344
  logical_width = right - left
297
- if logical_width <= 0:
298
- return 1.0
345
+ logical_height = bottom - top
346
+ if logical_width <= 0 or logical_height <= 0:
347
+ raise CaptureError("Cannot read the main display size, so the capture scale is unknown.")
348
+ return left, top, logical_width, logical_height
349
+
299
350
 
351
+ def _scale_from_logical_width(*, physical_width: int, logical_width: int) -> float:
352
+ """Return physical pixels per logical point for sizes that are already known."""
353
+ if physical_width <= 0 or logical_width <= 0:
354
+ raise CaptureError("Cannot compute scale: width must be positive.")
300
355
  ratio = physical_width / logical_width
301
356
  for expected in (1.0, 2.0, 3.0):
302
- if abs(ratio - expected) < 0.15:
357
+ if abs(ratio - expected) <= 0.02:
303
358
  return expected
304
359
  return round(ratio, 2)
@@ -0,0 +1,49 @@
1
+ """Metadata written beside a capture so a later click can map image pixels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ from openframe.types import Frame
9
+
10
+ SIDECAR_SUFFIX = ".openframe.json"
11
+
12
+
13
+ def sidecar_path(image_path: Path) -> Path:
14
+ """Return the metadata path for a capture PNG."""
15
+ return Path(str(image_path) + SIDECAR_SUFFIX)
16
+
17
+
18
+ def write_frame_sidecar(frame: Frame) -> None:
19
+ """Write scale and origin next to ``frame.image_path``."""
20
+ if not frame.image_path:
21
+ return
22
+ payload = {
23
+ "width": frame.width,
24
+ "height": frame.height,
25
+ "scale_factor": frame.scale_factor,
26
+ "origin_x": frame.origin_x,
27
+ "origin_y": frame.origin_y,
28
+ "source": frame.source,
29
+ }
30
+ path = sidecar_path(Path(frame.image_path))
31
+ path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
32
+
33
+
34
+ def load_frame_sidecar(image_path: Path) -> Frame | None:
35
+ """Load a frame from its sidecar, or None when the sidecar is absent."""
36
+ path = sidecar_path(image_path)
37
+ if not path.is_file():
38
+ return None
39
+ payload = json.loads(path.read_text(encoding="utf-8"))
40
+ return Frame(
41
+ width=int(payload["width"]),
42
+ height=int(payload["height"]),
43
+ scale_factor=float(payload["scale_factor"]),
44
+ source=str(payload.get("source") or f"file:{image_path.name}"),
45
+ image_path=str(image_path),
46
+ origin_x=int(payload.get("origin_x", 0)),
47
+ origin_y=int(payload.get("origin_y", 0)),
48
+ metadata={"input": "file", "sidecar": str(path)},
49
+ )
@@ -10,6 +10,7 @@ from pathlib import Path
10
10
 
11
11
  from openframe.act import ActError, Actuator
12
12
  from openframe.capture import CaptureError, list_displays, list_windows, region, screen, window
13
+ from openframe.capture.sidecar import load_frame_sidecar
13
14
  from openframe.doctor import doctor_ok, format_doctor_json, format_doctor_text, run_doctor
14
15
  from openframe.flow import load_flow
15
16
  from openframe.integrations.mcp import call_mcp_tool, list_mcp_tools
@@ -242,6 +243,9 @@ def main() -> int:
242
243
  }
243
244
  if overlay_path is not None:
244
245
  payload["overlay_path"] = str(overlay_path)
246
+ recognizer_errors = getattr(locator, "recognizer_errors", None)
247
+ if recognizer_errors:
248
+ payload["recognizer_errors"] = list(recognizer_errors)
245
249
  print(json.dumps(payload, indent=2))
246
250
  return 0
247
251
  for item in targets:
@@ -255,7 +259,9 @@ def main() -> int:
255
259
 
256
260
  if args.command == "click":
257
261
  try:
258
- frame = _resolve_find_frame(args.frame)
262
+ frame = _resolve_find_frame(
263
+ args.frame, require_sidecar=bool(args.frame) and not args.dry_run
264
+ )
259
265
  locator = build_default_locator()
260
266
  click_params: dict[str, object] = {}
261
267
  if getattr(args, "template", None):
@@ -282,7 +288,11 @@ def main() -> int:
282
288
 
283
289
  actuator = Actuator(dry_run=args.dry_run)
284
290
  point = actuator.click_target(
285
- selected, anchor=args.anchor, kind=args.kind, scale_factor=frame.scale_factor
291
+ selected,
292
+ anchor=args.anchor,
293
+ kind=args.kind,
294
+ scale_factor=frame.scale_factor,
295
+ origin=(frame.origin_x, frame.origin_y),
286
296
  )
287
297
  if args.dry_run and not args.verify:
288
298
  after_frame = frame
@@ -422,11 +432,19 @@ def main() -> int:
422
432
  return 2
423
433
 
424
434
 
425
- def _resolve_find_frame(frame_path: Path | None) -> Frame:
435
+ def _resolve_find_frame(frame_path: Path | None, *, require_sidecar: bool = False) -> Frame:
426
436
  if frame_path:
427
437
  image_path = frame_path.expanduser().resolve()
428
438
  if not image_path.exists():
429
439
  raise ValueError(f"Frame path does not exist: {image_path}")
440
+ loaded = load_frame_sidecar(image_path)
441
+ if loaded is not None:
442
+ return loaded
443
+ if require_sidecar:
444
+ raise ValueError(
445
+ "frame path has no capture metadata sidecar; "
446
+ "refusing a live click that cannot map image coordinates."
447
+ )
430
448
  return Frame(
431
449
  width=0,
432
450
  height=0,