openadapt-types 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/.github/workflows/publish.yml +4 -18
- openadapt_types-0.3.1/.github/workflows/test.yml +21 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/CHANGELOG.md +31 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/PKG-INFO +1 -1
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/__init__.py +11 -0
- openadapt_types-0.3.1/openadapt_types/benchmark.py +202 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/pyproject.toml +9 -1
- openadapt_types-0.3.1/scripts/verify_release_lock.py +124 -0
- openadapt_types-0.3.1/tests/test_benchmark.py +86 -0
- openadapt_types-0.3.1/tests/test_release_contract.py +91 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/uv.lock +1 -1
- openadapt_types-0.2.0/.github/workflows/test.yml +0 -16
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/.gitignore +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/LICENSE +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/README.md +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/_compat.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/action.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/computer_state.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/episode.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/failure.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/parsing.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/openadapt_types/py.typed +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/tests/__init__.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/tests/test_compat.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/tests/test_parsing.py +0 -0
- {openadapt_types-0.2.0 → openadapt_types-0.3.1}/tests/test_schemas.py +0 -0
|
@@ -16,7 +16,7 @@ jobs:
|
|
|
16
16
|
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout repository
|
|
19
|
-
uses: actions/checkout@v4
|
|
19
|
+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
20
20
|
with:
|
|
21
21
|
fetch-depth: 0
|
|
22
22
|
token: ${{ secrets.ADMIN_TOKEN }}
|
|
@@ -28,33 +28,19 @@ jobs:
|
|
|
28
28
|
echo "skip=true" >> $GITHUB_OUTPUT
|
|
29
29
|
fi
|
|
30
30
|
|
|
31
|
-
- name: Set up Python
|
|
32
|
-
if: steps.check_skip.outputs.skip != 'true'
|
|
33
|
-
uses: actions/setup-python@v5
|
|
34
|
-
with:
|
|
35
|
-
python-version: '3.12'
|
|
36
|
-
|
|
37
|
-
- name: Install uv
|
|
38
|
-
if: steps.check_skip.outputs.skip != 'true'
|
|
39
|
-
uses: astral-sh/setup-uv@v4
|
|
40
|
-
|
|
41
31
|
- name: Python Semantic Release
|
|
42
32
|
if: steps.check_skip.outputs.skip != 'true'
|
|
43
33
|
id: release
|
|
44
|
-
uses: python-semantic-release/python-semantic-release@v9.15.2
|
|
34
|
+
uses: python-semantic-release/python-semantic-release@7b3f71697ccfbaef884e1e754b6364e974b134cf # v9.15.2
|
|
45
35
|
with:
|
|
46
36
|
github_token: ${{ secrets.ADMIN_TOKEN }}
|
|
47
37
|
|
|
48
|
-
- name: Build package
|
|
49
|
-
if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true'
|
|
50
|
-
run: uv build
|
|
51
|
-
|
|
52
38
|
- name: Publish to PyPI
|
|
53
39
|
if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true'
|
|
54
|
-
uses: pypa/gh-action-pypi-publish@release/v1
|
|
40
|
+
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
|
|
55
41
|
|
|
56
42
|
- name: Publish to GitHub Releases
|
|
57
43
|
if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true'
|
|
58
|
-
uses: python-semantic-release/publish-action@v9.15.2
|
|
44
|
+
uses: python-semantic-release/publish-action@b9c41d4b0754dee5a6c7188d42b33f66e3a8aafd # v9.15.2
|
|
59
45
|
with:
|
|
60
46
|
github_token: ${{ secrets.ADMIN_TOKEN }}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
14
|
+
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.12"
|
|
17
|
+
- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
|
18
|
+
with:
|
|
19
|
+
version: "0.11.29"
|
|
20
|
+
- run: uv sync --locked --extra dev
|
|
21
|
+
- run: uv run pytest
|
|
@@ -1,6 +1,37 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
## v0.3.1 (2026-07-16)
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
- Keep release lock metadata consistent
|
|
9
|
+
([`e3c78f4`](https://github.com/OpenAdaptAI/openadapt-types/commit/e3c78f458e52aec69377c48e4ece7b77e8c88968))
|
|
10
|
+
|
|
11
|
+
Pin release tooling, verify editable lock metadata, and build the reviewed lock state during
|
|
12
|
+
semantic release.
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
## v0.3.0 (2026-07-13)
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
- Add canonical Benchmark* types (Task/Observation/Action/Agent)
|
|
20
|
+
([#5](https://github.com/OpenAdaptAI/openadapt-types/pull/5),
|
|
21
|
+
[`82ad485`](https://github.com/OpenAdaptAI/openadapt-types/commit/82ad48590e89e23c44cdf25ce636035195d6ea14))
|
|
22
|
+
|
|
23
|
+
Move the Benchmark* vocabulary into the canonical schema package so both openadapt-ml and
|
|
24
|
+
openadapt-evals can import it without depending on each other, breaking the historical ml<->evals
|
|
25
|
+
import cycle.
|
|
26
|
+
|
|
27
|
+
Definitions are dependency-free (dataclasses + abc) and faithfully match the previous
|
|
28
|
+
openadapt-evals definitions.
|
|
29
|
+
|
|
30
|
+
Claude-Session: https://claude.ai/code/session_01CKrVJJy5jWVCkXAqgUqtqZ
|
|
31
|
+
|
|
32
|
+
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
|
|
33
|
+
|
|
34
|
+
|
|
4
35
|
## v0.2.0 (2026-03-29)
|
|
5
36
|
|
|
6
37
|
### Continuous Integration
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openadapt-types
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Canonical Pydantic schemas for computer-use agents: ComputerState, Action, ActionResult, UINode
|
|
5
5
|
Project-URL: Homepage, https://github.com/OpenAdaptAI/openadapt-types
|
|
6
6
|
Project-URL: Repository, https://github.com/OpenAdaptAI/openadapt-types
|
|
@@ -26,6 +26,12 @@ from openadapt_types.action import (
|
|
|
26
26
|
ActionTarget,
|
|
27
27
|
ActionType,
|
|
28
28
|
)
|
|
29
|
+
from openadapt_types.benchmark import (
|
|
30
|
+
BenchmarkAction,
|
|
31
|
+
BenchmarkAgent,
|
|
32
|
+
BenchmarkObservation,
|
|
33
|
+
BenchmarkTask,
|
|
34
|
+
)
|
|
29
35
|
from openadapt_types.computer_state import (
|
|
30
36
|
BoundingBox,
|
|
31
37
|
ComputerState,
|
|
@@ -57,6 +63,11 @@ __all__ = [
|
|
|
57
63
|
"ActionResult",
|
|
58
64
|
"ActionTarget",
|
|
59
65
|
"ActionType",
|
|
66
|
+
# benchmark
|
|
67
|
+
"BenchmarkAction",
|
|
68
|
+
"BenchmarkAgent",
|
|
69
|
+
"BenchmarkObservation",
|
|
70
|
+
"BenchmarkTask",
|
|
60
71
|
# episode
|
|
61
72
|
"Episode",
|
|
62
73
|
"Step",
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Canonical benchmark vocabulary shared across the OpenAdapt ecosystem.
|
|
2
|
+
|
|
3
|
+
This module hosts the ``Benchmark*`` types that describe the interaction
|
|
4
|
+
surface between GUI-agent benchmarks and the agents/environments that run
|
|
5
|
+
them. They live here (in the canonical schema package) so that both
|
|
6
|
+
``openadapt-ml`` (model library) and ``openadapt-evals`` (evaluation
|
|
7
|
+
harness) can import them without depending on each other -- breaking the
|
|
8
|
+
historical ``ml <-> evals`` import cycle.
|
|
9
|
+
|
|
10
|
+
The definitions are deliberately dependency-free (plain ``dataclasses`` and
|
|
11
|
+
an ``abc.ABC``) so importing them never pulls in heavy optional deps.
|
|
12
|
+
|
|
13
|
+
Example::
|
|
14
|
+
|
|
15
|
+
from openadapt_types import (
|
|
16
|
+
BenchmarkAgent,
|
|
17
|
+
BenchmarkAction,
|
|
18
|
+
BenchmarkObservation,
|
|
19
|
+
BenchmarkTask,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
class MyAgent(BenchmarkAgent):
|
|
23
|
+
def act(self, observation, task, history=None):
|
|
24
|
+
return BenchmarkAction(type="click", x=0.5, y=0.5)
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from abc import ABC, abstractmethod
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class BenchmarkTask:
|
|
36
|
+
"""Canonical task representation.
|
|
37
|
+
|
|
38
|
+
Attributes:
|
|
39
|
+
task_id: Unique identifier for the task.
|
|
40
|
+
instruction: Natural language task instruction.
|
|
41
|
+
domain: Task domain ("web", "desktop", "mobile").
|
|
42
|
+
initial_state_ref: Reference to initial state (VM snapshot, URL, etc.).
|
|
43
|
+
time_limit_steps: Maximum steps allowed for the task.
|
|
44
|
+
raw_config: Original benchmark config (lossless preservation).
|
|
45
|
+
evaluation_spec: Benchmark-native evaluation specification.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
task_id: str
|
|
49
|
+
instruction: str
|
|
50
|
+
domain: str # "web", "desktop", "mobile"
|
|
51
|
+
|
|
52
|
+
# Environment setup
|
|
53
|
+
initial_state_ref: str | None = None # VM snapshot, storage_state, start URL
|
|
54
|
+
time_limit_steps: int | None = None
|
|
55
|
+
|
|
56
|
+
# Preserve original config losslessly
|
|
57
|
+
raw_config: dict[str, Any] = field(default_factory=dict)
|
|
58
|
+
|
|
59
|
+
# Evaluation spec (benchmark-native)
|
|
60
|
+
evaluation_spec: dict[str, Any] | None = None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class BenchmarkObservation:
|
|
65
|
+
"""Canonical observation at each step.
|
|
66
|
+
|
|
67
|
+
Supports multiple observation modalities:
|
|
68
|
+
- Visual: screenshots with viewport info
|
|
69
|
+
- Structured UI: accessibility tree (UIA/AXTree/DOM)
|
|
70
|
+
- Context: URL, window title, focused element
|
|
71
|
+
|
|
72
|
+
Attributes:
|
|
73
|
+
screenshot: PNG image bytes.
|
|
74
|
+
screenshot_path: Path to saved screenshot.
|
|
75
|
+
viewport: (width, height) of the viewport.
|
|
76
|
+
accessibility_tree: Platform-specific UI tree (UIA/AXTree/DOM).
|
|
77
|
+
dom_html: Raw HTML for web tasks.
|
|
78
|
+
url: Current URL for web tasks.
|
|
79
|
+
window_title: Active window title for desktop tasks.
|
|
80
|
+
focused_element: Currently focused UI element.
|
|
81
|
+
raw_observation: Original benchmark observation (lossless).
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
# Visual
|
|
85
|
+
screenshot: bytes | None = None # PNG image bytes
|
|
86
|
+
screenshot_path: str | None = None
|
|
87
|
+
viewport: tuple[int, int] | None = None # (width, height)
|
|
88
|
+
|
|
89
|
+
# Structured UI (format varies by platform)
|
|
90
|
+
accessibility_tree: dict | None = None # UIA (Windows), AXTree (macOS), DOM (web)
|
|
91
|
+
dom_html: str | None = None # Raw HTML for web
|
|
92
|
+
|
|
93
|
+
# Context
|
|
94
|
+
url: str | None = None # For web tasks
|
|
95
|
+
window_title: str | None = None # For desktop tasks
|
|
96
|
+
app_name: str | None = None # Active application
|
|
97
|
+
focused_element: dict | None = None # {node_id, bbox, text}
|
|
98
|
+
|
|
99
|
+
# Raw benchmark-specific data (lossless)
|
|
100
|
+
raw_observation: dict[str, Any] | None = None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass
|
|
104
|
+
class BenchmarkAction:
|
|
105
|
+
"""Canonical action representation.
|
|
106
|
+
|
|
107
|
+
Supports multiple action types with both coordinate-based and element-based
|
|
108
|
+
grounding. The "grounding-first" approach stores both when available.
|
|
109
|
+
|
|
110
|
+
Attributes:
|
|
111
|
+
type: Action type ("click", "type", "scroll", "key", "drag", "answer", "done").
|
|
112
|
+
x: X coordinate (normalized [0,1] or pixels).
|
|
113
|
+
y: Y coordinate (normalized [0,1] or pixels).
|
|
114
|
+
target_node_id: Element ID from accessibility tree.
|
|
115
|
+
target_bbox: Element bounding box.
|
|
116
|
+
target_role: Element role (button, textfield, etc.).
|
|
117
|
+
target_name: Element accessible name.
|
|
118
|
+
text: Text to type (for "type" action).
|
|
119
|
+
key: Single key (for "key" action, e.g., "Enter", "Tab").
|
|
120
|
+
modifiers: Key modifiers (["ctrl", "shift", "alt"]).
|
|
121
|
+
scroll_direction: Scroll direction ("up", "down", "left", "right").
|
|
122
|
+
scroll_amount: Scroll amount (pixels or normalized).
|
|
123
|
+
end_x: Drag end X coordinate.
|
|
124
|
+
end_y: Drag end Y coordinate.
|
|
125
|
+
answer: Answer string (for benchmarks that score by answer).
|
|
126
|
+
raw_action: Original benchmark action (lossless).
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
type: str # "click", "type", "scroll", "key", "drag", "answer", "done", "error"
|
|
130
|
+
|
|
131
|
+
# Pointer actions - coordinates
|
|
132
|
+
x: float | None = None # Normalized [0,1] or pixel
|
|
133
|
+
y: float | None = None
|
|
134
|
+
|
|
135
|
+
# Element grounding (when available)
|
|
136
|
+
target_node_id: str | None = None # DOM/AX/UIA node ID
|
|
137
|
+
target_bbox: tuple[float, float, float, float] | None = None
|
|
138
|
+
target_role: str | None = None # "button", "textfield", etc.
|
|
139
|
+
target_name: str | None = None # Accessible name
|
|
140
|
+
|
|
141
|
+
# Keyboard actions
|
|
142
|
+
text: str | None = None # For "type" action - text to type
|
|
143
|
+
key: str | None = None # For "key" action - single key
|
|
144
|
+
modifiers: list[str] | None = None # ["ctrl", "shift", "alt"]
|
|
145
|
+
|
|
146
|
+
# Scroll actions
|
|
147
|
+
scroll_direction: str | None = None # "up", "down", "left", "right"
|
|
148
|
+
scroll_amount: float | None = None # Pixels or normalized
|
|
149
|
+
|
|
150
|
+
# Drag actions
|
|
151
|
+
end_x: float | None = None
|
|
152
|
+
end_y: float | None = None
|
|
153
|
+
|
|
154
|
+
# Answer action (some benchmarks score by final answer)
|
|
155
|
+
answer: str | None = None
|
|
156
|
+
|
|
157
|
+
# Raw benchmark-specific format (lossless)
|
|
158
|
+
raw_action: dict[str, Any] | None = None
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class BenchmarkAgent(ABC):
|
|
162
|
+
"""Abstract interface for agents evaluated on benchmarks.
|
|
163
|
+
|
|
164
|
+
Agents must implement the `act` method to receive observations
|
|
165
|
+
and return actions. The agent can maintain internal state across
|
|
166
|
+
steps within an episode.
|
|
167
|
+
"""
|
|
168
|
+
|
|
169
|
+
@abstractmethod
|
|
170
|
+
def act(
|
|
171
|
+
self,
|
|
172
|
+
observation: BenchmarkObservation,
|
|
173
|
+
task: BenchmarkTask,
|
|
174
|
+
history: list[tuple[BenchmarkObservation, BenchmarkAction]] | None = None,
|
|
175
|
+
) -> BenchmarkAction:
|
|
176
|
+
"""Given observation and task, return next action.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
observation: Current observation from the environment.
|
|
180
|
+
task: Task being performed.
|
|
181
|
+
history: Optional list of previous (observation, action) pairs.
|
|
182
|
+
|
|
183
|
+
Returns:
|
|
184
|
+
Action to execute.
|
|
185
|
+
"""
|
|
186
|
+
...
|
|
187
|
+
|
|
188
|
+
def reset(self) -> None:
|
|
189
|
+
"""Reset agent state between episodes.
|
|
190
|
+
|
|
191
|
+
Called before starting a new task. Override to clear any
|
|
192
|
+
internal state.
|
|
193
|
+
"""
|
|
194
|
+
pass
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
__all__ = [
|
|
198
|
+
"BenchmarkAction",
|
|
199
|
+
"BenchmarkAgent",
|
|
200
|
+
"BenchmarkObservation",
|
|
201
|
+
"BenchmarkTask",
|
|
202
|
+
]
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "openadapt-types"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Canonical Pydantic schemas for computer-use agents: ComputerState, Action, ActionResult, UINode"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -40,6 +40,8 @@ dev = [
|
|
|
40
40
|
"pytest-cov",
|
|
41
41
|
]
|
|
42
42
|
|
|
43
|
+
|
|
44
|
+
|
|
43
45
|
[tool.hatch.build.targets.wheel]
|
|
44
46
|
packages = ["openadapt_types"]
|
|
45
47
|
|
|
@@ -50,6 +52,12 @@ testpaths = ["tests"]
|
|
|
50
52
|
version_toml = ["pyproject.toml:project.version"]
|
|
51
53
|
commit_message = "chore: release {version}"
|
|
52
54
|
major_on_zero = false
|
|
55
|
+
allow_zero_version = true
|
|
56
|
+
# Semantic Release changes only project metadata. Stamp that version into the
|
|
57
|
+
# existing editable lock entry without resolving dependencies, stage it in the
|
|
58
|
+
# generated release commit, and build exactly those reviewed inputs. The PSR
|
|
59
|
+
# action is a Docker action, so uv must be installed inside its container.
|
|
60
|
+
build_command = "python -m pip install uv==0.11.29 && python scripts/verify_release_lock.py --write && git add uv.lock && uv build"
|
|
53
61
|
|
|
54
62
|
[tool.semantic_release.branches.main]
|
|
55
63
|
match = "main"
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Verify that project metadata and the editable uv lock entry agree."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _quoted_table_value(text: str, table: str, key: str) -> str:
|
|
13
|
+
table_match = re.search(
|
|
14
|
+
rf"(?ms)^\[{re.escape(table)}\]\s*$\n(?P<body>.*?)(?=^\[|\Z)",
|
|
15
|
+
text,
|
|
16
|
+
)
|
|
17
|
+
if table_match is None:
|
|
18
|
+
raise ValueError(f"missing [{table}] table")
|
|
19
|
+
value_match = re.search(
|
|
20
|
+
rf'(?m)^{re.escape(key)}\s*=\s*"(?P<value>[^"]+)"\s*$',
|
|
21
|
+
table_match.group("body"),
|
|
22
|
+
)
|
|
23
|
+
if value_match is None:
|
|
24
|
+
raise ValueError(f"missing quoted [{table}].{key}")
|
|
25
|
+
return value_match.group("value")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _editable_lock_entry(lock_text: str, package_name: str) -> tuple[int, int, str]:
|
|
29
|
+
matches: list[tuple[int, int, str]] = []
|
|
30
|
+
for block_match in re.finditer(
|
|
31
|
+
r"(?ms)^\[\[package\]\]\s*$\n.*?(?=^\[\[package\]\]\s*$|\Z)",
|
|
32
|
+
lock_text,
|
|
33
|
+
):
|
|
34
|
+
block = block_match.group(0)
|
|
35
|
+
name_match = re.search(r'(?m)^name\s*=\s*"(?P<value>[^"]+)"\s*$', block)
|
|
36
|
+
if name_match is None or name_match.group("value") != package_name:
|
|
37
|
+
continue
|
|
38
|
+
if not re.search(
|
|
39
|
+
r'(?m)^source\s*=\s*\{\s*editable\s*=\s*"\."\s*\}\s*$',
|
|
40
|
+
block,
|
|
41
|
+
):
|
|
42
|
+
continue
|
|
43
|
+
version_match = re.search(
|
|
44
|
+
r'(?m)^version\s*=\s*"(?P<value>[^"]+)"\s*$', block
|
|
45
|
+
)
|
|
46
|
+
if version_match is None:
|
|
47
|
+
raise ValueError(f"editable {package_name!r} lock entry has no version")
|
|
48
|
+
matches.append(
|
|
49
|
+
(
|
|
50
|
+
block_match.start() + version_match.start("value"),
|
|
51
|
+
block_match.start() + version_match.end("value"),
|
|
52
|
+
version_match.group("value"),
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
if len(matches) != 1:
|
|
56
|
+
raise ValueError(
|
|
57
|
+
f"expected exactly one editable {package_name!r} lock entry; "
|
|
58
|
+
f"found {len(matches)}"
|
|
59
|
+
)
|
|
60
|
+
return matches[0]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _project_identity(root: Path) -> tuple[str, str]:
|
|
64
|
+
project_text = (root / "pyproject.toml").read_text(encoding="utf-8")
|
|
65
|
+
return (
|
|
66
|
+
_quoted_table_value(project_text, "project", "name"),
|
|
67
|
+
_quoted_table_value(project_text, "project", "version"),
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def release_versions(root: Path = ROOT) -> tuple[str, str]:
|
|
72
|
+
"""Return ``(project_version, editable_lock_version)``."""
|
|
73
|
+
package_name, project_version = _project_identity(root)
|
|
74
|
+
lock_text = (root / "uv.lock").read_text(encoding="utf-8")
|
|
75
|
+
_, _, lock_version = _editable_lock_entry(lock_text, package_name)
|
|
76
|
+
return project_version, lock_version
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def synchronize_release_lock(root: Path = ROOT) -> bool:
|
|
80
|
+
"""Stamp only the editable root version, preserving reviewed resolution."""
|
|
81
|
+
package_name, project_version = _project_identity(root)
|
|
82
|
+
lock_path = root / "uv.lock"
|
|
83
|
+
lock_text = lock_path.read_text(encoding="utf-8")
|
|
84
|
+
version_start, version_end, lock_version = _editable_lock_entry(
|
|
85
|
+
lock_text, package_name
|
|
86
|
+
)
|
|
87
|
+
if lock_version == project_version:
|
|
88
|
+
return False
|
|
89
|
+
lock_path.write_text(
|
|
90
|
+
lock_text[:version_start] + project_version + lock_text[version_end:],
|
|
91
|
+
encoding="utf-8",
|
|
92
|
+
)
|
|
93
|
+
return True
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def verify_release_lock(root: Path = ROOT) -> None:
|
|
97
|
+
project_version, lock_version = release_versions(root)
|
|
98
|
+
if project_version != lock_version:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
"release version drift: "
|
|
101
|
+
f"pyproject.toml={project_version}, uv.lock={lock_version}; "
|
|
102
|
+
"run `python scripts/verify_release_lock.py --write`"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def main() -> int:
|
|
107
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
108
|
+
parser.add_argument(
|
|
109
|
+
"--write",
|
|
110
|
+
action="store_true",
|
|
111
|
+
help="stamp the project version into the editable lock entry before checking",
|
|
112
|
+
)
|
|
113
|
+
args = parser.parse_args()
|
|
114
|
+
try:
|
|
115
|
+
if args.write:
|
|
116
|
+
synchronize_release_lock()
|
|
117
|
+
verify_release_lock()
|
|
118
|
+
except ValueError as exc:
|
|
119
|
+
parser.exit(1, f"{exc}\n")
|
|
120
|
+
return 0
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
if __name__ == "__main__":
|
|
124
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Tests for the canonical Benchmark* vocabulary."""
|
|
2
|
+
|
|
3
|
+
from openadapt_types import (
|
|
4
|
+
BenchmarkAction,
|
|
5
|
+
BenchmarkAgent,
|
|
6
|
+
BenchmarkObservation,
|
|
7
|
+
BenchmarkTask,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_benchmark_task_defaults():
|
|
12
|
+
task = BenchmarkTask(task_id="t1", instruction="do it", domain="desktop")
|
|
13
|
+
assert task.task_id == "t1"
|
|
14
|
+
assert task.instruction == "do it"
|
|
15
|
+
assert task.domain == "desktop"
|
|
16
|
+
assert task.initial_state_ref is None
|
|
17
|
+
assert task.time_limit_steps is None
|
|
18
|
+
assert task.raw_config == {}
|
|
19
|
+
assert task.evaluation_spec is None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_benchmark_task_raw_config_is_independent():
|
|
23
|
+
a = BenchmarkTask(task_id="a", instruction="i", domain="web")
|
|
24
|
+
b = BenchmarkTask(task_id="b", instruction="i", domain="web")
|
|
25
|
+
a.raw_config["k"] = "v"
|
|
26
|
+
assert b.raw_config == {}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_benchmark_observation_defaults():
|
|
30
|
+
obs = BenchmarkObservation()
|
|
31
|
+
assert obs.screenshot is None
|
|
32
|
+
assert obs.viewport is None
|
|
33
|
+
assert obs.raw_observation is None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_benchmark_observation_fields():
|
|
37
|
+
obs = BenchmarkObservation(
|
|
38
|
+
screenshot=b"png",
|
|
39
|
+
viewport=(1920, 1080),
|
|
40
|
+
url="https://example.com",
|
|
41
|
+
window_title="Notepad",
|
|
42
|
+
)
|
|
43
|
+
assert obs.screenshot == b"png"
|
|
44
|
+
assert obs.viewport == (1920, 1080)
|
|
45
|
+
assert obs.url == "https://example.com"
|
|
46
|
+
assert obs.window_title == "Notepad"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_benchmark_action_click():
|
|
50
|
+
action = BenchmarkAction(type="click", x=0.5, y=0.25)
|
|
51
|
+
assert action.type == "click"
|
|
52
|
+
assert action.x == 0.5
|
|
53
|
+
assert action.y == 0.25
|
|
54
|
+
assert action.text is None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_benchmark_action_type_and_key():
|
|
58
|
+
typing = BenchmarkAction(type="type", text="hello")
|
|
59
|
+
assert typing.text == "hello"
|
|
60
|
+
key = BenchmarkAction(type="key", key="Enter", modifiers=["ctrl"])
|
|
61
|
+
assert key.key == "Enter"
|
|
62
|
+
assert key.modifiers == ["ctrl"]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_benchmark_agent_is_abstract():
|
|
66
|
+
# BenchmarkAgent is an ABC; instantiating directly must fail.
|
|
67
|
+
try:
|
|
68
|
+
BenchmarkAgent() # type: ignore[abstract]
|
|
69
|
+
except TypeError:
|
|
70
|
+
pass
|
|
71
|
+
else: # pragma: no cover
|
|
72
|
+
raise AssertionError("BenchmarkAgent should be abstract")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_benchmark_agent_subclass_roundtrip():
|
|
76
|
+
class Echo(BenchmarkAgent):
|
|
77
|
+
def act(self, observation, task, history=None):
|
|
78
|
+
return BenchmarkAction(type="done")
|
|
79
|
+
|
|
80
|
+
agent = Echo()
|
|
81
|
+
obs = BenchmarkObservation()
|
|
82
|
+
task = BenchmarkTask(task_id="t", instruction="i", domain="desktop")
|
|
83
|
+
action = agent.act(obs, task)
|
|
84
|
+
assert action.type == "done"
|
|
85
|
+
# Default reset is a no-op and must not raise.
|
|
86
|
+
agent.reset()
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib.util
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
8
|
+
SCRIPT = ROOT / "scripts" / "verify_release_lock.py"
|
|
9
|
+
SPEC = importlib.util.spec_from_file_location("verify_release_lock", SCRIPT)
|
|
10
|
+
assert SPEC and SPEC.loader
|
|
11
|
+
release_lock = importlib.util.module_from_spec(SPEC)
|
|
12
|
+
SPEC.loader.exec_module(release_lock)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _write_release_files(root: Path, project_version: str, lock_version: str) -> None:
|
|
16
|
+
(root / "pyproject.toml").write_text(
|
|
17
|
+
'[project]\nname = "example-package"\n'
|
|
18
|
+
f'version = "{project_version}"\n',
|
|
19
|
+
encoding="utf-8",
|
|
20
|
+
)
|
|
21
|
+
(root / "uv.lock").write_text(
|
|
22
|
+
'[[package]]\nname = "dependency"\nversion = "8.1.8"\n'
|
|
23
|
+
'source = { registry = "https://pypi.org/simple" }\n\n'
|
|
24
|
+
'[[package]]\nname = "example-package"\n'
|
|
25
|
+
f'version = "{lock_version}"\nsource = {{ editable = "." }}\n',
|
|
26
|
+
encoding="utf-8",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_real_release_metadata_is_consistent() -> None:
|
|
31
|
+
project_version, lock_version = release_lock.release_versions()
|
|
32
|
+
assert project_version == lock_version
|
|
33
|
+
release_lock.verify_release_lock()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_release_lock_rejects_version_drift(tmp_path: Path) -> None:
|
|
37
|
+
_write_release_files(tmp_path, "0.4.0", "0.3.0")
|
|
38
|
+
try:
|
|
39
|
+
release_lock.verify_release_lock(tmp_path)
|
|
40
|
+
except ValueError as exc:
|
|
41
|
+
assert "pyproject.toml=0.4.0, uv.lock=0.3.0" in str(exc)
|
|
42
|
+
else:
|
|
43
|
+
raise AssertionError("version drift was accepted")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_sync_changes_only_editable_root_and_is_idempotent(tmp_path: Path) -> None:
|
|
47
|
+
_write_release_files(tmp_path, "0.4.0", "0.3.0")
|
|
48
|
+
before = (tmp_path / "uv.lock").read_text(encoding="utf-8")
|
|
49
|
+
assert release_lock.synchronize_release_lock(tmp_path) is True
|
|
50
|
+
after = (tmp_path / "uv.lock").read_text(encoding="utf-8")
|
|
51
|
+
assert after == before.replace(
|
|
52
|
+
'name = "example-package"\nversion = "0.3.0"',
|
|
53
|
+
'name = "example-package"\nversion = "0.4.0"',
|
|
54
|
+
)
|
|
55
|
+
assert release_lock.synchronize_release_lock(tmp_path) is False
|
|
56
|
+
assert (tmp_path / "uv.lock").read_text(encoding="utf-8") == after
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_release_configuration_is_fail_closed() -> None:
|
|
60
|
+
metadata = (ROOT / "pyproject.toml").read_text(encoding="utf-8")
|
|
61
|
+
workflow = (ROOT / ".github/workflows/publish.yml").read_text(encoding="utf-8")
|
|
62
|
+
test_workflow = (ROOT / ".github/workflows/test.yml").read_text(
|
|
63
|
+
encoding="utf-8"
|
|
64
|
+
)
|
|
65
|
+
assert "major_on_zero = false" in metadata
|
|
66
|
+
assert "allow_zero_version = true" in metadata
|
|
67
|
+
assert (
|
|
68
|
+
"python -m pip install uv==0.11.29 && "
|
|
69
|
+
"python scripts/verify_release_lock.py --write && "
|
|
70
|
+
"git add uv.lock && uv build"
|
|
71
|
+
) in metadata
|
|
72
|
+
assert metadata.index("python -m pip install uv==0.11.29") < metadata.index(
|
|
73
|
+
"python scripts/verify_release_lock.py --write"
|
|
74
|
+
) < metadata.index("git add uv.lock") < metadata.index("uv build")
|
|
75
|
+
assert "run: uv build" not in workflow
|
|
76
|
+
assert "astral-sh/setup-uv" not in workflow
|
|
77
|
+
assert "actions/setup-python" not in workflow
|
|
78
|
+
assert workflow.count("secrets.ADMIN_TOKEN") >= 3
|
|
79
|
+
assert "secrets.GITHUB_TOKEN" not in workflow
|
|
80
|
+
assert 'version: "0.11.29"' in test_workflow
|
|
81
|
+
assert 'python-version: "3.12"' in test_workflow
|
|
82
|
+
assert "uv sync --locked --extra dev" in test_workflow
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_all_third_party_actions_are_commit_pinned() -> None:
|
|
86
|
+
action_pattern = re.compile(r"uses:\s*([^\s@]+)@([^\s#]+)")
|
|
87
|
+
for path in (ROOT / ".github" / "workflows").glob("*.yml"):
|
|
88
|
+
for action, action_ref in action_pattern.findall(path.read_text(encoding="utf-8")):
|
|
89
|
+
assert re.fullmatch(r"[0-9a-f]{40}", action_ref), (
|
|
90
|
+
f"{path.name}: {action}@{action_ref} is not pinned to a commit"
|
|
91
|
+
)
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
name: test
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches: [main]
|
|
6
|
-
pull_request:
|
|
7
|
-
branches: [main]
|
|
8
|
-
|
|
9
|
-
jobs:
|
|
10
|
-
test:
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
steps:
|
|
13
|
-
- uses: actions/checkout@v4
|
|
14
|
-
- uses: astral-sh/setup-uv@v4
|
|
15
|
-
- run: uv sync --extra dev
|
|
16
|
-
- run: uv run pytest
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|