inspect-ai 0.3.70__py3-none-any.whl → 0.3.72__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inspect_ai/_cli/eval.py +14 -8
- inspect_ai/_display/core/display.py +2 -0
- inspect_ai/_display/core/footer.py +13 -3
- inspect_ai/_display/plain/display.py +6 -2
- inspect_ai/_display/rich/display.py +19 -6
- inspect_ai/_display/textual/app.py +6 -1
- inspect_ai/_display/textual/display.py +4 -0
- inspect_ai/_display/textual/widgets/transcript.py +10 -6
- inspect_ai/_eval/task/run.py +5 -8
- inspect_ai/_util/content.py +20 -1
- inspect_ai/_util/transcript.py +10 -4
- inspect_ai/_util/working.py +4 -0
- inspect_ai/_view/www/App.css +6 -0
- inspect_ai/_view/www/dist/assets/index.css +115 -87
- inspect_ai/_view/www/dist/assets/index.js +5324 -2276
- inspect_ai/_view/www/eslint.config.mjs +24 -1
- inspect_ai/_view/www/log-schema.json +283 -20
- inspect_ai/_view/www/package.json +8 -3
- inspect_ai/_view/www/src/App.tsx +2 -2
- inspect_ai/_view/www/src/components/AnsiDisplay.tsx +4 -3
- inspect_ai/_view/www/src/components/Card.tsx +9 -8
- inspect_ai/_view/www/src/components/DownloadButton.tsx +2 -1
- inspect_ai/_view/www/src/components/EmptyPanel.tsx +2 -2
- inspect_ai/_view/www/src/components/ErrorPanel.tsx +4 -3
- inspect_ai/_view/www/src/components/ExpandablePanel.tsx +13 -5
- inspect_ai/_view/www/src/components/FindBand.tsx +3 -3
- inspect_ai/_view/www/src/components/HumanBaselineView.tsx +3 -3
- inspect_ai/_view/www/src/components/LabeledValue.tsx +5 -4
- inspect_ai/_view/www/src/components/LargeModal.tsx +18 -13
- inspect_ai/_view/www/src/components/{LightboxCarousel.css → LightboxCarousel.module.css} +22 -18
- inspect_ai/_view/www/src/components/LightboxCarousel.tsx +36 -27
- inspect_ai/_view/www/src/components/MessageBand.tsx +2 -1
- inspect_ai/_view/www/src/components/NavPills.tsx +9 -8
- inspect_ai/_view/www/src/components/ProgressBar.tsx +2 -1
- inspect_ai/_view/www/src/components/TabSet.tsx +21 -15
- inspect_ai/_view/www/src/index.tsx +2 -2
- inspect_ai/_view/www/src/metadata/MetaDataGrid.tsx +11 -9
- inspect_ai/_view/www/src/metadata/MetaDataView.tsx +3 -2
- inspect_ai/_view/www/src/metadata/MetadataGrid.module.css +1 -0
- inspect_ai/_view/www/src/metadata/RenderedContent.tsx +16 -0
- inspect_ai/_view/www/src/plan/DatasetDetailView.tsx +3 -2
- inspect_ai/_view/www/src/plan/DetailStep.tsx +2 -1
- inspect_ai/_view/www/src/plan/PlanCard.tsx +2 -5
- inspect_ai/_view/www/src/plan/PlanDetailView.tsx +6 -9
- inspect_ai/_view/www/src/plan/ScorerDetailView.tsx +2 -1
- inspect_ai/_view/www/src/plan/SolverDetailView.tsx +3 -3
- inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx +2 -2
- inspect_ai/_view/www/src/samples/SampleDialog.tsx +3 -3
- inspect_ai/_view/www/src/samples/SampleDisplay.tsx +2 -2
- inspect_ai/_view/www/src/samples/SampleSummaryView.tsx +2 -2
- inspect_ai/_view/www/src/samples/SamplesTools.tsx +2 -1
- inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx +3 -19
- inspect_ai/_view/www/src/samples/chat/ChatMessageRenderer.tsx +2 -1
- inspect_ai/_view/www/src/samples/chat/ChatMessageRow.tsx +2 -1
- inspect_ai/_view/www/src/samples/chat/ChatView.tsx +2 -1
- inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.tsx +22 -7
- inspect_ai/_view/www/src/samples/chat/MessageContent.tsx +35 -6
- inspect_ai/_view/www/src/samples/chat/MessageContents.tsx +2 -2
- inspect_ai/_view/www/src/samples/chat/messages.ts +15 -2
- inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.tsx +13 -4
- inspect_ai/_view/www/src/samples/chat/tools/ToolInput.module.css +2 -2
- inspect_ai/_view/www/src/samples/chat/tools/ToolInput.tsx +18 -19
- inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.module.css +1 -1
- inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.tsx +4 -3
- inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.tsx +2 -2
- inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.tsx +2 -3
- inspect_ai/_view/www/src/samples/error/SampleErrorView.tsx +3 -2
- inspect_ai/_view/www/src/samples/list/SampleFooter.tsx +2 -1
- inspect_ai/_view/www/src/samples/list/SampleHeader.tsx +2 -1
- inspect_ai/_view/www/src/samples/list/SampleList.tsx +57 -45
- inspect_ai/_view/www/src/samples/list/SampleRow.tsx +2 -1
- inspect_ai/_view/www/src/samples/list/SampleSeparator.tsx +2 -1
- inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.tsx +2 -2
- inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx +4 -3
- inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx +2 -5
- inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx +2 -2
- inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx +2 -1
- inspect_ai/_view/www/src/samples/scores/SampleScores.tsx +2 -2
- inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/LoggerEventView.module.css +4 -0
- inspect_ai/_view/www/src/samples/transcript/LoggerEventView.tsx +12 -2
- inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css +1 -1
- inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx +25 -28
- inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx +5 -4
- inspect_ai/_view/www/src/samples/transcript/SampleTranscript.tsx +2 -2
- inspect_ai/_view/www/src/samples/transcript/SandboxEventView.tsx +8 -7
- inspect_ai/_view/www/src/samples/transcript/ScoreEventView.tsx +2 -2
- inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx +3 -3
- inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.tsx +18 -14
- inspect_ai/_view/www/src/samples/transcript/ToolEventView.tsx +5 -5
- inspect_ai/_view/www/src/samples/transcript/TranscriptView.tsx +34 -15
- inspect_ai/_view/www/src/samples/transcript/event/EventNav.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/event/EventNavs.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/event/EventRow.tsx +3 -2
- inspect_ai/_view/www/src/samples/transcript/event/EventSection.tsx +2 -2
- inspect_ai/_view/www/src/samples/transcript/event/EventTimingPanel.module.css +28 -0
- inspect_ai/_view/www/src/samples/transcript/event/EventTimingPanel.tsx +115 -0
- inspect_ai/_view/www/src/samples/transcript/event/utils.ts +29 -0
- inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.tsx +2 -1
- inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx +3 -3
- inspect_ai/_view/www/src/samples/transcript/state/StateEventView.tsx +11 -8
- inspect_ai/_view/www/src/types/log.d.ts +129 -34
- inspect_ai/_view/www/src/usage/ModelTokenTable.tsx +6 -10
- inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css +4 -0
- inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx +32 -9
- inspect_ai/_view/www/src/usage/TokenTable.tsx +4 -6
- inspect_ai/_view/www/src/usage/UsageCard.tsx +2 -1
- inspect_ai/_view/www/src/utils/format.ts +1 -1
- inspect_ai/_view/www/src/utils/json.ts +24 -0
- inspect_ai/_view/www/src/workspace/WorkSpace.tsx +6 -5
- inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx +9 -2
- inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.tsx +2 -1
- inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx +2 -1
- inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx +3 -3
- inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx +4 -3
- inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx +5 -4
- inspect_ai/_view/www/src/workspace/navbar/StatusPanel.tsx +5 -8
- inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.tsx +5 -4
- inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.tsx +2 -1
- inspect_ai/_view/www/src/workspace/sidebar/Sidebar.tsx +2 -1
- inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.tsx +2 -2
- inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx +2 -1
- inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx +2 -2
- inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx +2 -2
- inspect_ai/_view/www/src/workspace/tabs/JsonTab.tsx +2 -5
- inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx +12 -11
- inspect_ai/_view/www/yarn.lock +241 -5
- inspect_ai/log/_condense.py +3 -0
- inspect_ai/log/_recorders/eval.py +6 -1
- inspect_ai/log/_transcript.py +58 -1
- inspect_ai/model/__init__.py +2 -0
- inspect_ai/model/_call_tools.py +7 -0
- inspect_ai/model/_chat_message.py +22 -7
- inspect_ai/model/_conversation.py +10 -8
- inspect_ai/model/_generate_config.py +25 -4
- inspect_ai/model/_model.py +133 -57
- inspect_ai/model/_model_output.py +3 -0
- inspect_ai/model/_openai.py +106 -40
- inspect_ai/model/_providers/anthropic.py +281 -153
- inspect_ai/model/_providers/google.py +27 -8
- inspect_ai/model/_providers/groq.py +9 -4
- inspect_ai/model/_providers/openai.py +57 -4
- inspect_ai/model/_providers/openai_o1.py +10 -0
- inspect_ai/model/_providers/providers.py +1 -1
- inspect_ai/model/_reasoning.py +15 -2
- inspect_ai/scorer/_model.py +23 -19
- inspect_ai/solver/_human_agent/agent.py +14 -10
- inspect_ai/solver/_human_agent/commands/__init__.py +7 -3
- inspect_ai/solver/_human_agent/commands/submit.py +76 -30
- inspect_ai/tool/__init__.py +2 -0
- inspect_ai/tool/_tool.py +3 -1
- inspect_ai/tool/_tools/_computer/_common.py +117 -58
- inspect_ai/tool/_tools/_computer/_computer.py +80 -57
- inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/Code/User/settings.json +7 -1
- inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml +91 -0
- inspect_ai/tool/_tools/_computer/_resources/tool/.pylintrc +8 -0
- inspect_ai/tool/_tools/_computer/_resources/tool/.vscode/settings.json +12 -0
- inspect_ai/tool/_tools/_computer/_resources/tool/_args.py +78 -0
- inspect_ai/tool/_tools/_computer/_resources/tool/_constants.py +20 -0
- inspect_ai/tool/_tools/_computer/_resources/tool/_run.py +1 -1
- inspect_ai/tool/_tools/_computer/_resources/tool/_x11_client.py +175 -113
- inspect_ai/tool/_tools/_computer/_resources/tool/computer_tool.py +76 -20
- inspect_ai/tool/_tools/_computer/_resources/tool/pyproject.toml +65 -0
- inspect_ai/tool/_tools/_computer/test_args.py +151 -0
- inspect_ai/tool/_tools/_web_browser/_resources/.pylintrc +8 -0
- inspect_ai/tool/_tools/_web_browser/_resources/.vscode/launch.json +24 -0
- inspect_ai/tool/_tools/_web_browser/_resources/.vscode/settings.json +25 -0
- inspect_ai/tool/_tools/_web_browser/_resources/Dockerfile +5 -6
- inspect_ai/tool/_tools/_web_browser/_resources/README.md +10 -11
- inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree.py +71 -0
- inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree_node.py +323 -0
- inspect_ai/tool/_tools/_web_browser/_resources/cdp/__init__.py +5 -0
- inspect_ai/tool/_tools/_web_browser/_resources/cdp/a11y.py +279 -0
- inspect_ai/tool/_tools/_web_browser/_resources/cdp/dom.py +9 -0
- inspect_ai/tool/_tools/_web_browser/_resources/cdp/dom_snapshot.py +293 -0
- inspect_ai/tool/_tools/_web_browser/_resources/cdp/page.py +94 -0
- inspect_ai/tool/_tools/_web_browser/_resources/constants.py +2 -0
- inspect_ai/tool/_tools/_web_browser/_resources/images/usage_diagram.svg +2 -0
- inspect_ai/tool/_tools/_web_browser/_resources/playwright_browser.py +50 -0
- inspect_ai/tool/_tools/_web_browser/_resources/playwright_crawler.py +31 -359
- inspect_ai/tool/_tools/_web_browser/_resources/playwright_page_crawler.py +280 -0
- inspect_ai/tool/_tools/_web_browser/_resources/pyproject.toml +65 -0
- inspect_ai/tool/_tools/_web_browser/_resources/rectangle.py +64 -0
- inspect_ai/tool/_tools/_web_browser/_resources/rpc_client_helpers.py +146 -0
- inspect_ai/tool/_tools/_web_browser/_resources/scale_factor.py +64 -0
- inspect_ai/tool/_tools/_web_browser/_resources/test_accessibility_tree_node.py +180 -0
- inspect_ai/tool/_tools/_web_browser/_resources/test_playwright_crawler.py +15 -9
- inspect_ai/tool/_tools/_web_browser/_resources/test_rectangle.py +15 -0
- inspect_ai/tool/_tools/_web_browser/_resources/test_web_client.py +44 -0
- inspect_ai/tool/_tools/_web_browser/_resources/web_browser_rpc_types.py +39 -0
- inspect_ai/tool/_tools/_web_browser/_resources/web_client.py +198 -48
- inspect_ai/tool/_tools/_web_browser/_resources/web_client_new_session.py +26 -25
- inspect_ai/tool/_tools/_web_browser/_resources/web_server.py +178 -39
- inspect_ai/tool/_tools/_web_browser/_web_browser.py +38 -19
- inspect_ai/util/__init__.py +2 -1
- inspect_ai/util/_display.py +12 -0
- inspect_ai/util/_sandbox/events.py +55 -21
- inspect_ai/util/_sandbox/self_check.py +131 -43
- inspect_ai/util/_subtask.py +11 -0
- {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/METADATA +1 -1
- {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/RECORD +209 -186
- {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/WHEEL +1 -1
- inspect_ai/_view/www/src/components/VirtualList.module.css +0 -19
- inspect_ai/_view/www/src/components/VirtualList.tsx +0 -292
- inspect_ai/tool/_tools/_computer/_computer_split.py +0 -198
- inspect_ai/tool/_tools/_web_browser/_resources/accessibility_node.py +0 -312
- inspect_ai/tool/_tools/_web_browser/_resources/dm_env_servicer.py +0 -275
- inspect_ai/tool/_tools/_web_browser/_resources/images/usage_diagram.png +0 -0
- inspect_ai/tool/_tools/_web_browser/_resources/test_accessibility_node.py +0 -176
- inspect_ai/tool/_tools/_web_browser/_resources/test_dm_env_servicer.py +0 -135
- inspect_ai/tool/_tools/_web_browser/_resources/test_web_environment.py +0 -71
- inspect_ai/tool/_tools/_web_browser/_resources/web_environment.py +0 -184
- {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/LICENSE +0 -0
- {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/entry_points.txt +0 -0
- {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,151 @@
|
|
1
|
+
import pytest
|
2
|
+
|
3
|
+
from ._resources.tool._args import parse_arguments
|
4
|
+
|
5
|
+
|
6
|
+
def test_parse_args_screenshot() -> None:
|
7
|
+
args = parse_arguments(["screenshot"])
|
8
|
+
assert args.action == "screenshot"
|
9
|
+
|
10
|
+
|
11
|
+
def test_parse_args_cursor_position() -> None:
|
12
|
+
args = parse_arguments(["cursor_position"])
|
13
|
+
assert args.action == "cursor_position"
|
14
|
+
|
15
|
+
|
16
|
+
def test_parse_args_type() -> None:
|
17
|
+
args = parse_arguments(["type", "--text", "hello"])
|
18
|
+
assert args.action == "type"
|
19
|
+
assert args.text == "hello"
|
20
|
+
|
21
|
+
|
22
|
+
def test_parse_args_mouse_move() -> None:
|
23
|
+
args = parse_arguments(["mouse_move", "--coordinate", "100", "200"])
|
24
|
+
assert args.action == "mouse_move"
|
25
|
+
assert args.coordinate == [100, 200]
|
26
|
+
|
27
|
+
|
28
|
+
def test_parse_args_left_click() -> None:
|
29
|
+
args = parse_arguments(["left_click", "--coordinate", "100", "200"])
|
30
|
+
assert args.action == "left_click"
|
31
|
+
assert args.coordinate == [100, 200]
|
32
|
+
|
33
|
+
|
34
|
+
def test_parse_args_right_click() -> None:
|
35
|
+
args = parse_arguments(["right_click", "--coordinate", "100", "200"])
|
36
|
+
assert args.action == "right_click"
|
37
|
+
assert args.coordinate == [100, 200]
|
38
|
+
|
39
|
+
|
40
|
+
def test_parse_args_middle_click() -> None:
|
41
|
+
args = parse_arguments(["middle_click", "--coordinate", "100", "200"])
|
42
|
+
assert args.action == "middle_click"
|
43
|
+
assert args.coordinate == [100, 200]
|
44
|
+
|
45
|
+
|
46
|
+
def test_parse_args_double_click() -> None:
|
47
|
+
args = parse_arguments(["double_click", "--coordinate", "100", "200"])
|
48
|
+
assert args.action == "double_click"
|
49
|
+
assert args.coordinate == [100, 200]
|
50
|
+
|
51
|
+
|
52
|
+
def test_parse_args_triple_click() -> None:
|
53
|
+
args = parse_arguments(["triple_click", "--coordinate", "100", "200"])
|
54
|
+
assert args.action == "triple_click"
|
55
|
+
assert args.coordinate == [100, 200]
|
56
|
+
|
57
|
+
|
58
|
+
def test_parse_args_hold_key() -> None:
|
59
|
+
args = parse_arguments(["hold_key", "--text", "a", "--duration", "5"])
|
60
|
+
assert args.action == "hold_key"
|
61
|
+
assert args.text == "a"
|
62
|
+
assert args.duration == 5
|
63
|
+
|
64
|
+
|
65
|
+
def test_parse_args_left_click_drag() -> None:
|
66
|
+
args = parse_arguments(
|
67
|
+
[
|
68
|
+
"left_click_drag",
|
69
|
+
"--start_coordinate",
|
70
|
+
"100",
|
71
|
+
"200",
|
72
|
+
"--coordinate",
|
73
|
+
"300",
|
74
|
+
"400",
|
75
|
+
"--text",
|
76
|
+
"drag",
|
77
|
+
]
|
78
|
+
)
|
79
|
+
assert args.action == "left_click_drag"
|
80
|
+
assert args.start_coordinate == [100, 200]
|
81
|
+
assert args.coordinate == [300, 400]
|
82
|
+
assert args.text == "drag"
|
83
|
+
|
84
|
+
|
85
|
+
def test_parse_args_scroll() -> None:
|
86
|
+
args = parse_arguments(
|
87
|
+
[
|
88
|
+
"scroll",
|
89
|
+
"--scroll_direction",
|
90
|
+
"up",
|
91
|
+
"--scroll_amount",
|
92
|
+
"10",
|
93
|
+
"--coordinate",
|
94
|
+
"100",
|
95
|
+
"200",
|
96
|
+
]
|
97
|
+
)
|
98
|
+
assert args.action == "scroll"
|
99
|
+
assert args.scroll_direction == "up"
|
100
|
+
assert args.scroll_amount == 10
|
101
|
+
assert args.coordinate == [100, 200]
|
102
|
+
|
103
|
+
|
104
|
+
def test_parse_args_wait() -> None:
|
105
|
+
args = parse_arguments(["wait", "--duration", "5"])
|
106
|
+
assert args.action == "wait"
|
107
|
+
assert args.duration == 5
|
108
|
+
|
109
|
+
|
110
|
+
def test_parse_args_type_missing_text() -> None:
|
111
|
+
with pytest.raises(SystemExit):
|
112
|
+
parse_arguments(["type"])
|
113
|
+
|
114
|
+
|
115
|
+
def test_parse_args_invalid_action() -> None:
|
116
|
+
with pytest.raises(SystemExit):
|
117
|
+
parse_arguments(["invalid_action"])
|
118
|
+
|
119
|
+
|
120
|
+
def test_parse_args_mouse_move_missing_coordinate() -> None:
|
121
|
+
with pytest.raises(SystemExit):
|
122
|
+
parse_arguments(["mouse_move"])
|
123
|
+
|
124
|
+
|
125
|
+
def test_parse_args_click_invalid_coordinate() -> None:
|
126
|
+
with pytest.raises(SystemExit):
|
127
|
+
parse_arguments(["left_click", "--coordinate", "100"])
|
128
|
+
|
129
|
+
|
130
|
+
def test_parse_args_hold_key_missing_duration() -> None:
|
131
|
+
with pytest.raises(SystemExit):
|
132
|
+
parse_arguments(["hold_key", "--text", "a"])
|
133
|
+
|
134
|
+
|
135
|
+
def test_parse_args_left_click_drag_missing_start_coordinate() -> None:
|
136
|
+
with pytest.raises(SystemExit):
|
137
|
+
parse_arguments(
|
138
|
+
["left_click_drag", "--coordinate", "300", "400", "--text", "drag"]
|
139
|
+
)
|
140
|
+
|
141
|
+
|
142
|
+
def test_parse_args_scroll_missing_scroll_direction() -> None:
|
143
|
+
with pytest.raises(SystemExit):
|
144
|
+
parse_arguments(
|
145
|
+
["scroll", "--scroll_amount", "10", "--coordinate", "100", "200"]
|
146
|
+
)
|
147
|
+
|
148
|
+
|
149
|
+
def test_parse_args_wait_missing_duration() -> None:
|
150
|
+
with pytest.raises(SystemExit):
|
151
|
+
parse_arguments(["wait"])
|
@@ -0,0 +1,24 @@
|
|
1
|
+
{
|
2
|
+
"version": "0.2.0",
|
3
|
+
"configurations": [
|
4
|
+
{
|
5
|
+
"type": "debugpy",
|
6
|
+
"request": "launch",
|
7
|
+
"name": "Debug Web Server",
|
8
|
+
"program": "${workspaceFolder}/web_server.py"
|
9
|
+
},
|
10
|
+
{
|
11
|
+
"type": "debugpy",
|
12
|
+
"request": "launch",
|
13
|
+
"name": "Debug Web Client interactive mode",
|
14
|
+
"program": "${workspaceFolder}/web_client.py"
|
15
|
+
},
|
16
|
+
{
|
17
|
+
"type": "debugpy",
|
18
|
+
"request": "launch",
|
19
|
+
"name": "Debug Web Client w/arguments",
|
20
|
+
"program": "${workspaceFolder}/web_client.py",
|
21
|
+
"args": ["${command:pickArgs}"]
|
22
|
+
}
|
23
|
+
]
|
24
|
+
}
|
@@ -0,0 +1,25 @@
|
|
1
|
+
{
|
2
|
+
"cSpell.words": [
|
3
|
+
"activedescendant",
|
4
|
+
"describedby",
|
5
|
+
"domcontentloaded",
|
6
|
+
"figcaption",
|
7
|
+
"flowto",
|
8
|
+
"framenavigated",
|
9
|
+
"headful",
|
10
|
+
"idref",
|
11
|
+
"jsonrpcclient",
|
12
|
+
"jsonrpcserver",
|
13
|
+
"keepalive",
|
14
|
+
"keyshortcuts",
|
15
|
+
"labelfor",
|
16
|
+
"labelledby",
|
17
|
+
"labelwrapped",
|
18
|
+
"multiselectable",
|
19
|
+
"Rects",
|
20
|
+
"roledescription",
|
21
|
+
"rubyannotation",
|
22
|
+
"tablecaption",
|
23
|
+
"valuetext"
|
24
|
+
]
|
25
|
+
}
|
@@ -8,16 +8,15 @@ RUN apt-get update
|
|
8
8
|
|
9
9
|
RUN pip install --upgrade pip
|
10
10
|
|
11
|
+
RUN pip install playwright jsonrpcclient jsonrpcserver httpx aiohttp pillow pydantic tenacity
|
12
|
+
|
11
13
|
# Install playwright
|
12
|
-
RUN pip install playwright
|
13
14
|
RUN playwright install
|
14
15
|
RUN playwright install-deps
|
15
16
|
|
16
|
-
# Install other dependancies
|
17
|
-
RUN pip install dm-env-rpc pillow bs4 lxml
|
18
|
-
|
19
17
|
# Copy Python files alongside the Dockerfile
|
20
|
-
COPY
|
18
|
+
COPY . .
|
21
19
|
|
22
20
|
# Run the server
|
23
|
-
CMD ["python3", "/app/web_browser/web_server.py"]
|
21
|
+
CMD ["python3", "/app/web_browser/web_server.py"]
|
22
|
+
# CMD ["tail", "-f", "/dev/null"]
|
@@ -1,7 +1,6 @@
|
|
1
1
|
## Headless Browser Tool
|
2
2
|
|
3
|
-
This directory contains an implementation for the Headless Browser Tool which can be used to test web browsing agents.
|
4
|
-
|
3
|
+
This directory contains an implementation for the Headless Browser Tool which can be used to test web browsing agents.
|
5
4
|
|
6
5
|
### Usage
|
7
6
|
|
@@ -37,27 +36,27 @@ The result will be printed out in _stdout_ in the following format:
|
|
37
36
|
|
38
37
|
```
|
39
38
|
# Inside the Docker container
|
40
|
-
error: <an ERROR message if one
|
39
|
+
error: <an ERROR message if one occurred>
|
41
40
|
info: <general info about the container>
|
42
41
|
web_url: <the URL of the page the browser is currently at>
|
43
42
|
web_at: <accessibility tree of the visible elements of the page>
|
44
|
-
```
|
45
|
-
|
43
|
+
```
|
46
44
|
|
47
45
|
### Design
|
48
46
|
|
49
47
|
The following diagram describes the design and the intended usage of the tool:
|
50
48
|
|
51
|
-

|
52
50
|
|
53
51
|
The tool consists of the following components:
|
54
52
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
53
|
+
- [WebServer](web_server.py) - a server which launches a stateful session with the headless chromium browser and interacts with it through the [Playwright API](https://playwright.dev/python/docs/intro) upon receiving client commands. The server components are:
|
54
|
+
|
55
|
+
- _dm_env_servicer.py_ - an implementation for the gRPC Service based on [dm_env_rpc protocol](https://github.com/google-deepmind/dm_env_rpc).
|
56
|
+
- _web_environment.py_ - an environment which gets instantiated by the servicer and which launches the browser, stores its state and maps client commands to Playwright API.
|
57
|
+
- _playwright_crawler.py_ - a wrapper over the sync Playwright API.
|
59
58
|
|
60
|
-
|
59
|
+
- [WebClient](web_client.py) - a simple stateless client to interact with the server. When launched, the client:
|
61
60
|
1. creates a connection with the server;
|
62
61
|
2. sends user command to the server;
|
63
62
|
3. receives the response in the form of observations and prints them to stdout;
|
@@ -0,0 +1,71 @@
|
|
1
|
+
from functools import reduce
|
2
|
+
from typing import Iterable, TypedDict
|
3
|
+
|
4
|
+
from accessibility_tree_node import AccessibilityTreeNode
|
5
|
+
from cdp.a11y import AXNode, AXNodeId
|
6
|
+
from cdp.dom_snapshot import DOMSnapshot, create_snapshot_context
|
7
|
+
from rectangle import Rectangle
|
8
|
+
|
9
|
+
_AccType = tuple[
|
10
|
+
AXNode | None,
|
11
|
+
dict[AXNodeId, AXNode],
|
12
|
+
]
|
13
|
+
|
14
|
+
|
15
|
+
class AccessibilityTree(TypedDict):
|
16
|
+
root: AccessibilityTreeNode
|
17
|
+
nodes: dict[AXNodeId, AccessibilityTreeNode]
|
18
|
+
|
19
|
+
|
20
|
+
def create_accessibility_tree(
|
21
|
+
*,
|
22
|
+
ax_nodes: Iterable[AXNode],
|
23
|
+
dom_snapshot: DOMSnapshot,
|
24
|
+
device_scale_factor: float,
|
25
|
+
window_bounds: Rectangle,
|
26
|
+
) -> AccessibilityTree | None:
|
27
|
+
"""
|
28
|
+
Creates an accessibility tree from the given Chrome DevTools Protocol AX nodes and DOM snapshot.
|
29
|
+
|
30
|
+
Args:
|
31
|
+
ax_nodes (Iterable[AXNode]): An iterable of AXNode objects representing the accessibility nodes.
|
32
|
+
dom_snapshot (DOMSnapshot): A snapshot of the DOM at the time of accessibility tree creation.
|
33
|
+
device_scale_factor (float): The scale factor of the device.
|
34
|
+
window_bounds (Bounds): The bounds of the window.
|
35
|
+
|
36
|
+
Returns:
|
37
|
+
AccessibilityTree: The accessibility tree.
|
38
|
+
"""
|
39
|
+
|
40
|
+
# first make a dict of AXNodeId's to AXNode's and find the root on the way
|
41
|
+
def reducer(acc: _AccType, ax_node: AXNode) -> _AccType:
|
42
|
+
root_node, nodes = acc
|
43
|
+
nodes[ax_node.nodeId] = ax_node
|
44
|
+
return (
|
45
|
+
# TODO: What do we want for multiple roots?
|
46
|
+
root_node or (ax_node if ax_node.parentId is None else None),
|
47
|
+
nodes,
|
48
|
+
)
|
49
|
+
|
50
|
+
initial_acc: _AccType = (None, {}) # The inference engine is weak
|
51
|
+
root_node, nodes = reduce(reducer, ax_nodes, initial_acc)
|
52
|
+
|
53
|
+
if not root_node:
|
54
|
+
return None
|
55
|
+
|
56
|
+
# Now create the AccessibilityTreeNode hierarchy
|
57
|
+
snapshot_context = create_snapshot_context(dom_snapshot)
|
58
|
+
all_accessibility_tree_nodes: dict[AXNodeId, AccessibilityTreeNode] = {}
|
59
|
+
|
60
|
+
return AccessibilityTree(
|
61
|
+
root=AccessibilityTreeNode(
|
62
|
+
ax_node=root_node,
|
63
|
+
ax_nodes=nodes,
|
64
|
+
parent=None,
|
65
|
+
all_accessibility_tree_nodes=all_accessibility_tree_nodes,
|
66
|
+
snapshot_context=snapshot_context,
|
67
|
+
device_scale_factor=device_scale_factor,
|
68
|
+
window_bounds=window_bounds,
|
69
|
+
),
|
70
|
+
nodes=all_accessibility_tree_nodes,
|
71
|
+
)
|
@@ -0,0 +1,323 @@
|
|
1
|
+
from functools import reduce
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from cdp.a11y import (
|
5
|
+
AXNode,
|
6
|
+
AXNodeId,
|
7
|
+
AXProperty,
|
8
|
+
AXPropertyName,
|
9
|
+
node_has_property,
|
10
|
+
string_from_ax_value,
|
11
|
+
)
|
12
|
+
from cdp.dom_snapshot import (
|
13
|
+
DOMSnapshotContext,
|
14
|
+
bounds_for_node_index,
|
15
|
+
current_url_src_for_node_index,
|
16
|
+
index_for_node_id,
|
17
|
+
text_value_for_node_index,
|
18
|
+
)
|
19
|
+
from rectangle import Rectangle
|
20
|
+
|
21
|
+
# Properties to ignore when printing out the accessibility tree.
|
22
|
+
_IGNORED_AT_PROPERTIES: set[AXPropertyName] = {
|
23
|
+
"focusable",
|
24
|
+
"readonly",
|
25
|
+
"level",
|
26
|
+
"settable",
|
27
|
+
"multiline",
|
28
|
+
"invalid",
|
29
|
+
}
|
30
|
+
|
31
|
+
# The AXRole enum is here (https://chromium.googlesource.com/chromium/chromium/+/HEAD/ui/accessibility/ax_enums.idl#62)
|
32
|
+
_ROLES_IGNORED = {"horizontal_rule", "ignored"}
|
33
|
+
|
34
|
+
# These are noisy and provide no real benefit unless there's a name associated with them
|
35
|
+
_ROLES_REQUIRING_A_NAME = {"generic", "paragraph", "div"}
|
36
|
+
|
37
|
+
# Used for debugging, include the elements bounding rect during output.
|
38
|
+
_INCLUDE_BOUNDS_IN_OUTPUT = False
|
39
|
+
|
40
|
+
|
41
|
+
class AccessibilityTreeNode:
|
42
|
+
def __init__(
|
43
|
+
self,
|
44
|
+
*,
|
45
|
+
ax_node: AXNode,
|
46
|
+
ax_nodes: dict[AXNodeId, AXNode],
|
47
|
+
parent: Optional["AccessibilityTreeNode"],
|
48
|
+
all_accessibility_tree_nodes: dict[AXNodeId, "AccessibilityTreeNode"],
|
49
|
+
snapshot_context: DOMSnapshotContext,
|
50
|
+
device_scale_factor: float,
|
51
|
+
window_bounds: Rectangle,
|
52
|
+
):
|
53
|
+
all_accessibility_tree_nodes[ax_node.nodeId] = self
|
54
|
+
|
55
|
+
self._is_expanded = node_has_property(ax_node, "expanded")
|
56
|
+
self._ax_node = ax_node
|
57
|
+
self._parent = parent
|
58
|
+
self._is_ignored: bool | None = None
|
59
|
+
self._bounds: Rectangle | None = None
|
60
|
+
self._is_visible: bool | None = None
|
61
|
+
self._current_src_url: str | None = None
|
62
|
+
self._input: str | None = None
|
63
|
+
self.__closest_non_ignored_parent: AccessibilityTreeNode | None = None
|
64
|
+
|
65
|
+
dom_node_index = (
|
66
|
+
index_for_node_id(ax_node.backendDOMNodeId, snapshot_context)
|
67
|
+
if ax_node.backendDOMNodeId
|
68
|
+
else None
|
69
|
+
)
|
70
|
+
|
71
|
+
# The following three variables (_bounds, _is_visible, and _current_src_url) depend on
|
72
|
+
# info from the DOM element associated with this AXNode
|
73
|
+
if dom_node_index:
|
74
|
+
self._bounds = (
|
75
|
+
Rectangle(*layout_bounds).scale(1 / device_scale_factor)
|
76
|
+
if (
|
77
|
+
layout_bounds := bounds_for_node_index(
|
78
|
+
dom_node_index, snapshot_context
|
79
|
+
)
|
80
|
+
)
|
81
|
+
is not None
|
82
|
+
else None
|
83
|
+
)
|
84
|
+
self._is_visible = (
|
85
|
+
(self._bounds.has_area and self._bounds.overlaps(window_bounds))
|
86
|
+
if self._bounds
|
87
|
+
else None
|
88
|
+
)
|
89
|
+
self._current_src_url = (
|
90
|
+
current_url_src_for_node_index(dom_node_index, snapshot_context)
|
91
|
+
if self.role == "image"
|
92
|
+
else None
|
93
|
+
)
|
94
|
+
|
95
|
+
if node_has_property(ax_node, "editable"):
|
96
|
+
# Sometimes a node will have it's input stored as 'value' other times we
|
97
|
+
# need to go looking through the DOM tree to find its matching string.
|
98
|
+
self._input = (
|
99
|
+
string_from_ax_value(ax_node.value)
|
100
|
+
if ax_node.value
|
101
|
+
else text_value_for_node_index(dom_node_index, snapshot_context)
|
102
|
+
if dom_node_index is not None
|
103
|
+
else None
|
104
|
+
)
|
105
|
+
|
106
|
+
# this is a little awkward, but the old code ensured that menuitems were visible in given the following condition
|
107
|
+
if self.role == "menuitem" and parent is not None and parent.is_expanded:
|
108
|
+
self._is_visible = True
|
109
|
+
|
110
|
+
# It's important to set all other properties before creating children in case
|
111
|
+
# the children sample parent.xxx
|
112
|
+
self.children = (
|
113
|
+
[
|
114
|
+
AccessibilityTreeNode(
|
115
|
+
ax_node=ax_nodes[child_id],
|
116
|
+
ax_nodes=ax_nodes,
|
117
|
+
parent=self,
|
118
|
+
all_accessibility_tree_nodes=all_accessibility_tree_nodes,
|
119
|
+
snapshot_context=snapshot_context,
|
120
|
+
device_scale_factor=device_scale_factor,
|
121
|
+
window_bounds=window_bounds,
|
122
|
+
)
|
123
|
+
for child_id in ax_node.childIds
|
124
|
+
]
|
125
|
+
if ax_node.childIds
|
126
|
+
else None
|
127
|
+
)
|
128
|
+
|
129
|
+
def __str__(self) -> str:
|
130
|
+
bounds_string = str(self.bounds) if _INCLUDE_BOUNDS_IN_OUTPUT else None
|
131
|
+
|
132
|
+
input_string = f'Current input: "{self._input}"' if self._input else None
|
133
|
+
|
134
|
+
name_string = f'"{self.name}"' if self.name else None
|
135
|
+
# Without bounds we can't click on the element, and therefore the ID
|
136
|
+
# can not be used. Still useful to show these elements though (e.g.)
|
137
|
+
# to let user know about options.
|
138
|
+
id_string = f"[{'*' if not self._bounds or not self._bounds.has_area else self._ax_node.nodeId}]"
|
139
|
+
url_string = self._current_src_url if self._current_src_url else None
|
140
|
+
|
141
|
+
url_string = _maybe_redact_base64_url(self._current_src_url)
|
142
|
+
|
143
|
+
return " ".join(
|
144
|
+
[
|
145
|
+
item
|
146
|
+
for item in [
|
147
|
+
id_string,
|
148
|
+
self.role,
|
149
|
+
name_string,
|
150
|
+
url_string,
|
151
|
+
input_string,
|
152
|
+
self._property_string(),
|
153
|
+
bounds_string,
|
154
|
+
]
|
155
|
+
if item is not None
|
156
|
+
]
|
157
|
+
)
|
158
|
+
|
159
|
+
@property
|
160
|
+
def node_id(self) -> AXNodeId:
|
161
|
+
return self._ax_node.nodeId
|
162
|
+
|
163
|
+
@property
|
164
|
+
def name(self) -> str:
|
165
|
+
return string_from_ax_value(self._ax_node.name)
|
166
|
+
|
167
|
+
@property
|
168
|
+
def is_expanded(self) -> bool:
|
169
|
+
return self._is_expanded
|
170
|
+
|
171
|
+
@property
|
172
|
+
def is_visible(self) -> bool:
|
173
|
+
return self._is_visible or False
|
174
|
+
|
175
|
+
@property
|
176
|
+
def bounds(self) -> Rectangle | None:
|
177
|
+
return self._bounds
|
178
|
+
|
179
|
+
@property
|
180
|
+
def current_src_url(self) -> str | None:
|
181
|
+
return self._current_src_url
|
182
|
+
|
183
|
+
@property
|
184
|
+
def input(self) -> str | None:
|
185
|
+
return self._input
|
186
|
+
|
187
|
+
@property
|
188
|
+
def is_ignored(self) -> bool:
|
189
|
+
# computing _is_ignored is very expensive, do it on demand and remember the result
|
190
|
+
if self._is_ignored is None:
|
191
|
+
self._is_ignored = self._compute_is_ignored()
|
192
|
+
return self._is_ignored
|
193
|
+
|
194
|
+
@property
|
195
|
+
def role(self) -> str:
|
196
|
+
return string_from_ax_value(self._ax_node.role)
|
197
|
+
|
198
|
+
@property
|
199
|
+
def _closest_non_ignored_parent(self) -> Optional["AccessibilityTreeNode"]:
|
200
|
+
if self._parent and self.__closest_non_ignored_parent is None:
|
201
|
+
self.__closest_non_ignored_parent = (
|
202
|
+
None
|
203
|
+
if not self._parent
|
204
|
+
else self._parent
|
205
|
+
if not self._parent.is_ignored
|
206
|
+
else self._parent._closest_non_ignored_parent # pylint: disable=protected-access
|
207
|
+
)
|
208
|
+
return self.__closest_non_ignored_parent
|
209
|
+
|
210
|
+
def render_accessibility_tree(self, indent: int = 0) -> str:
|
211
|
+
"""Returns the string representation of the node and its children."""
|
212
|
+
result = ""
|
213
|
+
if not self.is_ignored:
|
214
|
+
result = " " * indent + str(self) + "\n"
|
215
|
+
indent += 1
|
216
|
+
children_strings = [
|
217
|
+
child.render_accessibility_tree(indent) for child in self.children or []
|
218
|
+
]
|
219
|
+
result = result + "".join([s for s in children_strings if s])
|
220
|
+
return result
|
221
|
+
|
222
|
+
def render_main_content(self) -> str | None:
|
223
|
+
return (
|
224
|
+
main_node._extract_text() # pylint: disable=protected-access
|
225
|
+
if (main_node := self._get_main_content_node())
|
226
|
+
else None
|
227
|
+
)
|
228
|
+
|
229
|
+
def _compute_is_ignored(self) -> bool:
|
230
|
+
if self._ax_node.ignored or not self.is_visible or self.role in _ROLES_IGNORED:
|
231
|
+
return True
|
232
|
+
|
233
|
+
# if the parent is a link/button with a name, and this node is within the parent's bounds, we can ignore it.
|
234
|
+
if (
|
235
|
+
(closest_parent := self._closest_non_ignored_parent)
|
236
|
+
and closest_parent.bounds
|
237
|
+
and self.bounds
|
238
|
+
and (closest_parent.role == "link" or closest_parent.role == "button")
|
239
|
+
and closest_parent.name
|
240
|
+
and self.bounds.within(closest_parent.bounds)
|
241
|
+
):
|
242
|
+
return True
|
243
|
+
|
244
|
+
if bool(self.name): # excludes None or ""
|
245
|
+
# This simplifies things like links/buttons whose contents are sub-elements
|
246
|
+
# e.g.
|
247
|
+
# [158] button "Menu"
|
248
|
+
# [1166] StaticText "Menu"
|
249
|
+
# becomes
|
250
|
+
# [158] button "Menu"
|
251
|
+
return self._parent is not None and self._parent.name == self.name
|
252
|
+
|
253
|
+
return self.role in _ROLES_REQUIRING_A_NAME
|
254
|
+
|
255
|
+
def _get_main_content_node(self) -> Optional["AccessibilityTreeNode"]:
|
256
|
+
if self.role == "main":
|
257
|
+
return self
|
258
|
+
|
259
|
+
return next(
|
260
|
+
(
|
261
|
+
result
|
262
|
+
for child in self.children or []
|
263
|
+
if (result := child._get_main_content_node()) # pylint: disable=protected-access
|
264
|
+
),
|
265
|
+
None,
|
266
|
+
)
|
267
|
+
|
268
|
+
def _extract_text(self):
|
269
|
+
"""
|
270
|
+
Recursively extracts and concatenates text from the accessibility tree nodes.
|
271
|
+
|
272
|
+
This method traverses the accessibility tree starting from the current node,
|
273
|
+
concatenating the text of each node.
|
274
|
+
|
275
|
+
Returns:
|
276
|
+
str: The concatenated text from the accessibility tree nodes, with
|
277
|
+
appropriate spacing based on node roles.
|
278
|
+
"""
|
279
|
+
if not self.is_visible:
|
280
|
+
return ""
|
281
|
+
|
282
|
+
def reducer(acc, child):
|
283
|
+
child_text = child._extract_text() # pylint: disable=protected-access
|
284
|
+
return (
|
285
|
+
acc
|
286
|
+
+ (
|
287
|
+
"\n\n"
|
288
|
+
if child.role
|
289
|
+
in {"paragraph", "note", "row", "div", "heading", "region"}
|
290
|
+
else " "
|
291
|
+
)
|
292
|
+
+ child_text
|
293
|
+
)
|
294
|
+
|
295
|
+
return reduce(reducer, self.children or [], self.name).strip()
|
296
|
+
|
297
|
+
def _property_string(self) -> str:
|
298
|
+
properties = [
|
299
|
+
_prop_name_value_str(node_property)
|
300
|
+
if node_property.value.value
|
301
|
+
else f"{node_property.name}"
|
302
|
+
for node_property in self._ax_node.properties or ()
|
303
|
+
if node_property.name not in _IGNORED_AT_PROPERTIES
|
304
|
+
]
|
305
|
+
return " [" + ", ".join(properties) + "]" if properties else ""
|
306
|
+
|
307
|
+
|
308
|
+
def _prop_name_value_str(node_property: AXProperty) -> str:
|
309
|
+
# pre-commit hook (debug-statements) crashes if value is inlined below
|
310
|
+
value = (
|
311
|
+
_maybe_redact_base64_url(node_property.value.value)
|
312
|
+
if node_property.name == "url" and isinstance(node_property.value.value, str)
|
313
|
+
else node_property.value.value
|
314
|
+
)
|
315
|
+
return f"{node_property.name}: {value}"
|
316
|
+
|
317
|
+
|
318
|
+
def _maybe_redact_base64_url(url: str | None) -> str | None:
|
319
|
+
return (
|
320
|
+
"data:<base64-data-removed>"
|
321
|
+
if url and url.startswith("data:") and "base64" in url
|
322
|
+
else url
|
323
|
+
)
|