inspect-ai 0.3.70__py3-none-any.whl → 0.3.72__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. inspect_ai/_cli/eval.py +14 -8
  2. inspect_ai/_display/core/display.py +2 -0
  3. inspect_ai/_display/core/footer.py +13 -3
  4. inspect_ai/_display/plain/display.py +6 -2
  5. inspect_ai/_display/rich/display.py +19 -6
  6. inspect_ai/_display/textual/app.py +6 -1
  7. inspect_ai/_display/textual/display.py +4 -0
  8. inspect_ai/_display/textual/widgets/transcript.py +10 -6
  9. inspect_ai/_eval/task/run.py +5 -8
  10. inspect_ai/_util/content.py +20 -1
  11. inspect_ai/_util/transcript.py +10 -4
  12. inspect_ai/_util/working.py +4 -0
  13. inspect_ai/_view/www/App.css +6 -0
  14. inspect_ai/_view/www/dist/assets/index.css +115 -87
  15. inspect_ai/_view/www/dist/assets/index.js +5324 -2276
  16. inspect_ai/_view/www/eslint.config.mjs +24 -1
  17. inspect_ai/_view/www/log-schema.json +283 -20
  18. inspect_ai/_view/www/package.json +8 -3
  19. inspect_ai/_view/www/src/App.tsx +2 -2
  20. inspect_ai/_view/www/src/components/AnsiDisplay.tsx +4 -3
  21. inspect_ai/_view/www/src/components/Card.tsx +9 -8
  22. inspect_ai/_view/www/src/components/DownloadButton.tsx +2 -1
  23. inspect_ai/_view/www/src/components/EmptyPanel.tsx +2 -2
  24. inspect_ai/_view/www/src/components/ErrorPanel.tsx +4 -3
  25. inspect_ai/_view/www/src/components/ExpandablePanel.tsx +13 -5
  26. inspect_ai/_view/www/src/components/FindBand.tsx +3 -3
  27. inspect_ai/_view/www/src/components/HumanBaselineView.tsx +3 -3
  28. inspect_ai/_view/www/src/components/LabeledValue.tsx +5 -4
  29. inspect_ai/_view/www/src/components/LargeModal.tsx +18 -13
  30. inspect_ai/_view/www/src/components/{LightboxCarousel.css → LightboxCarousel.module.css} +22 -18
  31. inspect_ai/_view/www/src/components/LightboxCarousel.tsx +36 -27
  32. inspect_ai/_view/www/src/components/MessageBand.tsx +2 -1
  33. inspect_ai/_view/www/src/components/NavPills.tsx +9 -8
  34. inspect_ai/_view/www/src/components/ProgressBar.tsx +2 -1
  35. inspect_ai/_view/www/src/components/TabSet.tsx +21 -15
  36. inspect_ai/_view/www/src/index.tsx +2 -2
  37. inspect_ai/_view/www/src/metadata/MetaDataGrid.tsx +11 -9
  38. inspect_ai/_view/www/src/metadata/MetaDataView.tsx +3 -2
  39. inspect_ai/_view/www/src/metadata/MetadataGrid.module.css +1 -0
  40. inspect_ai/_view/www/src/metadata/RenderedContent.tsx +16 -0
  41. inspect_ai/_view/www/src/plan/DatasetDetailView.tsx +3 -2
  42. inspect_ai/_view/www/src/plan/DetailStep.tsx +2 -1
  43. inspect_ai/_view/www/src/plan/PlanCard.tsx +2 -5
  44. inspect_ai/_view/www/src/plan/PlanDetailView.tsx +6 -9
  45. inspect_ai/_view/www/src/plan/ScorerDetailView.tsx +2 -1
  46. inspect_ai/_view/www/src/plan/SolverDetailView.tsx +3 -3
  47. inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx +2 -2
  48. inspect_ai/_view/www/src/samples/SampleDialog.tsx +3 -3
  49. inspect_ai/_view/www/src/samples/SampleDisplay.tsx +2 -2
  50. inspect_ai/_view/www/src/samples/SampleSummaryView.tsx +2 -2
  51. inspect_ai/_view/www/src/samples/SamplesTools.tsx +2 -1
  52. inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx +3 -19
  53. inspect_ai/_view/www/src/samples/chat/ChatMessageRenderer.tsx +2 -1
  54. inspect_ai/_view/www/src/samples/chat/ChatMessageRow.tsx +2 -1
  55. inspect_ai/_view/www/src/samples/chat/ChatView.tsx +2 -1
  56. inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.tsx +22 -7
  57. inspect_ai/_view/www/src/samples/chat/MessageContent.tsx +35 -6
  58. inspect_ai/_view/www/src/samples/chat/MessageContents.tsx +2 -2
  59. inspect_ai/_view/www/src/samples/chat/messages.ts +15 -2
  60. inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.tsx +13 -4
  61. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.module.css +2 -2
  62. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.tsx +18 -19
  63. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.module.css +1 -1
  64. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.tsx +4 -3
  65. inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.tsx +2 -2
  66. inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.tsx +2 -3
  67. inspect_ai/_view/www/src/samples/error/SampleErrorView.tsx +3 -2
  68. inspect_ai/_view/www/src/samples/list/SampleFooter.tsx +2 -1
  69. inspect_ai/_view/www/src/samples/list/SampleHeader.tsx +2 -1
  70. inspect_ai/_view/www/src/samples/list/SampleList.tsx +57 -45
  71. inspect_ai/_view/www/src/samples/list/SampleRow.tsx +2 -1
  72. inspect_ai/_view/www/src/samples/list/SampleSeparator.tsx +2 -1
  73. inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.tsx +2 -2
  74. inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx +4 -3
  75. inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx +2 -5
  76. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx +2 -2
  77. inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx +2 -1
  78. inspect_ai/_view/www/src/samples/scores/SampleScores.tsx +2 -2
  79. inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.tsx +2 -1
  80. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx +2 -1
  81. inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx +2 -1
  82. inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx +2 -1
  83. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.module.css +4 -0
  84. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.tsx +12 -2
  85. inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css +1 -1
  86. inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx +25 -28
  87. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.tsx +2 -1
  88. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx +5 -4
  89. inspect_ai/_view/www/src/samples/transcript/SampleTranscript.tsx +2 -2
  90. inspect_ai/_view/www/src/samples/transcript/SandboxEventView.tsx +8 -7
  91. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.tsx +2 -2
  92. inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx +3 -3
  93. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.tsx +18 -14
  94. inspect_ai/_view/www/src/samples/transcript/ToolEventView.tsx +5 -5
  95. inspect_ai/_view/www/src/samples/transcript/TranscriptView.tsx +34 -15
  96. inspect_ai/_view/www/src/samples/transcript/event/EventNav.tsx +2 -1
  97. inspect_ai/_view/www/src/samples/transcript/event/EventNavs.tsx +2 -1
  98. inspect_ai/_view/www/src/samples/transcript/event/EventRow.tsx +3 -2
  99. inspect_ai/_view/www/src/samples/transcript/event/EventSection.tsx +2 -2
  100. inspect_ai/_view/www/src/samples/transcript/event/EventTimingPanel.module.css +28 -0
  101. inspect_ai/_view/www/src/samples/transcript/event/EventTimingPanel.tsx +115 -0
  102. inspect_ai/_view/www/src/samples/transcript/event/utils.ts +29 -0
  103. inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.tsx +2 -1
  104. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx +3 -3
  105. inspect_ai/_view/www/src/samples/transcript/state/StateEventView.tsx +11 -8
  106. inspect_ai/_view/www/src/types/log.d.ts +129 -34
  107. inspect_ai/_view/www/src/usage/ModelTokenTable.tsx +6 -10
  108. inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css +4 -0
  109. inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx +32 -9
  110. inspect_ai/_view/www/src/usage/TokenTable.tsx +4 -6
  111. inspect_ai/_view/www/src/usage/UsageCard.tsx +2 -1
  112. inspect_ai/_view/www/src/utils/format.ts +1 -1
  113. inspect_ai/_view/www/src/utils/json.ts +24 -0
  114. inspect_ai/_view/www/src/workspace/WorkSpace.tsx +6 -5
  115. inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx +9 -2
  116. inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.tsx +2 -1
  117. inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx +2 -1
  118. inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx +3 -3
  119. inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx +4 -3
  120. inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx +5 -4
  121. inspect_ai/_view/www/src/workspace/navbar/StatusPanel.tsx +5 -8
  122. inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.tsx +5 -4
  123. inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.tsx +2 -1
  124. inspect_ai/_view/www/src/workspace/sidebar/Sidebar.tsx +2 -1
  125. inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.tsx +2 -2
  126. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx +2 -1
  127. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx +2 -2
  128. inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx +2 -2
  129. inspect_ai/_view/www/src/workspace/tabs/JsonTab.tsx +2 -5
  130. inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx +12 -11
  131. inspect_ai/_view/www/yarn.lock +241 -5
  132. inspect_ai/log/_condense.py +3 -0
  133. inspect_ai/log/_recorders/eval.py +6 -1
  134. inspect_ai/log/_transcript.py +58 -1
  135. inspect_ai/model/__init__.py +2 -0
  136. inspect_ai/model/_call_tools.py +7 -0
  137. inspect_ai/model/_chat_message.py +22 -7
  138. inspect_ai/model/_conversation.py +10 -8
  139. inspect_ai/model/_generate_config.py +25 -4
  140. inspect_ai/model/_model.py +133 -57
  141. inspect_ai/model/_model_output.py +3 -0
  142. inspect_ai/model/_openai.py +106 -40
  143. inspect_ai/model/_providers/anthropic.py +281 -153
  144. inspect_ai/model/_providers/google.py +27 -8
  145. inspect_ai/model/_providers/groq.py +9 -4
  146. inspect_ai/model/_providers/openai.py +57 -4
  147. inspect_ai/model/_providers/openai_o1.py +10 -0
  148. inspect_ai/model/_providers/providers.py +1 -1
  149. inspect_ai/model/_reasoning.py +15 -2
  150. inspect_ai/scorer/_model.py +23 -19
  151. inspect_ai/solver/_human_agent/agent.py +14 -10
  152. inspect_ai/solver/_human_agent/commands/__init__.py +7 -3
  153. inspect_ai/solver/_human_agent/commands/submit.py +76 -30
  154. inspect_ai/tool/__init__.py +2 -0
  155. inspect_ai/tool/_tool.py +3 -1
  156. inspect_ai/tool/_tools/_computer/_common.py +117 -58
  157. inspect_ai/tool/_tools/_computer/_computer.py +80 -57
  158. inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/Code/User/settings.json +7 -1
  159. inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml +91 -0
  160. inspect_ai/tool/_tools/_computer/_resources/tool/.pylintrc +8 -0
  161. inspect_ai/tool/_tools/_computer/_resources/tool/.vscode/settings.json +12 -0
  162. inspect_ai/tool/_tools/_computer/_resources/tool/_args.py +78 -0
  163. inspect_ai/tool/_tools/_computer/_resources/tool/_constants.py +20 -0
  164. inspect_ai/tool/_tools/_computer/_resources/tool/_run.py +1 -1
  165. inspect_ai/tool/_tools/_computer/_resources/tool/_x11_client.py +175 -113
  166. inspect_ai/tool/_tools/_computer/_resources/tool/computer_tool.py +76 -20
  167. inspect_ai/tool/_tools/_computer/_resources/tool/pyproject.toml +65 -0
  168. inspect_ai/tool/_tools/_computer/test_args.py +151 -0
  169. inspect_ai/tool/_tools/_web_browser/_resources/.pylintrc +8 -0
  170. inspect_ai/tool/_tools/_web_browser/_resources/.vscode/launch.json +24 -0
  171. inspect_ai/tool/_tools/_web_browser/_resources/.vscode/settings.json +25 -0
  172. inspect_ai/tool/_tools/_web_browser/_resources/Dockerfile +5 -6
  173. inspect_ai/tool/_tools/_web_browser/_resources/README.md +10 -11
  174. inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree.py +71 -0
  175. inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree_node.py +323 -0
  176. inspect_ai/tool/_tools/_web_browser/_resources/cdp/__init__.py +5 -0
  177. inspect_ai/tool/_tools/_web_browser/_resources/cdp/a11y.py +279 -0
  178. inspect_ai/tool/_tools/_web_browser/_resources/cdp/dom.py +9 -0
  179. inspect_ai/tool/_tools/_web_browser/_resources/cdp/dom_snapshot.py +293 -0
  180. inspect_ai/tool/_tools/_web_browser/_resources/cdp/page.py +94 -0
  181. inspect_ai/tool/_tools/_web_browser/_resources/constants.py +2 -0
  182. inspect_ai/tool/_tools/_web_browser/_resources/images/usage_diagram.svg +2 -0
  183. inspect_ai/tool/_tools/_web_browser/_resources/playwright_browser.py +50 -0
  184. inspect_ai/tool/_tools/_web_browser/_resources/playwright_crawler.py +31 -359
  185. inspect_ai/tool/_tools/_web_browser/_resources/playwright_page_crawler.py +280 -0
  186. inspect_ai/tool/_tools/_web_browser/_resources/pyproject.toml +65 -0
  187. inspect_ai/tool/_tools/_web_browser/_resources/rectangle.py +64 -0
  188. inspect_ai/tool/_tools/_web_browser/_resources/rpc_client_helpers.py +146 -0
  189. inspect_ai/tool/_tools/_web_browser/_resources/scale_factor.py +64 -0
  190. inspect_ai/tool/_tools/_web_browser/_resources/test_accessibility_tree_node.py +180 -0
  191. inspect_ai/tool/_tools/_web_browser/_resources/test_playwright_crawler.py +15 -9
  192. inspect_ai/tool/_tools/_web_browser/_resources/test_rectangle.py +15 -0
  193. inspect_ai/tool/_tools/_web_browser/_resources/test_web_client.py +44 -0
  194. inspect_ai/tool/_tools/_web_browser/_resources/web_browser_rpc_types.py +39 -0
  195. inspect_ai/tool/_tools/_web_browser/_resources/web_client.py +198 -48
  196. inspect_ai/tool/_tools/_web_browser/_resources/web_client_new_session.py +26 -25
  197. inspect_ai/tool/_tools/_web_browser/_resources/web_server.py +178 -39
  198. inspect_ai/tool/_tools/_web_browser/_web_browser.py +38 -19
  199. inspect_ai/util/__init__.py +2 -1
  200. inspect_ai/util/_display.py +12 -0
  201. inspect_ai/util/_sandbox/events.py +55 -21
  202. inspect_ai/util/_sandbox/self_check.py +131 -43
  203. inspect_ai/util/_subtask.py +11 -0
  204. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/METADATA +1 -1
  205. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/RECORD +209 -186
  206. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/WHEEL +1 -1
  207. inspect_ai/_view/www/src/components/VirtualList.module.css +0 -19
  208. inspect_ai/_view/www/src/components/VirtualList.tsx +0 -292
  209. inspect_ai/tool/_tools/_computer/_computer_split.py +0 -198
  210. inspect_ai/tool/_tools/_web_browser/_resources/accessibility_node.py +0 -312
  211. inspect_ai/tool/_tools/_web_browser/_resources/dm_env_servicer.py +0 -275
  212. inspect_ai/tool/_tools/_web_browser/_resources/images/usage_diagram.png +0 -0
  213. inspect_ai/tool/_tools/_web_browser/_resources/test_accessibility_node.py +0 -176
  214. inspect_ai/tool/_tools/_web_browser/_resources/test_dm_env_servicer.py +0 -135
  215. inspect_ai/tool/_tools/_web_browser/_resources/test_web_environment.py +0 -71
  216. inspect_ai/tool/_tools/_web_browser/_resources/web_environment.py +0 -184
  217. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/LICENSE +0 -0
  218. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/entry_points.txt +0 -0
  219. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.72.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,151 @@
1
+ import pytest
2
+
3
+ from ._resources.tool._args import parse_arguments
4
+
5
+
6
+ def test_parse_args_screenshot() -> None:
7
+ args = parse_arguments(["screenshot"])
8
+ assert args.action == "screenshot"
9
+
10
+
11
+ def test_parse_args_cursor_position() -> None:
12
+ args = parse_arguments(["cursor_position"])
13
+ assert args.action == "cursor_position"
14
+
15
+
16
+ def test_parse_args_type() -> None:
17
+ args = parse_arguments(["type", "--text", "hello"])
18
+ assert args.action == "type"
19
+ assert args.text == "hello"
20
+
21
+
22
+ def test_parse_args_mouse_move() -> None:
23
+ args = parse_arguments(["mouse_move", "--coordinate", "100", "200"])
24
+ assert args.action == "mouse_move"
25
+ assert args.coordinate == [100, 200]
26
+
27
+
28
+ def test_parse_args_left_click() -> None:
29
+ args = parse_arguments(["left_click", "--coordinate", "100", "200"])
30
+ assert args.action == "left_click"
31
+ assert args.coordinate == [100, 200]
32
+
33
+
34
+ def test_parse_args_right_click() -> None:
35
+ args = parse_arguments(["right_click", "--coordinate", "100", "200"])
36
+ assert args.action == "right_click"
37
+ assert args.coordinate == [100, 200]
38
+
39
+
40
+ def test_parse_args_middle_click() -> None:
41
+ args = parse_arguments(["middle_click", "--coordinate", "100", "200"])
42
+ assert args.action == "middle_click"
43
+ assert args.coordinate == [100, 200]
44
+
45
+
46
+ def test_parse_args_double_click() -> None:
47
+ args = parse_arguments(["double_click", "--coordinate", "100", "200"])
48
+ assert args.action == "double_click"
49
+ assert args.coordinate == [100, 200]
50
+
51
+
52
+ def test_parse_args_triple_click() -> None:
53
+ args = parse_arguments(["triple_click", "--coordinate", "100", "200"])
54
+ assert args.action == "triple_click"
55
+ assert args.coordinate == [100, 200]
56
+
57
+
58
+ def test_parse_args_hold_key() -> None:
59
+ args = parse_arguments(["hold_key", "--text", "a", "--duration", "5"])
60
+ assert args.action == "hold_key"
61
+ assert args.text == "a"
62
+ assert args.duration == 5
63
+
64
+
65
+ def test_parse_args_left_click_drag() -> None:
66
+ args = parse_arguments(
67
+ [
68
+ "left_click_drag",
69
+ "--start_coordinate",
70
+ "100",
71
+ "200",
72
+ "--coordinate",
73
+ "300",
74
+ "400",
75
+ "--text",
76
+ "drag",
77
+ ]
78
+ )
79
+ assert args.action == "left_click_drag"
80
+ assert args.start_coordinate == [100, 200]
81
+ assert args.coordinate == [300, 400]
82
+ assert args.text == "drag"
83
+
84
+
85
+ def test_parse_args_scroll() -> None:
86
+ args = parse_arguments(
87
+ [
88
+ "scroll",
89
+ "--scroll_direction",
90
+ "up",
91
+ "--scroll_amount",
92
+ "10",
93
+ "--coordinate",
94
+ "100",
95
+ "200",
96
+ ]
97
+ )
98
+ assert args.action == "scroll"
99
+ assert args.scroll_direction == "up"
100
+ assert args.scroll_amount == 10
101
+ assert args.coordinate == [100, 200]
102
+
103
+
104
+ def test_parse_args_wait() -> None:
105
+ args = parse_arguments(["wait", "--duration", "5"])
106
+ assert args.action == "wait"
107
+ assert args.duration == 5
108
+
109
+
110
+ def test_parse_args_type_missing_text() -> None:
111
+ with pytest.raises(SystemExit):
112
+ parse_arguments(["type"])
113
+
114
+
115
+ def test_parse_args_invalid_action() -> None:
116
+ with pytest.raises(SystemExit):
117
+ parse_arguments(["invalid_action"])
118
+
119
+
120
+ def test_parse_args_mouse_move_missing_coordinate() -> None:
121
+ with pytest.raises(SystemExit):
122
+ parse_arguments(["mouse_move"])
123
+
124
+
125
+ def test_parse_args_click_invalid_coordinate() -> None:
126
+ with pytest.raises(SystemExit):
127
+ parse_arguments(["left_click", "--coordinate", "100"])
128
+
129
+
130
+ def test_parse_args_hold_key_missing_duration() -> None:
131
+ with pytest.raises(SystemExit):
132
+ parse_arguments(["hold_key", "--text", "a"])
133
+
134
+
135
+ def test_parse_args_left_click_drag_missing_start_coordinate() -> None:
136
+ with pytest.raises(SystemExit):
137
+ parse_arguments(
138
+ ["left_click_drag", "--coordinate", "300", "400", "--text", "drag"]
139
+ )
140
+
141
+
142
+ def test_parse_args_scroll_missing_scroll_direction() -> None:
143
+ with pytest.raises(SystemExit):
144
+ parse_arguments(
145
+ ["scroll", "--scroll_amount", "10", "--coordinate", "100", "200"]
146
+ )
147
+
148
+
149
+ def test_parse_args_wait_missing_duration() -> None:
150
+ with pytest.raises(SystemExit):
151
+ parse_arguments(["wait"])
@@ -0,0 +1,8 @@
1
+ [MASTER]
2
+ ; R - Refactorings
3
+ ; C - Convention
4
+ ; W - Warning
5
+ ; E - Error
6
+ enable=C,R,W,E
7
+ disable=R0903,C0114,C0115,C0116,C0301,C0411,C1804,C1805,W0120,W0511,E0401,E1101,E0611,E1128
8
+ score=no
@@ -0,0 +1,24 @@
1
+ {
2
+ "version": "0.2.0",
3
+ "configurations": [
4
+ {
5
+ "type": "debugpy",
6
+ "request": "launch",
7
+ "name": "Debug Web Server",
8
+ "program": "${workspaceFolder}/web_server.py"
9
+ },
10
+ {
11
+ "type": "debugpy",
12
+ "request": "launch",
13
+ "name": "Debug Web Client interactive mode",
14
+ "program": "${workspaceFolder}/web_client.py"
15
+ },
16
+ {
17
+ "type": "debugpy",
18
+ "request": "launch",
19
+ "name": "Debug Web Client w/arguments",
20
+ "program": "${workspaceFolder}/web_client.py",
21
+ "args": ["${command:pickArgs}"]
22
+ }
23
+ ]
24
+ }
@@ -0,0 +1,25 @@
1
+ {
2
+ "cSpell.words": [
3
+ "activedescendant",
4
+ "describedby",
5
+ "domcontentloaded",
6
+ "figcaption",
7
+ "flowto",
8
+ "framenavigated",
9
+ "headful",
10
+ "idref",
11
+ "jsonrpcclient",
12
+ "jsonrpcserver",
13
+ "keepalive",
14
+ "keyshortcuts",
15
+ "labelfor",
16
+ "labelledby",
17
+ "labelwrapped",
18
+ "multiselectable",
19
+ "Rects",
20
+ "roledescription",
21
+ "rubyannotation",
22
+ "tablecaption",
23
+ "valuetext"
24
+ ]
25
+ }
@@ -8,16 +8,15 @@ RUN apt-get update
8
8
 
9
9
  RUN pip install --upgrade pip
10
10
 
11
+ RUN pip install playwright jsonrpcclient jsonrpcserver httpx aiohttp pillow pydantic tenacity
12
+
11
13
  # Install playwright
12
- RUN pip install playwright
13
14
  RUN playwright install
14
15
  RUN playwright install-deps
15
16
 
16
- # Install other dependancies
17
- RUN pip install dm-env-rpc pillow bs4 lxml
18
-
19
17
  # Copy Python files alongside the Dockerfile
20
- COPY *.py ./
18
+ COPY . .
21
19
 
22
20
  # Run the server
23
- CMD ["python3", "/app/web_browser/web_server.py"]
21
+ CMD ["python3", "/app/web_browser/web_server.py"]
22
+ # CMD ["tail", "-f", "/dev/null"]
@@ -1,7 +1,6 @@
1
1
  ## Headless Browser Tool
2
2
 
3
- This directory contains an implementation for the Headless Browser Tool which can be used to test web browsing agents.
4
-
3
+ This directory contains an implementation for the Headless Browser Tool which can be used to test web browsing agents.
5
4
 
6
5
  ### Usage
7
6
 
@@ -37,27 +36,27 @@ The result will be printed out in _stdout_ in the following format:
37
36
 
38
37
  ```
39
38
  # Inside the Docker container
40
- error: <an ERROR message if one occured>
39
+ error: <an ERROR message if one occurred>
41
40
  info: <general info about the container>
42
41
  web_url: <the URL of the page the browser is currently at>
43
42
  web_at: <accessibility tree of the visible elements of the page>
44
- ```
45
-
43
+ ```
46
44
 
47
45
  ### Design
48
46
 
49
47
  The following diagram describes the design and the intended usage of the tool:
50
48
 
51
- ![diagram](images/usage_diagram.png)
49
+ ![diagram](images/usage_diagram.svg)
52
50
 
53
51
  The tool consists of the following components:
54
52
 
55
- * [WebServer](web_server.py) - a server which launches a stateful session with the headless chromium browser and interracts with it through the [Playwright API](https://playwright.dev/python/docs/intro) upon receiving client commands. The server components are:
56
- * _dm_env_servicer.py_ - an implementation for the gRPC Service based on [dm_env_rpc protocol](https://github.com/google-deepmind/dm_env_rpc).
57
- * _web_environment.py_ - an environment which gets instantiated by the servicer and which launches the browser, stores its state and maps client commands to Playwright API.
58
- * _playwright_crawler.py_ - a wrapper over the sync Playwright API.
53
+ - [WebServer](web_server.py) - a server which launches a stateful session with the headless chromium browser and interacts with it through the [Playwright API](https://playwright.dev/python/docs/intro) upon receiving client commands. The server components are:
54
+
55
+ - _dm_env_servicer.py_ - an implementation for the gRPC Service based on [dm_env_rpc protocol](https://github.com/google-deepmind/dm_env_rpc).
56
+ - _web_environment.py_ - an environment which gets instantiated by the servicer and which launches the browser, stores its state and maps client commands to Playwright API.
57
+ - _playwright_crawler.py_ - a wrapper over the sync Playwright API.
59
58
 
60
- * [WebClient](web_client.py) - a simple stateless client to interact with the server. When launched, the client:
59
+ - [WebClient](web_client.py) - a simple stateless client to interact with the server. When launched, the client:
61
60
  1. creates a connection with the server;
62
61
  2. sends user command to the server;
63
62
  3. receives the response in the form of observations and prints them to stdout;
@@ -0,0 +1,71 @@
1
+ from functools import reduce
2
+ from typing import Iterable, TypedDict
3
+
4
+ from accessibility_tree_node import AccessibilityTreeNode
5
+ from cdp.a11y import AXNode, AXNodeId
6
+ from cdp.dom_snapshot import DOMSnapshot, create_snapshot_context
7
+ from rectangle import Rectangle
8
+
9
+ _AccType = tuple[
10
+ AXNode | None,
11
+ dict[AXNodeId, AXNode],
12
+ ]
13
+
14
+
15
+ class AccessibilityTree(TypedDict):
16
+ root: AccessibilityTreeNode
17
+ nodes: dict[AXNodeId, AccessibilityTreeNode]
18
+
19
+
20
+ def create_accessibility_tree(
21
+ *,
22
+ ax_nodes: Iterable[AXNode],
23
+ dom_snapshot: DOMSnapshot,
24
+ device_scale_factor: float,
25
+ window_bounds: Rectangle,
26
+ ) -> AccessibilityTree | None:
27
+ """
28
+ Creates an accessibility tree from the given Chrome DevTools Protocol AX nodes and DOM snapshot.
29
+
30
+ Args:
31
+ ax_nodes (Iterable[AXNode]): An iterable of AXNode objects representing the accessibility nodes.
32
+ dom_snapshot (DOMSnapshot): A snapshot of the DOM at the time of accessibility tree creation.
33
+ device_scale_factor (float): The scale factor of the device.
34
+ window_bounds (Bounds): The bounds of the window.
35
+
36
+ Returns:
37
+ AccessibilityTree: The accessibility tree.
38
+ """
39
+
40
+ # first make a dict of AXNodeId's to AXNode's and find the root on the way
41
+ def reducer(acc: _AccType, ax_node: AXNode) -> _AccType:
42
+ root_node, nodes = acc
43
+ nodes[ax_node.nodeId] = ax_node
44
+ return (
45
+ # TODO: What do we want for multiple roots?
46
+ root_node or (ax_node if ax_node.parentId is None else None),
47
+ nodes,
48
+ )
49
+
50
+ initial_acc: _AccType = (None, {}) # The inference engine is weak
51
+ root_node, nodes = reduce(reducer, ax_nodes, initial_acc)
52
+
53
+ if not root_node:
54
+ return None
55
+
56
+ # Now create the AccessibilityTreeNode hierarchy
57
+ snapshot_context = create_snapshot_context(dom_snapshot)
58
+ all_accessibility_tree_nodes: dict[AXNodeId, AccessibilityTreeNode] = {}
59
+
60
+ return AccessibilityTree(
61
+ root=AccessibilityTreeNode(
62
+ ax_node=root_node,
63
+ ax_nodes=nodes,
64
+ parent=None,
65
+ all_accessibility_tree_nodes=all_accessibility_tree_nodes,
66
+ snapshot_context=snapshot_context,
67
+ device_scale_factor=device_scale_factor,
68
+ window_bounds=window_bounds,
69
+ ),
70
+ nodes=all_accessibility_tree_nodes,
71
+ )
@@ -0,0 +1,323 @@
1
+ from functools import reduce
2
+ from typing import Optional
3
+
4
+ from cdp.a11y import (
5
+ AXNode,
6
+ AXNodeId,
7
+ AXProperty,
8
+ AXPropertyName,
9
+ node_has_property,
10
+ string_from_ax_value,
11
+ )
12
+ from cdp.dom_snapshot import (
13
+ DOMSnapshotContext,
14
+ bounds_for_node_index,
15
+ current_url_src_for_node_index,
16
+ index_for_node_id,
17
+ text_value_for_node_index,
18
+ )
19
+ from rectangle import Rectangle
20
+
21
+ # Properties to ignore when printing out the accessibility tree.
22
+ _IGNORED_AT_PROPERTIES: set[AXPropertyName] = {
23
+ "focusable",
24
+ "readonly",
25
+ "level",
26
+ "settable",
27
+ "multiline",
28
+ "invalid",
29
+ }
30
+
31
+ # The AXRole enum is here (https://chromium.googlesource.com/chromium/chromium/+/HEAD/ui/accessibility/ax_enums.idl#62)
32
+ _ROLES_IGNORED = {"horizontal_rule", "ignored"}
33
+
34
+ # These are noisy and provide no real benefit unless there's a name associated with them
35
+ _ROLES_REQUIRING_A_NAME = {"generic", "paragraph", "div"}
36
+
37
+ # Used for debugging, include the elements bounding rect during output.
38
+ _INCLUDE_BOUNDS_IN_OUTPUT = False
39
+
40
+
41
+ class AccessibilityTreeNode:
42
+ def __init__(
43
+ self,
44
+ *,
45
+ ax_node: AXNode,
46
+ ax_nodes: dict[AXNodeId, AXNode],
47
+ parent: Optional["AccessibilityTreeNode"],
48
+ all_accessibility_tree_nodes: dict[AXNodeId, "AccessibilityTreeNode"],
49
+ snapshot_context: DOMSnapshotContext,
50
+ device_scale_factor: float,
51
+ window_bounds: Rectangle,
52
+ ):
53
+ all_accessibility_tree_nodes[ax_node.nodeId] = self
54
+
55
+ self._is_expanded = node_has_property(ax_node, "expanded")
56
+ self._ax_node = ax_node
57
+ self._parent = parent
58
+ self._is_ignored: bool | None = None
59
+ self._bounds: Rectangle | None = None
60
+ self._is_visible: bool | None = None
61
+ self._current_src_url: str | None = None
62
+ self._input: str | None = None
63
+ self.__closest_non_ignored_parent: AccessibilityTreeNode | None = None
64
+
65
+ dom_node_index = (
66
+ index_for_node_id(ax_node.backendDOMNodeId, snapshot_context)
67
+ if ax_node.backendDOMNodeId
68
+ else None
69
+ )
70
+
71
+ # The following three variables (_bounds, _is_visible, and _current_src_url) depend on
72
+ # info from the DOM element associated with this AXNode
73
+ if dom_node_index:
74
+ self._bounds = (
75
+ Rectangle(*layout_bounds).scale(1 / device_scale_factor)
76
+ if (
77
+ layout_bounds := bounds_for_node_index(
78
+ dom_node_index, snapshot_context
79
+ )
80
+ )
81
+ is not None
82
+ else None
83
+ )
84
+ self._is_visible = (
85
+ (self._bounds.has_area and self._bounds.overlaps(window_bounds))
86
+ if self._bounds
87
+ else None
88
+ )
89
+ self._current_src_url = (
90
+ current_url_src_for_node_index(dom_node_index, snapshot_context)
91
+ if self.role == "image"
92
+ else None
93
+ )
94
+
95
+ if node_has_property(ax_node, "editable"):
96
+ # Sometimes a node will have it's input stored as 'value' other times we
97
+ # need to go looking through the DOM tree to find its matching string.
98
+ self._input = (
99
+ string_from_ax_value(ax_node.value)
100
+ if ax_node.value
101
+ else text_value_for_node_index(dom_node_index, snapshot_context)
102
+ if dom_node_index is not None
103
+ else None
104
+ )
105
+
106
+ # this is a little awkward, but the old code ensured that menuitems were visible in given the following condition
107
+ if self.role == "menuitem" and parent is not None and parent.is_expanded:
108
+ self._is_visible = True
109
+
110
+ # It's important to set all other properties before creating children in case
111
+ # the children sample parent.xxx
112
+ self.children = (
113
+ [
114
+ AccessibilityTreeNode(
115
+ ax_node=ax_nodes[child_id],
116
+ ax_nodes=ax_nodes,
117
+ parent=self,
118
+ all_accessibility_tree_nodes=all_accessibility_tree_nodes,
119
+ snapshot_context=snapshot_context,
120
+ device_scale_factor=device_scale_factor,
121
+ window_bounds=window_bounds,
122
+ )
123
+ for child_id in ax_node.childIds
124
+ ]
125
+ if ax_node.childIds
126
+ else None
127
+ )
128
+
129
+ def __str__(self) -> str:
130
+ bounds_string = str(self.bounds) if _INCLUDE_BOUNDS_IN_OUTPUT else None
131
+
132
+ input_string = f'Current input: "{self._input}"' if self._input else None
133
+
134
+ name_string = f'"{self.name}"' if self.name else None
135
+ # Without bounds we can't click on the element, and therefore the ID
136
+ # can not be used. Still useful to show these elements though (e.g.)
137
+ # to let user know about options.
138
+ id_string = f"[{'*' if not self._bounds or not self._bounds.has_area else self._ax_node.nodeId}]"
139
+ url_string = self._current_src_url if self._current_src_url else None
140
+
141
+ url_string = _maybe_redact_base64_url(self._current_src_url)
142
+
143
+ return " ".join(
144
+ [
145
+ item
146
+ for item in [
147
+ id_string,
148
+ self.role,
149
+ name_string,
150
+ url_string,
151
+ input_string,
152
+ self._property_string(),
153
+ bounds_string,
154
+ ]
155
+ if item is not None
156
+ ]
157
+ )
158
+
159
+ @property
160
+ def node_id(self) -> AXNodeId:
161
+ return self._ax_node.nodeId
162
+
163
+ @property
164
+ def name(self) -> str:
165
+ return string_from_ax_value(self._ax_node.name)
166
+
167
+ @property
168
+ def is_expanded(self) -> bool:
169
+ return self._is_expanded
170
+
171
+ @property
172
+ def is_visible(self) -> bool:
173
+ return self._is_visible or False
174
+
175
+ @property
176
+ def bounds(self) -> Rectangle | None:
177
+ return self._bounds
178
+
179
+ @property
180
+ def current_src_url(self) -> str | None:
181
+ return self._current_src_url
182
+
183
+ @property
184
+ def input(self) -> str | None:
185
+ return self._input
186
+
187
+ @property
188
+ def is_ignored(self) -> bool:
189
+ # computing _is_ignored is very expensive, do it on demand and remember the result
190
+ if self._is_ignored is None:
191
+ self._is_ignored = self._compute_is_ignored()
192
+ return self._is_ignored
193
+
194
+ @property
195
+ def role(self) -> str:
196
+ return string_from_ax_value(self._ax_node.role)
197
+
198
+ @property
199
+ def _closest_non_ignored_parent(self) -> Optional["AccessibilityTreeNode"]:
200
+ if self._parent and self.__closest_non_ignored_parent is None:
201
+ self.__closest_non_ignored_parent = (
202
+ None
203
+ if not self._parent
204
+ else self._parent
205
+ if not self._parent.is_ignored
206
+ else self._parent._closest_non_ignored_parent # pylint: disable=protected-access
207
+ )
208
+ return self.__closest_non_ignored_parent
209
+
210
+ def render_accessibility_tree(self, indent: int = 0) -> str:
211
+ """Returns the string representation of the node and its children."""
212
+ result = ""
213
+ if not self.is_ignored:
214
+ result = " " * indent + str(self) + "\n"
215
+ indent += 1
216
+ children_strings = [
217
+ child.render_accessibility_tree(indent) for child in self.children or []
218
+ ]
219
+ result = result + "".join([s for s in children_strings if s])
220
+ return result
221
+
222
+ def render_main_content(self) -> str | None:
223
+ return (
224
+ main_node._extract_text() # pylint: disable=protected-access
225
+ if (main_node := self._get_main_content_node())
226
+ else None
227
+ )
228
+
229
+ def _compute_is_ignored(self) -> bool:
230
+ if self._ax_node.ignored or not self.is_visible or self.role in _ROLES_IGNORED:
231
+ return True
232
+
233
+ # if the parent is a link/button with a name, and this node is within the parent's bounds, we can ignore it.
234
+ if (
235
+ (closest_parent := self._closest_non_ignored_parent)
236
+ and closest_parent.bounds
237
+ and self.bounds
238
+ and (closest_parent.role == "link" or closest_parent.role == "button")
239
+ and closest_parent.name
240
+ and self.bounds.within(closest_parent.bounds)
241
+ ):
242
+ return True
243
+
244
+ if bool(self.name): # excludes None or ""
245
+ # This simplifies things like links/buttons whose contents are sub-elements
246
+ # e.g.
247
+ # [158] button "Menu"
248
+ # [1166] StaticText "Menu"
249
+ # becomes
250
+ # [158] button "Menu"
251
+ return self._parent is not None and self._parent.name == self.name
252
+
253
+ return self.role in _ROLES_REQUIRING_A_NAME
254
+
255
+ def _get_main_content_node(self) -> Optional["AccessibilityTreeNode"]:
256
+ if self.role == "main":
257
+ return self
258
+
259
+ return next(
260
+ (
261
+ result
262
+ for child in self.children or []
263
+ if (result := child._get_main_content_node()) # pylint: disable=protected-access
264
+ ),
265
+ None,
266
+ )
267
+
268
+ def _extract_text(self):
269
+ """
270
+ Recursively extracts and concatenates text from the accessibility tree nodes.
271
+
272
+ This method traverses the accessibility tree starting from the current node,
273
+ concatenating the text of each node.
274
+
275
+ Returns:
276
+ str: The concatenated text from the accessibility tree nodes, with
277
+ appropriate spacing based on node roles.
278
+ """
279
+ if not self.is_visible:
280
+ return ""
281
+
282
+ def reducer(acc, child):
283
+ child_text = child._extract_text() # pylint: disable=protected-access
284
+ return (
285
+ acc
286
+ + (
287
+ "\n\n"
288
+ if child.role
289
+ in {"paragraph", "note", "row", "div", "heading", "region"}
290
+ else " "
291
+ )
292
+ + child_text
293
+ )
294
+
295
+ return reduce(reducer, self.children or [], self.name).strip()
296
+
297
+ def _property_string(self) -> str:
298
+ properties = [
299
+ _prop_name_value_str(node_property)
300
+ if node_property.value.value
301
+ else f"{node_property.name}"
302
+ for node_property in self._ax_node.properties or ()
303
+ if node_property.name not in _IGNORED_AT_PROPERTIES
304
+ ]
305
+ return " [" + ", ".join(properties) + "]" if properties else ""
306
+
307
+
308
+ def _prop_name_value_str(node_property: AXProperty) -> str:
309
+ # pre-commit hook (debug-statements) crashes if value is inlined below
310
+ value = (
311
+ _maybe_redact_base64_url(node_property.value.value)
312
+ if node_property.name == "url" and isinstance(node_property.value.value, str)
313
+ else node_property.value.value
314
+ )
315
+ return f"{node_property.name}: {value}"
316
+
317
+
318
+ def _maybe_redact_base64_url(url: str | None) -> str | None:
319
+ return (
320
+ "data:<base64-data-removed>"
321
+ if url and url.startswith("data:") and "base64" in url
322
+ else url
323
+ )
@@ -0,0 +1,5 @@
1
+ """
2
+ Types and pure functional helpers associated with Chrome DevTools Protocol
3
+
4
+ https://chromedevtools.github.io/devtools-protocol/
5
+ """