inspect-ai 0.3.70__py3-none-any.whl → 0.3.71__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. inspect_ai/_cli/eval.py +14 -8
  2. inspect_ai/_display/core/display.py +2 -0
  3. inspect_ai/_display/core/footer.py +13 -3
  4. inspect_ai/_display/plain/display.py +6 -2
  5. inspect_ai/_display/rich/display.py +19 -6
  6. inspect_ai/_display/textual/app.py +6 -1
  7. inspect_ai/_display/textual/display.py +4 -0
  8. inspect_ai/_display/textual/widgets/transcript.py +10 -6
  9. inspect_ai/_eval/task/run.py +5 -8
  10. inspect_ai/_util/content.py +20 -1
  11. inspect_ai/_util/transcript.py +10 -4
  12. inspect_ai/_util/working.py +4 -0
  13. inspect_ai/_view/www/App.css +6 -0
  14. inspect_ai/_view/www/dist/assets/index.css +115 -87
  15. inspect_ai/_view/www/dist/assets/index.js +5324 -2276
  16. inspect_ai/_view/www/eslint.config.mjs +24 -1
  17. inspect_ai/_view/www/log-schema.json +283 -20
  18. inspect_ai/_view/www/package.json +8 -3
  19. inspect_ai/_view/www/src/App.tsx +2 -2
  20. inspect_ai/_view/www/src/components/AnsiDisplay.tsx +4 -3
  21. inspect_ai/_view/www/src/components/Card.tsx +9 -8
  22. inspect_ai/_view/www/src/components/DownloadButton.tsx +2 -1
  23. inspect_ai/_view/www/src/components/EmptyPanel.tsx +2 -2
  24. inspect_ai/_view/www/src/components/ErrorPanel.tsx +4 -3
  25. inspect_ai/_view/www/src/components/ExpandablePanel.tsx +13 -5
  26. inspect_ai/_view/www/src/components/FindBand.tsx +3 -3
  27. inspect_ai/_view/www/src/components/HumanBaselineView.tsx +3 -3
  28. inspect_ai/_view/www/src/components/LabeledValue.tsx +5 -4
  29. inspect_ai/_view/www/src/components/LargeModal.tsx +18 -13
  30. inspect_ai/_view/www/src/components/{LightboxCarousel.css → LightboxCarousel.module.css} +22 -18
  31. inspect_ai/_view/www/src/components/LightboxCarousel.tsx +36 -27
  32. inspect_ai/_view/www/src/components/MessageBand.tsx +2 -1
  33. inspect_ai/_view/www/src/components/NavPills.tsx +9 -8
  34. inspect_ai/_view/www/src/components/ProgressBar.tsx +2 -1
  35. inspect_ai/_view/www/src/components/TabSet.tsx +21 -15
  36. inspect_ai/_view/www/src/index.tsx +2 -2
  37. inspect_ai/_view/www/src/metadata/MetaDataGrid.tsx +11 -9
  38. inspect_ai/_view/www/src/metadata/MetaDataView.tsx +3 -2
  39. inspect_ai/_view/www/src/metadata/MetadataGrid.module.css +1 -0
  40. inspect_ai/_view/www/src/metadata/RenderedContent.tsx +16 -0
  41. inspect_ai/_view/www/src/plan/DatasetDetailView.tsx +3 -2
  42. inspect_ai/_view/www/src/plan/DetailStep.tsx +2 -1
  43. inspect_ai/_view/www/src/plan/PlanCard.tsx +2 -5
  44. inspect_ai/_view/www/src/plan/PlanDetailView.tsx +6 -9
  45. inspect_ai/_view/www/src/plan/ScorerDetailView.tsx +2 -1
  46. inspect_ai/_view/www/src/plan/SolverDetailView.tsx +3 -3
  47. inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx +2 -2
  48. inspect_ai/_view/www/src/samples/SampleDialog.tsx +3 -3
  49. inspect_ai/_view/www/src/samples/SampleDisplay.tsx +2 -2
  50. inspect_ai/_view/www/src/samples/SampleSummaryView.tsx +2 -2
  51. inspect_ai/_view/www/src/samples/SamplesTools.tsx +2 -1
  52. inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx +3 -19
  53. inspect_ai/_view/www/src/samples/chat/ChatMessageRenderer.tsx +2 -1
  54. inspect_ai/_view/www/src/samples/chat/ChatMessageRow.tsx +2 -1
  55. inspect_ai/_view/www/src/samples/chat/ChatView.tsx +2 -1
  56. inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.tsx +22 -7
  57. inspect_ai/_view/www/src/samples/chat/MessageContent.tsx +35 -6
  58. inspect_ai/_view/www/src/samples/chat/MessageContents.tsx +2 -2
  59. inspect_ai/_view/www/src/samples/chat/messages.ts +15 -2
  60. inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.tsx +13 -4
  61. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.module.css +2 -2
  62. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.tsx +18 -19
  63. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.module.css +1 -1
  64. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.tsx +4 -3
  65. inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.tsx +2 -2
  66. inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.tsx +2 -3
  67. inspect_ai/_view/www/src/samples/error/SampleErrorView.tsx +3 -2
  68. inspect_ai/_view/www/src/samples/list/SampleFooter.tsx +2 -1
  69. inspect_ai/_view/www/src/samples/list/SampleHeader.tsx +2 -1
  70. inspect_ai/_view/www/src/samples/list/SampleList.tsx +57 -45
  71. inspect_ai/_view/www/src/samples/list/SampleRow.tsx +2 -1
  72. inspect_ai/_view/www/src/samples/list/SampleSeparator.tsx +2 -1
  73. inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.tsx +2 -2
  74. inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx +4 -3
  75. inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx +2 -5
  76. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx +2 -2
  77. inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx +2 -1
  78. inspect_ai/_view/www/src/samples/scores/SampleScores.tsx +2 -2
  79. inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.tsx +2 -1
  80. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx +2 -1
  81. inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx +2 -1
  82. inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx +2 -1
  83. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.module.css +4 -0
  84. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.tsx +12 -2
  85. inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css +1 -1
  86. inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx +25 -28
  87. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.tsx +2 -1
  88. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx +5 -4
  89. inspect_ai/_view/www/src/samples/transcript/SampleTranscript.tsx +2 -2
  90. inspect_ai/_view/www/src/samples/transcript/SandboxEventView.tsx +8 -7
  91. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.tsx +2 -2
  92. inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx +3 -3
  93. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.tsx +18 -14
  94. inspect_ai/_view/www/src/samples/transcript/ToolEventView.tsx +5 -5
  95. inspect_ai/_view/www/src/samples/transcript/TranscriptView.tsx +34 -15
  96. inspect_ai/_view/www/src/samples/transcript/event/EventNav.tsx +2 -1
  97. inspect_ai/_view/www/src/samples/transcript/event/EventNavs.tsx +2 -1
  98. inspect_ai/_view/www/src/samples/transcript/event/EventRow.tsx +3 -2
  99. inspect_ai/_view/www/src/samples/transcript/event/EventSection.tsx +2 -2
  100. inspect_ai/_view/www/src/samples/transcript/event/EventTimingPanel.module.css +28 -0
  101. inspect_ai/_view/www/src/samples/transcript/event/EventTimingPanel.tsx +115 -0
  102. inspect_ai/_view/www/src/samples/transcript/event/utils.ts +29 -0
  103. inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.tsx +2 -1
  104. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx +3 -3
  105. inspect_ai/_view/www/src/samples/transcript/state/StateEventView.tsx +11 -8
  106. inspect_ai/_view/www/src/types/log.d.ts +129 -34
  107. inspect_ai/_view/www/src/usage/ModelTokenTable.tsx +6 -10
  108. inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css +4 -0
  109. inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx +32 -9
  110. inspect_ai/_view/www/src/usage/TokenTable.tsx +4 -6
  111. inspect_ai/_view/www/src/usage/UsageCard.tsx +2 -1
  112. inspect_ai/_view/www/src/utils/format.ts +1 -1
  113. inspect_ai/_view/www/src/utils/json.ts +24 -0
  114. inspect_ai/_view/www/src/workspace/WorkSpace.tsx +6 -5
  115. inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx +9 -2
  116. inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.tsx +2 -1
  117. inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx +2 -1
  118. inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx +3 -3
  119. inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx +4 -3
  120. inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx +5 -4
  121. inspect_ai/_view/www/src/workspace/navbar/StatusPanel.tsx +5 -8
  122. inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.tsx +5 -4
  123. inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.tsx +2 -1
  124. inspect_ai/_view/www/src/workspace/sidebar/Sidebar.tsx +2 -1
  125. inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.tsx +2 -2
  126. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx +2 -1
  127. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx +2 -2
  128. inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx +2 -2
  129. inspect_ai/_view/www/src/workspace/tabs/JsonTab.tsx +2 -5
  130. inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx +12 -11
  131. inspect_ai/_view/www/yarn.lock +241 -5
  132. inspect_ai/log/_condense.py +3 -0
  133. inspect_ai/log/_recorders/eval.py +6 -1
  134. inspect_ai/log/_transcript.py +58 -1
  135. inspect_ai/model/__init__.py +2 -0
  136. inspect_ai/model/_call_tools.py +7 -0
  137. inspect_ai/model/_chat_message.py +22 -7
  138. inspect_ai/model/_conversation.py +10 -8
  139. inspect_ai/model/_generate_config.py +25 -4
  140. inspect_ai/model/_model.py +133 -57
  141. inspect_ai/model/_model_output.py +3 -0
  142. inspect_ai/model/_openai.py +106 -40
  143. inspect_ai/model/_providers/anthropic.py +134 -26
  144. inspect_ai/model/_providers/google.py +27 -8
  145. inspect_ai/model/_providers/groq.py +9 -4
  146. inspect_ai/model/_providers/openai.py +57 -4
  147. inspect_ai/model/_providers/openai_o1.py +10 -0
  148. inspect_ai/model/_providers/providers.py +1 -1
  149. inspect_ai/model/_reasoning.py +15 -2
  150. inspect_ai/scorer/_model.py +23 -19
  151. inspect_ai/solver/_human_agent/agent.py +14 -10
  152. inspect_ai/solver/_human_agent/commands/__init__.py +7 -3
  153. inspect_ai/solver/_human_agent/commands/submit.py +76 -30
  154. inspect_ai/tool/__init__.py +2 -0
  155. inspect_ai/tool/_tool.py +3 -1
  156. inspect_ai/tool/_tools/_computer/_resources/tool/_run.py +1 -1
  157. inspect_ai/tool/_tools/_web_browser/_resources/.pylintrc +8 -0
  158. inspect_ai/tool/_tools/_web_browser/_resources/.vscode/launch.json +24 -0
  159. inspect_ai/tool/_tools/_web_browser/_resources/.vscode/settings.json +25 -0
  160. inspect_ai/tool/_tools/_web_browser/_resources/Dockerfile +5 -6
  161. inspect_ai/tool/_tools/_web_browser/_resources/README.md +10 -11
  162. inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree.py +71 -0
  163. inspect_ai/tool/_tools/_web_browser/_resources/accessibility_tree_node.py +323 -0
  164. inspect_ai/tool/_tools/_web_browser/_resources/cdp/__init__.py +5 -0
  165. inspect_ai/tool/_tools/_web_browser/_resources/cdp/a11y.py +279 -0
  166. inspect_ai/tool/_tools/_web_browser/_resources/cdp/dom.py +9 -0
  167. inspect_ai/tool/_tools/_web_browser/_resources/cdp/dom_snapshot.py +293 -0
  168. inspect_ai/tool/_tools/_web_browser/_resources/cdp/page.py +94 -0
  169. inspect_ai/tool/_tools/_web_browser/_resources/constants.py +2 -0
  170. inspect_ai/tool/_tools/_web_browser/_resources/images/usage_diagram.svg +2 -0
  171. inspect_ai/tool/_tools/_web_browser/_resources/playwright_browser.py +50 -0
  172. inspect_ai/tool/_tools/_web_browser/_resources/playwright_crawler.py +31 -359
  173. inspect_ai/tool/_tools/_web_browser/_resources/playwright_page_crawler.py +280 -0
  174. inspect_ai/tool/_tools/_web_browser/_resources/pyproject.toml +65 -0
  175. inspect_ai/tool/_tools/_web_browser/_resources/rectangle.py +64 -0
  176. inspect_ai/tool/_tools/_web_browser/_resources/rpc_client_helpers.py +146 -0
  177. inspect_ai/tool/_tools/_web_browser/_resources/scale_factor.py +64 -0
  178. inspect_ai/tool/_tools/_web_browser/_resources/test_accessibility_tree_node.py +180 -0
  179. inspect_ai/tool/_tools/_web_browser/_resources/test_playwright_crawler.py +15 -9
  180. inspect_ai/tool/_tools/_web_browser/_resources/test_rectangle.py +15 -0
  181. inspect_ai/tool/_tools/_web_browser/_resources/test_web_client.py +44 -0
  182. inspect_ai/tool/_tools/_web_browser/_resources/web_browser_rpc_types.py +39 -0
  183. inspect_ai/tool/_tools/_web_browser/_resources/web_client.py +198 -48
  184. inspect_ai/tool/_tools/_web_browser/_resources/web_client_new_session.py +26 -25
  185. inspect_ai/tool/_tools/_web_browser/_resources/web_server.py +178 -39
  186. inspect_ai/tool/_tools/_web_browser/_web_browser.py +38 -19
  187. inspect_ai/util/__init__.py +2 -1
  188. inspect_ai/util/_display.py +12 -0
  189. inspect_ai/util/_sandbox/events.py +55 -21
  190. inspect_ai/util/_sandbox/self_check.py +131 -43
  191. inspect_ai/util/_subtask.py +11 -0
  192. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.71.dist-info}/METADATA +1 -1
  193. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.71.dist-info}/RECORD +197 -182
  194. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.71.dist-info}/WHEEL +1 -1
  195. inspect_ai/_view/www/node_modules/flatted/python/flatted.py +0 -149
  196. inspect_ai/_view/www/node_modules/flatted/python/test.py +0 -63
  197. inspect_ai/_view/www/src/components/VirtualList.module.css +0 -19
  198. inspect_ai/_view/www/src/components/VirtualList.tsx +0 -292
  199. inspect_ai/tool/_tools/_web_browser/_resources/accessibility_node.py +0 -312
  200. inspect_ai/tool/_tools/_web_browser/_resources/dm_env_servicer.py +0 -275
  201. inspect_ai/tool/_tools/_web_browser/_resources/images/usage_diagram.png +0 -0
  202. inspect_ai/tool/_tools/_web_browser/_resources/test_accessibility_node.py +0 -176
  203. inspect_ai/tool/_tools/_web_browser/_resources/test_dm_env_servicer.py +0 -135
  204. inspect_ai/tool/_tools/_web_browser/_resources/test_web_environment.py +0 -71
  205. inspect_ai/tool/_tools/_web_browser/_resources/web_environment.py +0 -184
  206. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.71.dist-info}/LICENSE +0 -0
  207. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.71.dist-info}/entry_points.txt +0 -0
  208. {inspect_ai-0.3.70.dist-info → inspect_ai-0.3.71.dist-info}/top_level.txt +0 -0
@@ -5,372 +5,44 @@ Largely based on https://github.com/web-arena-x/webarena
5
5
 
6
6
  from __future__ import annotations
7
7
 
8
- import enum
9
- import logging
10
- import re
11
- import time
12
- from os import getenv
13
- from typing import Any, Literal
8
+ from asyncio.futures import Future
14
9
 
15
- import accessibility_node as at
16
- from playwright.sync_api import sync_playwright
10
+ from playwright.async_api import BrowserContext, Page
17
11
 
18
- # Number of seconds to wait for page to load before processing it.
19
- WAIT_FOR_PAGE_TIME = 2.0
20
-
21
- # The waiting strategy to use between browser commands.
22
- # see https://playwright.dev/docs/api/class-page.
23
- WAIT_STRATEGY = "domcontentloaded"
24
-
25
-
26
- class CrawlerOutputFormat(enum.Enum):
27
- # Raw HTML.
28
- HTML = 0
29
- # Raw Document Object Model.
30
- DOM = 1
31
- # Accessibility tree.
32
- AT = 2
33
- # A pixel-based rending of the webpage.
34
- PIXELS = 3
35
-
36
-
37
- class PlaywrightBrowser:
38
- """Stores the browser and creates new contexts."""
39
-
40
- WIDTH = 1280
41
- HEIGHT = 1080
42
- _playwright_api = None
43
-
44
- def __init__(self):
45
- """Creates the browser."""
46
- if PlaywrightBrowser._playwright_api is None:
47
- PlaywrightBrowser._playwright_api = sync_playwright().start()
48
-
49
- logging.info("Starting chromium in headless mode.")
50
-
51
- self._browser = PlaywrightBrowser._playwright_api.chromium.launch(
52
- headless=True,
53
- # Required for Gmail signup see
54
- # https://stackoverflow.com/questions/65139098/how-to-login-to-google-account-with-playwright
55
- args=["--disable-blink-features=AutomationControlled"],
56
- )
57
-
58
- def get_new_context(self):
59
- return self._browser.new_context(
60
- geolocation={"longitude": -0.12, "latitude": 51},
61
- locale="en-GB",
62
- permissions=["geolocation"],
63
- timezone_id="Europe/London",
64
- viewport={"width": self.WIDTH, "height": self.HEIGHT},
65
- ignore_https_errors=getenv("IGNORE_HTTPS_ERRORS", "") != "",
66
- )
67
-
68
- def close(self):
69
- self._browser.close()
70
- if PlaywrightBrowser._playwright_api is not None:
71
- PlaywrightBrowser._playwright_api.stop()
72
- PlaywrightBrowser._playwright_api = None
12
+ from playwright_page_crawler import PageCrawler
73
13
 
74
14
 
75
15
  class PlaywrightCrawler:
76
- """Stores the accessibility tree."""
77
-
78
- def __init__(self, browser_context):
79
- """Initialize the craweler."""
80
- self._context = browser_context
81
-
82
- self._page = None
83
- self._client = None
84
- self._root = None
85
- self._nodes = {}
86
- self._dom_tree = None
87
-
88
- self._initialize_context()
89
-
90
- def _initialize_context(self):
91
- """Creates playwright page, and client."""
92
- if self._page:
93
- # Close the previous page if it was open.
94
- self._page.close()
95
-
96
- self._page = self._context.new_page()
97
-
98
- # Enable chrome development tools, and accessabiltiy tree output.
99
- self._client = self._page.context.new_cdp_session(self._page)
100
- self._client.send("Accessibility.enable")
101
-
102
- # Start with an empty accessibility tree and DOM.
103
- self._root = None
104
- self._nodes = {}
105
- self._dom_tree = None
106
-
107
- def lookup_node(
108
- self, node_id_or_tag: int | str, include_ignored: bool = False
109
- ) -> at.AccessibilityNode:
110
- """Looks up the node by id or tag.
111
-
112
- Args:
113
- node_id_or_tag: Either the id number (as int or str), or <tag_name>
114
- include_ignored: If true will also lookup ignored (hidden) node.
115
-
116
- Returns:
117
- Node.
118
-
119
- Raise:
120
- Value error if node is not matched.
121
- """
122
- if re.match("^<.*>", str(node_id_or_tag)):
123
- tag = node_id_or_tag[1:-1]
124
- # This is a smart tag, try to resolve it.
125
- for node in self._nodes.values():
126
- # We match on anything that starts with the code, this is potentially
127
- # a little brittle, can be replaced with an RE if there are issues.
128
- if node.name.lower().startswith(tag.lower()):
129
- if not node.is_ignored or include_ignored:
130
- return node
131
- else:
132
- raise ValueError(
133
- f"Could not find tag {node_id_or_tag} from"
134
- + f" {[node.name for node in self._nodes.values() if node.name]}"
135
- )
136
- else:
137
- node_id = str(node_id_or_tag)
138
- node = self._nodes.get(node_id, None)
139
- if node and (include_ignored or not node.is_ignored):
140
- return node
141
- else:
142
- raise ValueError(f"Could not find element with id {node_id}")
143
-
144
- def update(self):
145
- """Updates the accessibility tree and DOM from current page."""
146
- # Wait for page to load.
147
- self._page.wait_for_load_state(WAIT_STRATEGY)
148
- time.sleep(WAIT_FOR_PAGE_TIME)
149
-
150
- # Get the DOM
151
- self._dom_tree = self._client.send(
152
- "DOMSnapshot.captureSnapshot",
153
- {
154
- "computedStyles": [],
155
- "includeDOMRects": True,
156
- "includePaintOrder": True,
157
- },
158
- )
159
-
160
- at_nodes = self._client.send("Accessibility.getFullAXTree", {})["nodes"]
161
-
162
- document = self._dom_tree["documents"][0]
163
- nodes = document["nodes"]
164
- layouts = document["layout"]
165
- srcs = nodes["currentSourceURL"]
166
- backendnode_ids = nodes["backendNodeId"]
167
- strings = self._dom_tree["strings"]
168
-
169
- text_value = nodes["textValue"]
170
-
171
- # Check current screen bounds.
172
- win_upper_bound = self._page.evaluate("window.pageYOffset")
173
- win_left_bound = self._page.evaluate("window.pageXOffset")
174
- win_width = self._page.evaluate("window.screen.width")
175
- win_height = self._page.evaluate("window.screen.height")
176
- window_bounds = at.NodeBounds(
177
- win_left_bound, win_upper_bound, win_width, win_height
16
+ @classmethod
17
+ async def create(
18
+ cls, browser_context: BrowserContext, device_scale_factor: float | None = None
19
+ ) -> PlaywrightCrawler:
20
+ page_crawler = await PageCrawler.create(
21
+ await browser_context.new_page(), device_scale_factor
178
22
  )
179
23
 
180
- # We have ids for the DOM and for the AT, and need to map between them.
181
- dom_to_at = {}
182
-
183
- # Build the AT tree.
184
- self._nodes: dict[str, at.AccessibilityNode] = {}
185
- self._root = None
186
- for at_node in at_nodes:
187
- node = at.AccessibilityNode(at_node)
188
- self._nodes[node.node_id] = node
189
- if not node.is_ignored:
190
- self._root = self._root or node
191
- dom_to_at[node.dom_id] = node.node_id
192
- # Default node to invisible. For it to be made visible it must turn up in
193
- # the layout below.
194
- node["is_visible"] = False
195
- # Also keep track of any AT nodes that did not show up in the DOM tree.
196
- node["is_matched"] = False
24
+ return PlaywrightCrawler(browser_context, page_crawler, device_scale_factor)
197
25
 
198
- # Create a lookup so that the matching below is not O(n^2)
199
- backend_index_lookup = {idx: index for index, idx in enumerate(backendnode_ids)}
200
-
201
- layout_from_backend = {}
202
- for i, key in enumerate(layouts["nodeIndex"]):
203
- layout_from_backend[key] = {
204
- k: layouts[k][i]
205
- for k in layouts.keys()
206
- if len(layouts[k]) == len(layouts["nodeIndex"])
207
- }
208
-
209
- text_from_backend = {
210
- index: value
211
- for index, value in zip(text_value["index"], text_value["value"])
212
- }
213
-
214
- src_from_backend = {
215
- index: value for index, value in zip(srcs["index"], srcs["value"])
216
- }
217
-
218
- # Set element bounds and visibility.
219
- # To do this we first lookup the position of an AT node's dom_id in the
220
- # backendNodeIds. We then lookup this index in layout_node_index, which
221
- # gives us the "index of the index" which is what we need to find the bounds
222
- # of this element.
223
- for node in self._nodes.values():
224
- if node.dom_id not in backend_index_lookup:
225
- # Sometimes we can not match, but that's fine, just ignore.
226
- node["is_matched"] = False
227
- continue
228
-
229
- backend_index = backend_index_lookup[node.dom_id]
230
-
231
- if src := src_from_backend.get(backend_index, None):
232
- node["src"] = strings[src]
233
-
234
- layout = layout_from_backend.get(backend_index, None)
235
- if layout:
236
- node.bounds = at.NodeBounds(*layout["bounds"])
237
- used_bounds = (
238
- node.get_union_bounds() if at.USE_UNION_BOUNDS else node.bounds
239
- )
240
- has_bounds = used_bounds.area > 0
241
- on_screen = used_bounds.overlaps(window_bounds)
242
- node["is_visible"] = has_bounds and on_screen
243
- else:
244
- node["is_visible"] = False
245
-
246
- node["is_matched"] = True
247
-
248
- # For nodes that are editable also record their current input text.
249
- for node in filter(lambda x: x.is_editable, self._nodes.values()):
250
- # Sometimes a node will have it's input stored as 'value' othertimes we
251
- # need to go looking through the DOM tree to find its matching string.
252
-
253
- if node.value:
254
- node["input"] = node.value
255
- else:
256
- if node.dom_id not in backend_index_lookup:
257
- # Sometimes web_at nodes don't appear in the DOM for some reason.
258
- continue
259
- backend_index = backend_index_lookup[node.dom_id]
260
- text_index = text_from_backend.get(backend_index, -1)
261
- if text_index >= 0:
262
- node["input"] = strings[text_index]
263
-
264
- # Map AT children and parents.
265
- for node in self._nodes.values():
266
- node.link_children(self._nodes)
267
-
268
- # Make menuitem's visible
269
- for node in [node for node in self._nodes.values() if node.role == "menuitem"]:
270
- node["is_visible"] = (
271
- node.role == "menuitem"
272
- and node.parent is not None
273
- and node.parent.is_expanded
274
- )
275
-
276
- def render(self, output_format: CrawlerOutputFormat) -> Any:
277
- """Returns the current webpage in the desired format.
278
-
279
- Only elements visible on the screen will be rendered.
280
-
281
- Args:
282
- output_format: The rending format to output.
283
-
284
- Returns:
285
- the currently active webpage rendered using given format.
286
- """
287
- match output_format:
288
- case CrawlerOutputFormat.AT:
289
- return self._render_at()
290
- case _:
291
- # TODO: Implement DOM, HTML, PIXELS formats
292
- raise NotImplementedError(
293
- "Playwright crawler does not currently support"
294
- f" {output_format} output."
295
- )
296
-
297
- def _render_at(self) -> str:
298
- """Render the current page's accessibility tree to text."""
299
- if self._root is None:
300
- return "<empty>"
301
- return self._root.to_string()
302
-
303
- def go_to_page(self, url: str) -> None:
304
- """Goes to the given url.
305
-
306
- Args:
307
- url: The url to redirect crawler to.
308
- """
309
- if "://" not in url:
310
- url = f"https://{url}"
311
- self._page.goto(url, wait_until=WAIT_STRATEGY)
312
-
313
- def click(self, element_id: int | str) -> None:
314
- """Clicks the element with the given id.
315
-
316
- Args:
317
- element_id: The id for the element we want to click on.
318
- """
319
- element = self.lookup_node(element_id)
320
- # Mouse.click() requires coordinates relative to the viewport:
321
- # https://playwright.dev/python/docs/api/class-mouse#mouse-click,
322
- # thus adjusting the Y coordinate since we only scroll up/down.
323
- scroll_y = self._page.evaluate("window.scrollY")
324
- self._page.mouse.click(
325
- element.bounds.center_x, element.bounds.center_y - scroll_y
326
- )
327
-
328
- def clear(self, element_id: int) -> None:
329
- """Clears text within a field."""
330
- self.click(element_id)
331
- self._page.keyboard.press("Control+A")
332
- self._page.keyboard.press("Backspace")
333
-
334
- def type(self, element_id: int | str, text: str) -> None:
335
- """Types into the element with the given id."""
336
- self.click(element_id)
337
- self._page.keyboard.type(text)
338
-
339
- def scroll(self, direction: Literal["up", "down"]) -> None:
340
- """Scrolls the page to the given direction.
341
-
342
- Args:
343
- direction: The direction to scroll in ('up' or 'down')
344
- """
345
- match direction.lower():
346
- case "up":
347
- self._page.evaluate(
348
- "(document.scrollingElement || document.body).scrollTop ="
349
- " (document.scrollingElement || document.body).scrollTop -"
350
- " window.innerHeight;"
351
- )
352
- case "down":
353
- self._page.evaluate(
354
- "(document.scrollingElement || document.body).scrollTop ="
355
- " (document.scrollingElement || document.body).scrollTop +"
356
- " window.innerHeight;"
357
- )
358
-
359
- case _:
360
- raise ValueError(f"Invalid scroll direction {direction}")
361
-
362
- def forward(self) -> None:
363
- """Move browser forward one history step."""
364
- self._page.go_forward(wait_until=WAIT_STRATEGY)
365
-
366
- def back(self) -> None:
367
- """Move browser backward one history step."""
368
- self._page.go_back(wait_until=WAIT_STRATEGY)
369
-
370
- def refresh(self) -> None:
371
- """Refresh (reload) the page."""
372
- self._page.reload(wait_until=WAIT_STRATEGY)
26
+ def __init__(
27
+ self,
28
+ browser_context: BrowserContext,
29
+ page_crawler: PageCrawler,
30
+ device_scale_factor: float | None,
31
+ ):
32
+ self._device_scale_factor = device_scale_factor
33
+ self._page_future = Future[PageCrawler]()
34
+ self._page_future.set_result(page_crawler)
35
+ browser_context.on("page", self._on_page)
373
36
 
374
37
  @property
375
- def url(self) -> str:
376
- return self._page.url
38
+ async def current_page(self) -> PageCrawler:
39
+ return await self._page_future
40
+
41
+ async def _on_page(self, new_page: Page):
42
+ # we know we're switching pages, but it will take time to get the new page crawler, so
43
+ # reset the future to force new callers to wait.
44
+ # TODO: A race remains in the case that we get multiple on_pages before the first one sets the result
45
+ self._page_future = Future()
46
+ self._page_future.set_result(
47
+ await PageCrawler.create(new_page, self._device_scale_factor)
48
+ )
@@ -0,0 +1,280 @@
1
+ """A crawler implementation using Playwright.
2
+
3
+ Portions based on https://github.com/web-arena-x/webarena
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import asyncio
9
+ import re
10
+ from typing import Literal
11
+
12
+ from playwright.async_api import CDPSession, Page
13
+
14
+ from accessibility_tree import AccessibilityTree, create_accessibility_tree
15
+ from accessibility_tree_node import AccessibilityTreeNode
16
+ from cdp.a11y import AXNodeId, AXTree
17
+ from cdp.dom_snapshot import DOMSnapshot
18
+ from rectangle import Rectangle
19
+
20
+ # Number of seconds to wait for possible click induced navigation before proceeding
21
+ _WAIT_FOR_NAVIGATION_TIME = 2.0
22
+
23
+ # The waiting strategy to use between browser commands.
24
+ # see https://playwright.dev/docs/api/class-page.
25
+ _WAIT_STRATEGY: Literal["domcontentloaded"] = "domcontentloaded"
26
+
27
+
28
+ class PageCrawler:
29
+ @classmethod
30
+ async def create(
31
+ cls, page: Page, device_scale_factor: float | None = None
32
+ ) -> PageCrawler:
33
+ # Enable chrome development tools, and accessibility tree output.
34
+ cdp_session = await page.context.new_cdp_session(page)
35
+ await cdp_session.send("Accessibility.enable")
36
+ return PageCrawler(
37
+ page,
38
+ cdp_session,
39
+ device_scale_factor or await page.evaluate("window.devicePixelRatio"),
40
+ )
41
+
42
+ def __init__(
43
+ self, page: Page, cdp_session: CDPSession, device_scale_factor: float
44
+ ) -> None:
45
+ self._page = page
46
+ self._cdp_session = cdp_session
47
+
48
+ # Start with an empty accessibility tree
49
+ self._rendered_main_content: str | None = None
50
+ self._rendered_accessibility_tree: str = ""
51
+ self._accessibility_tree: AccessibilityTree | None = None
52
+ self._device_scale_factor = device_scale_factor
53
+
54
+ @property
55
+ def page(self) -> Page:
56
+ return self._page
57
+
58
+ @property
59
+ def url(self) -> str:
60
+ return self._page.url
61
+
62
+ def lookup_node(self, node_id_or_tag: int | str) -> AccessibilityTreeNode:
63
+ """Looks up the node by id or tag.
64
+
65
+ Args:
66
+ node_id_or_tag: Either the id number (as int or str), or <tag_name>
67
+
68
+ Returns:
69
+ AccessibilityNode.
70
+
71
+ Raise:
72
+ LookupError if node is not matched.
73
+ """
74
+ node: AccessibilityTreeNode | None = None
75
+ node_id_or_tag = str(node_id_or_tag)
76
+ nodes = self._accessibility_tree["nodes"] if self._accessibility_tree else {}
77
+ if re.match("^<.*>", node_id_or_tag):
78
+ tag = node_id_or_tag[1:-1].lower()
79
+ # This is a smart tag, try to resolve it.
80
+ if node := next(
81
+ # We match on anything that starts with the code, this is potentially
82
+ # a little brittle, can be replaced with an RE if there are issues.
83
+ (
84
+ n
85
+ for n in nodes.values()
86
+ if n.name.lower().startswith(tag) and not n.is_ignored
87
+ ),
88
+ None,
89
+ ):
90
+ return node
91
+ else:
92
+ raise LookupError(
93
+ f"Could not find tag {node_id_or_tag} from {[node.name for node in nodes.values() if node.name]}"
94
+ )
95
+ else:
96
+ if (
97
+ node := nodes.get(AXNodeId(node_id_or_tag), None)
98
+ ) and not node.is_ignored:
99
+ return node
100
+ else:
101
+ raise LookupError(f"Could not find element with id {node_id_or_tag}")
102
+
103
+ async def update(self) -> None:
104
+ """Updates the accessibility tree and DOM from current page."""
105
+ await self._page.wait_for_load_state(_WAIT_STRATEGY)
106
+
107
+ available_retries = 2
108
+ retry_delay = 0.25
109
+ while available_retries:
110
+ self._accessibility_tree = create_accessibility_tree(
111
+ ax_nodes=AXTree(
112
+ **await self._cdp_session.send("Accessibility.getFullAXTree", {})
113
+ ).nodes,
114
+ dom_snapshot=DOMSnapshot(
115
+ **await self._cdp_session.send(
116
+ "DOMSnapshot.captureSnapshot",
117
+ {
118
+ "computedStyles": [],
119
+ "includeDOMRects": True,
120
+ },
121
+ )
122
+ ),
123
+ device_scale_factor=self._device_scale_factor,
124
+ window_bounds=Rectangle(
125
+ await self._page.evaluate("window.pageXOffset"),
126
+ await self._page.evaluate("window.pageYOffset"),
127
+ await self._page.evaluate("window.screen.width"),
128
+ await self._page.evaluate("window.screen.height"),
129
+ ),
130
+ )
131
+
132
+ self._rendered_main_content, self._rendered_accessibility_tree = (
133
+ (
134
+ self._accessibility_tree["root"].render_main_content(),
135
+ self._accessibility_tree["root"].render_accessibility_tree(),
136
+ )
137
+ if self._accessibility_tree
138
+ else (None, "")
139
+ )
140
+
141
+ if self._rendered_accessibility_tree:
142
+ return
143
+ # sometimes, the entire tree is initially ignored. in such cases, it's typically
144
+ # because we're sampling too soon. Waiting a small amount of time and trying again
145
+ # resolves the issue.
146
+ available_retries = available_retries - 1
147
+ await asyncio.sleep(retry_delay)
148
+
149
+ def render_at(self) -> str:
150
+ """Returns the current webpage accessibility tree.
151
+
152
+ Only elements visible on the screen will be rendered.
153
+ """
154
+ return self._rendered_accessibility_tree
155
+
156
+ def render_main_content(self) -> str | None:
157
+ return self._rendered_main_content
158
+
159
+ async def go_to_url(self, url: str) -> None:
160
+ """Goes to the given url.
161
+
162
+ Args:
163
+ url: The url to redirect crawler to.
164
+ """
165
+ if "://" not in url:
166
+ url = f"https://{url}"
167
+ try:
168
+ await self._page.goto(url, wait_until=_WAIT_STRATEGY)
169
+ except Exception as e:
170
+ print(f"caught {e}")
171
+ raise
172
+
173
+ async def click(self, element_id: int | str) -> None:
174
+ """Clicks the element with the given id.
175
+
176
+ Args:
177
+ element_id: The id for the element we want to click on.
178
+ """
179
+ element = self.lookup_node(element_id)
180
+ if element.bounds is None:
181
+ raise LookupError(f"Element with id {element_id} has no layout info.")
182
+
183
+ # Mouse.click() requires coordinates relative to the viewport:
184
+ # https://playwright.dev/python/docs/api/class-mouse#mouse-click,
185
+ # thus adjusting the Y coordinate since we only scroll up/down.
186
+ scroll_y = await self._page.evaluate("window.scrollY")
187
+ await self._click_and_await_navigation(
188
+ element.bounds.center_x, element.bounds.center_y - scroll_y
189
+ )
190
+
191
+ async def clear(self, element_id: int | str) -> None:
192
+ """Clears text within a field."""
193
+ await self.click(element_id)
194
+ await self._page.keyboard.press("Control+A")
195
+ await self._page.keyboard.press("Backspace")
196
+
197
+ async def type(self, element_id: int | str, text: str) -> None:
198
+ """Types into the element with the given id."""
199
+ await self.click(element_id)
200
+ await self._page.keyboard.type(text)
201
+
202
+ async def scroll(self, direction: Literal["up", "down"]) -> None:
203
+ """Scrolls the page to the given direction.
204
+
205
+ Args:
206
+ direction: The direction to scroll in ('up' or 'down')
207
+ """
208
+ match direction.lower():
209
+ case "up":
210
+ await self._page.evaluate(
211
+ "(document.scrollingElement || document.body).scrollTop ="
212
+ " (document.scrollingElement || document.body).scrollTop -"
213
+ " window.innerHeight;"
214
+ )
215
+ case "down":
216
+ await self._page.evaluate(
217
+ "(document.scrollingElement || document.body).scrollTop ="
218
+ " (document.scrollingElement || document.body).scrollTop +"
219
+ " window.innerHeight;"
220
+ )
221
+
222
+ case _:
223
+ raise ValueError(f"Invalid scroll direction {direction}")
224
+
225
+ async def forward(self) -> None:
226
+ """Move browser forward one history step."""
227
+ await self._page.go_forward(wait_until=_WAIT_STRATEGY)
228
+
229
+ async def back(self) -> None:
230
+ """Move browser backward one history step."""
231
+ await self._page.go_back(wait_until=_WAIT_STRATEGY)
232
+
233
+ async def refresh(self) -> None:
234
+ """Refresh (reload) the page."""
235
+ await self._page.reload(wait_until=_WAIT_STRATEGY)
236
+
237
+ async def _click_and_await_navigation(self, x: float, y: float) -> None:
238
+ """
239
+ Clicks on the specified coordinates and waits for navigation (if any) to occur.
240
+
241
+ This function sets up event listeners to detect in-page navigation or new page
242
+ navigation, performs a mouse click at the given coordinates, and waits for the
243
+ navigation to complete within the specified timeout period.
244
+
245
+ The point of this is to allow enough time to switch our page in the event of a new
246
+ page being opened. The problem is that it takes some amount of time, and the challenge
247
+ is determining how long to wait.
248
+
249
+ A naïve approach would simply sleep for some amount of time. However, this time may
250
+ not be long enough AND it would delay the common case by that delay waiting for a new
251
+ page navigation that never comes.
252
+
253
+ This approach accomplishes waiting the minimal amount of time in the common cases of
254
+ a click inducing an in page or new page navigation. The downside is that clicks that
255
+ do not induce navigation are delayed by the timeout. Since navigating clicks are much
256
+ more common, this is a reasonable approach.
257
+ """
258
+ future = asyncio.Future[None]()
259
+
260
+ async def on_in_page_navigation(_frame):
261
+ if not future.done():
262
+ await self._page.wait_for_load_state(_WAIT_STRATEGY)
263
+ future.set_result()
264
+
265
+ async def on_new_page(new_page):
266
+ if not future.done():
267
+ await new_page.wait_for_load_state(_WAIT_STRATEGY)
268
+ future.set_result(None)
269
+
270
+ self._page.once("framenavigated", on_in_page_navigation)
271
+ self._page.context.once("page", on_new_page)
272
+
273
+ await self._page.mouse.click(x, y)
274
+
275
+ try:
276
+ await asyncio.wait_for(future, timeout=_WAIT_FOR_NAVIGATION_TIME)
277
+ # a navigation of some sort has occurred and gotten to domcontentloaded
278
+ except (asyncio.TimeoutError, TimeoutError):
279
+ # No navigation occurred within the timeout period
280
+ pass