inspect-ai 0.3.62__py3-none-any.whl → 0.3.63__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (415) hide show
  1. inspect_ai/_cli/main.py +1 -1
  2. inspect_ai/_cli/trace.py +8 -0
  3. inspect_ai/_display/core/active.py +2 -3
  4. inspect_ai/_eval/eval.py +4 -4
  5. inspect_ai/_eval/evalset.py +6 -6
  6. inspect_ai/_eval/task/run.py +3 -0
  7. inspect_ai/_util/hash.py +1 -1
  8. inspect_ai/_view/www/.vscode/extensions.json +3 -0
  9. inspect_ai/_view/www/.vscode/settings.json +8 -0
  10. inspect_ai/_view/www/App.css +92 -29
  11. inspect_ai/_view/www/dist/assets/index.css +16637 -14676
  12. inspect_ai/_view/www/dist/assets/index.js +58897 -51440
  13. inspect_ai/_view/www/dist/index.html +1 -1
  14. inspect_ai/_view/www/index.html +2 -2
  15. inspect_ai/_view/www/log-schema.json +1 -0
  16. inspect_ai/_view/www/package.json +22 -4
  17. inspect_ai/_view/www/postcss.config.cjs +8 -9
  18. inspect_ai/_view/www/src/{App.mjs → App.tsx} +355 -365
  19. inspect_ai/_view/www/src/AppErrorBoundary.tsx +47 -0
  20. inspect_ai/_view/www/src/api/api-browser.ts +2 -2
  21. inspect_ai/_view/www/src/api/api-http.ts +3 -5
  22. inspect_ai/_view/www/src/api/api-vscode.ts +6 -6
  23. inspect_ai/_view/www/src/api/client-api.ts +4 -4
  24. inspect_ai/_view/www/src/api/index.ts +4 -4
  25. inspect_ai/_view/www/src/api/{Types.ts → types.ts} +25 -9
  26. inspect_ai/_view/www/src/appearance/colors.ts +9 -0
  27. inspect_ai/_view/www/src/appearance/fonts.ts +39 -0
  28. inspect_ai/_view/www/src/appearance/icons.ts +100 -0
  29. inspect_ai/_view/www/src/appearance/{Styles.mjs → styles.ts} +2 -32
  30. inspect_ai/_view/www/src/components/AnsiDisplay.tsx +198 -0
  31. inspect_ai/_view/www/src/components/AsciinemaPlayer.tsx +86 -0
  32. inspect_ai/_view/www/src/components/Card.css +60 -0
  33. inspect_ai/_view/www/src/components/Card.tsx +109 -0
  34. inspect_ai/_view/www/src/components/CopyButton.module.css +11 -0
  35. inspect_ai/_view/www/src/components/CopyButton.tsx +58 -0
  36. inspect_ai/_view/www/src/components/DownloadButton.css +4 -0
  37. inspect_ai/_view/www/src/components/DownloadButton.tsx +25 -0
  38. inspect_ai/_view/www/src/components/DownloadPanel.css +10 -0
  39. inspect_ai/_view/www/src/components/DownloadPanel.tsx +30 -0
  40. inspect_ai/_view/www/src/components/EmptyPanel.css +12 -0
  41. inspect_ai/_view/www/src/components/EmptyPanel.tsx +15 -0
  42. inspect_ai/_view/www/src/components/ErrorPanel.css +37 -0
  43. inspect_ai/_view/www/src/components/ErrorPanel.tsx +39 -0
  44. inspect_ai/_view/www/src/components/ExpandablePanel.css +40 -0
  45. inspect_ai/_view/www/src/components/ExpandablePanel.tsx +115 -0
  46. inspect_ai/_view/www/src/components/FindBand.css +49 -0
  47. inspect_ai/_view/www/src/components/FindBand.tsx +130 -0
  48. inspect_ai/_view/www/src/components/HumanBaselineView.css +41 -0
  49. inspect_ai/_view/www/src/components/HumanBaselineView.tsx +162 -0
  50. inspect_ai/_view/www/src/components/JsonPanel.css +20 -0
  51. inspect_ai/_view/www/src/components/JsonPanel.tsx +82 -0
  52. inspect_ai/_view/www/src/components/LabeledValue.css +20 -0
  53. inspect_ai/_view/www/src/components/LabeledValue.tsx +41 -0
  54. inspect_ai/_view/www/src/components/LargeModal.module.css +54 -0
  55. inspect_ai/_view/www/src/components/LargeModal.tsx +199 -0
  56. inspect_ai/_view/www/src/components/LightboxCarousel.css +95 -0
  57. inspect_ai/_view/www/src/components/LightboxCarousel.tsx +132 -0
  58. inspect_ai/_view/www/src/components/MarkdownDiv.css +3 -0
  59. inspect_ai/_view/www/src/components/MarkdownDiv.tsx +133 -0
  60. inspect_ai/_view/www/src/components/MessageBand.css +43 -0
  61. inspect_ai/_view/www/src/components/MessageBand.tsx +39 -0
  62. inspect_ai/_view/www/src/components/MorePopOver.tsx +67 -0
  63. inspect_ai/_view/www/src/components/NavPills.module.css +18 -0
  64. inspect_ai/_view/www/src/components/NavPills.tsx +99 -0
  65. inspect_ai/_view/www/src/components/ProgressBar.module.css +37 -0
  66. inspect_ai/_view/www/src/components/ProgressBar.tsx +22 -0
  67. inspect_ai/_view/www/src/components/TabSet.module.css +40 -0
  68. inspect_ai/_view/www/src/components/TabSet.tsx +200 -0
  69. inspect_ai/_view/www/src/components/ToolButton.css +3 -0
  70. inspect_ai/_view/www/src/components/ToolButton.tsx +27 -0
  71. inspect_ai/_view/www/src/components/VirtualList.module.css +19 -0
  72. inspect_ai/_view/www/src/components/VirtualList.tsx +292 -0
  73. inspect_ai/_view/www/src/{index.js → index.tsx} +45 -19
  74. inspect_ai/_view/www/src/{log → logfile}/remoteLogFile.ts +3 -7
  75. inspect_ai/_view/www/src/{utils/remoteZipFile.mjs → logfile/remoteZipFile.ts} +86 -80
  76. inspect_ai/_view/www/src/metadata/MetaDataGrid.tsx +83 -0
  77. inspect_ai/_view/www/src/metadata/MetaDataView.module.css +35 -0
  78. inspect_ai/_view/www/src/metadata/MetaDataView.tsx +95 -0
  79. inspect_ai/_view/www/src/metadata/MetadataGrid.module.css +15 -0
  80. inspect_ai/_view/www/src/metadata/RenderedContent.module.css +12 -0
  81. inspect_ai/_view/www/src/{components/RenderedContent/RenderedContent.mjs → metadata/RenderedContent.tsx} +92 -73
  82. inspect_ai/_view/www/src/metadata/types.ts +18 -0
  83. inspect_ai/_view/www/src/plan/DatasetDetailView.module.css +3 -0
  84. inspect_ai/_view/www/src/plan/DatasetDetailView.tsx +37 -0
  85. inspect_ai/_view/www/src/plan/DetailStep.module.css +9 -0
  86. inspect_ai/_view/www/src/plan/DetailStep.tsx +31 -0
  87. inspect_ai/_view/www/src/plan/PlanCard.tsx +28 -0
  88. inspect_ai/_view/www/src/plan/PlanDetailView.module.css +48 -0
  89. inspect_ai/_view/www/src/plan/PlanDetailView.tsx +309 -0
  90. inspect_ai/_view/www/src/plan/ScorerDetailView.module.css +3 -0
  91. inspect_ai/_view/www/src/plan/ScorerDetailView.tsx +30 -0
  92. inspect_ai/_view/www/src/plan/SolverDetailView.module.css +15 -0
  93. inspect_ai/_view/www/src/plan/SolverDetailView.tsx +32 -0
  94. inspect_ai/_view/www/src/samples/InlineSampleDisplay.module.css +8 -0
  95. inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx +53 -0
  96. inspect_ai/_view/www/src/samples/SampleDialog.tsx +122 -0
  97. inspect_ai/_view/www/src/samples/SampleDisplay.module.css +29 -0
  98. inspect_ai/_view/www/src/samples/SampleDisplay.tsx +326 -0
  99. inspect_ai/_view/www/src/samples/SampleSummaryView.module.css +24 -0
  100. inspect_ai/_view/www/src/samples/SampleSummaryView.tsx +175 -0
  101. inspect_ai/_view/www/src/samples/SamplesTools.tsx +60 -0
  102. inspect_ai/_view/www/src/samples/chat/ChatMessage.module.css +29 -0
  103. inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx +76 -0
  104. inspect_ai/_view/www/src/samples/chat/ChatMessageRenderer.tsx +60 -0
  105. inspect_ai/_view/www/src/samples/chat/ChatMessageRow.module.css +9 -0
  106. inspect_ai/_view/www/src/samples/chat/ChatMessageRow.tsx +57 -0
  107. inspect_ai/_view/www/src/samples/chat/ChatView.tsx +46 -0
  108. inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.module.css +4 -0
  109. inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.tsx +58 -0
  110. inspect_ai/_view/www/src/samples/chat/MessageContent.module.css +4 -0
  111. inspect_ai/_view/www/src/samples/chat/MessageContent.tsx +143 -0
  112. inspect_ai/_view/www/src/samples/chat/MessageContents.module.css +3 -0
  113. inspect_ai/_view/www/src/samples/chat/MessageContents.tsx +131 -0
  114. inspect_ai/_view/www/src/samples/chat/messages.ts +112 -0
  115. inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.tsx +145 -0
  116. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.module.css +14 -0
  117. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.tsx +86 -0
  118. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.module.css +19 -0
  119. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.tsx +53 -0
  120. inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.module.css +4 -0
  121. inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.tsx +18 -0
  122. inspect_ai/_view/www/src/samples/chat/tools/tool.ts +107 -0
  123. inspect_ai/_view/www/src/samples/descriptor/samplesDescriptor.tsx +363 -0
  124. inspect_ai/_view/www/src/samples/descriptor/score/BooleanScoreDescriptor.module.css +22 -0
  125. inspect_ai/_view/www/src/samples/descriptor/score/BooleanScoreDescriptor.tsx +26 -0
  126. inspect_ai/_view/www/src/samples/descriptor/score/CategoricalScoreDescriptor.tsx +18 -0
  127. inspect_ai/_view/www/src/samples/descriptor/score/NumericScoreDescriptor.tsx +27 -0
  128. inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.module.css +18 -0
  129. inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.tsx +71 -0
  130. inspect_ai/_view/www/src/samples/descriptor/score/OtherScoreDescriptor.tsx +20 -0
  131. inspect_ai/_view/www/src/samples/descriptor/score/PassFailScoreDescriptor.module.css +28 -0
  132. inspect_ai/_view/www/src/samples/descriptor/score/PassFailScoreDescriptor.tsx +81 -0
  133. inspect_ai/_view/www/src/samples/descriptor/score/ScoreDescriptor.tsx +99 -0
  134. inspect_ai/_view/www/src/samples/descriptor/types.ts +55 -0
  135. inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.module.css +19 -0
  136. inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.tsx +22 -0
  137. inspect_ai/_view/www/src/samples/error/SampleErrorView.module.css +17 -0
  138. inspect_ai/_view/www/src/samples/error/SampleErrorView.tsx +31 -0
  139. inspect_ai/_view/www/src/samples/error/error.ts +15 -0
  140. inspect_ai/_view/www/src/samples/list/SampleFooter.module.css +9 -0
  141. inspect_ai/_view/www/src/samples/list/SampleFooter.tsx +14 -0
  142. inspect_ai/_view/www/src/samples/list/SampleHeader.module.css +13 -0
  143. inspect_ai/_view/www/src/samples/list/SampleHeader.tsx +36 -0
  144. inspect_ai/_view/www/src/samples/list/SampleList.module.css +11 -0
  145. inspect_ai/_view/www/src/samples/list/SampleList.tsx +247 -0
  146. inspect_ai/_view/www/src/samples/list/SampleRow.module.css +33 -0
  147. inspect_ai/_view/www/src/samples/list/SampleRow.tsx +98 -0
  148. inspect_ai/_view/www/src/samples/list/SampleSeparator.module.css +6 -0
  149. inspect_ai/_view/www/src/samples/list/SampleSeparator.tsx +24 -0
  150. inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.module.css +9 -0
  151. inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.tsx +51 -0
  152. inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.module.css +16 -0
  153. inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx +173 -0
  154. inspect_ai/_view/www/src/samples/sample-tools/SortFilter.module.css +9 -0
  155. inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx +182 -0
  156. inspect_ai/_view/www/src/samples/{tools/filters.mjs → sample-tools/filters.ts} +86 -81
  157. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.module.css +16 -0
  158. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx +288 -0
  159. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/completions.ts +346 -0
  160. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/language.ts +19 -0
  161. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/tokenize.ts +97 -0
  162. inspect_ai/_view/www/src/samples/{SampleLimit.mjs → sampleLimit.ts} +3 -6
  163. inspect_ai/_view/www/src/samples/scores/SampleScoreView.module.css +53 -0
  164. inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx +168 -0
  165. inspect_ai/_view/www/src/samples/scores/SampleScores.module.css +5 -0
  166. inspect_ai/_view/www/src/samples/scores/SampleScores.tsx +37 -0
  167. inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.tsx +66 -0
  168. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx +51 -0
  169. inspect_ai/_view/www/src/samples/transcript/InfoEventView.module.css +3 -0
  170. inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx +54 -0
  171. inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx +48 -0
  172. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.module.css +6 -0
  173. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.tsx +36 -0
  174. inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css +43 -0
  175. inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx +223 -0
  176. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.module.css +23 -0
  177. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.tsx +108 -0
  178. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx +75 -0
  179. inspect_ai/_view/www/src/samples/transcript/SampleTranscript.tsx +22 -0
  180. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.module.css +15 -0
  181. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.tsx +100 -0
  182. inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx +171 -0
  183. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.module.css +19 -0
  184. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.tsx +133 -0
  185. inspect_ai/_view/www/src/samples/transcript/ToolEventView.module.css +10 -0
  186. inspect_ai/_view/www/src/samples/transcript/ToolEventView.tsx +91 -0
  187. inspect_ai/_view/www/src/samples/transcript/TranscriptView.module.css +49 -0
  188. inspect_ai/_view/www/src/samples/transcript/TranscriptView.tsx +449 -0
  189. inspect_ai/_view/www/src/samples/transcript/event/EventNav.module.css +5 -0
  190. inspect_ai/_view/www/src/samples/transcript/event/EventNav.tsx +43 -0
  191. inspect_ai/_view/www/src/samples/transcript/event/EventNavs.module.css +3 -0
  192. inspect_ai/_view/www/src/samples/transcript/event/EventNavs.tsx +38 -0
  193. inspect_ai/_view/www/src/samples/transcript/event/EventPanel.module.css +25 -0
  194. inspect_ai/_view/www/src/samples/transcript/event/EventPanel.tsx +190 -0
  195. inspect_ai/_view/www/src/samples/transcript/event/EventRow.module.css +13 -0
  196. inspect_ai/_view/www/src/samples/transcript/event/EventRow.tsx +32 -0
  197. inspect_ai/_view/www/src/samples/transcript/event/EventSection.module.css +8 -0
  198. inspect_ai/_view/www/src/samples/transcript/event/EventSection.tsx +29 -0
  199. inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.tsx +67 -0
  200. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx +274 -0
  201. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenders.module.css +10 -0
  202. inspect_ai/_view/www/src/samples/transcript/state/StateEventView.module.css +9 -0
  203. inspect_ai/_view/www/src/samples/transcript/state/{StateEventView.mjs → StateEventView.tsx} +148 -110
  204. inspect_ai/_view/www/src/samples/transcript/types.ts +58 -0
  205. inspect_ai/_view/www/src/types/log.d.ts +1 -0
  206. inspect_ai/_view/www/src/types/prism.d.ts +11 -0
  207. inspect_ai/_view/www/src/types.ts +71 -0
  208. inspect_ai/_view/www/src/usage/ModelTokenTable.tsx +22 -0
  209. inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css +24 -0
  210. inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx +95 -0
  211. inspect_ai/_view/www/src/usage/TokenTable.module.css +17 -0
  212. inspect_ai/_view/www/src/usage/TokenTable.tsx +91 -0
  213. inspect_ai/_view/www/src/usage/UsageCard.module.css +15 -0
  214. inspect_ai/_view/www/src/usage/UsageCard.tsx +67 -0
  215. inspect_ai/_view/www/src/utils/attachments.ts +42 -0
  216. inspect_ai/_view/www/src/utils/{Base64.mjs → base64.ts} +1 -6
  217. inspect_ai/_view/www/src/{components/Browser.mjs → utils/browser.ts} +0 -1
  218. inspect_ai/_view/www/src/utils/debugging.ts +28 -0
  219. inspect_ai/_view/www/src/utils/dom.ts +30 -0
  220. inspect_ai/_view/www/src/utils/format.ts +194 -0
  221. inspect_ai/_view/www/src/utils/git.ts +7 -0
  222. inspect_ai/_view/www/src/utils/html.ts +6 -0
  223. inspect_ai/_view/www/src/utils/http.ts +14 -0
  224. inspect_ai/_view/www/src/utils/{Path.mjs → path.ts} +2 -9
  225. inspect_ai/_view/www/src/utils/{Print.mjs → print.ts} +34 -26
  226. inspect_ai/_view/www/src/utils/queue.ts +51 -0
  227. inspect_ai/_view/www/src/utils/sync.ts +114 -0
  228. inspect_ai/_view/www/src/utils/{Type.mjs → type.ts} +3 -6
  229. inspect_ai/_view/www/src/utils/vscode.ts +13 -0
  230. inspect_ai/_view/www/src/workspace/WorkSpace.tsx +324 -0
  231. inspect_ai/_view/www/src/workspace/WorkSpaceView.module.css +33 -0
  232. inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx +160 -0
  233. inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.module.css +3 -0
  234. inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.tsx +28 -0
  235. inspect_ai/_view/www/src/workspace/navbar/Navbar.module.css +54 -0
  236. inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx +68 -0
  237. inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.module.css +52 -0
  238. inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx +113 -0
  239. inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.module.css +67 -0
  240. inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx +156 -0
  241. inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.module.css +28 -0
  242. inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx +222 -0
  243. inspect_ai/_view/www/src/workspace/navbar/StatusPanel.module.css +14 -0
  244. inspect_ai/_view/www/src/workspace/navbar/StatusPanel.tsx +61 -0
  245. inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.module.css +15 -0
  246. inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.tsx +71 -0
  247. inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.module.css +5 -0
  248. inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.tsx +56 -0
  249. inspect_ai/_view/www/src/workspace/sidebar/Sidebar.module.css +68 -0
  250. inspect_ai/_view/www/src/workspace/sidebar/Sidebar.tsx +85 -0
  251. inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.module.css +29 -0
  252. inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.tsx +95 -0
  253. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.module.css +23 -0
  254. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx +41 -0
  255. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.module.css +35 -0
  256. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx +61 -0
  257. inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx +80 -0
  258. inspect_ai/_view/www/src/workspace/tabs/JsonTab.module.css +5 -0
  259. inspect_ai/_view/www/src/workspace/tabs/JsonTab.tsx +46 -0
  260. inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx +204 -0
  261. inspect_ai/_view/www/src/workspace/tabs/grouping.ts +195 -0
  262. inspect_ai/_view/www/src/workspace/tabs/types.ts +19 -0
  263. inspect_ai/_view/www/src/workspace/types.ts +10 -0
  264. inspect_ai/_view/www/tsconfig.json +23 -9
  265. inspect_ai/_view/www/vite.config.js +8 -17
  266. inspect_ai/_view/www/yarn.lock +627 -556
  267. inspect_ai/dataset/_dataset.py +36 -0
  268. inspect_ai/dataset/_sources/csv.py +8 -0
  269. inspect_ai/dataset/_sources/file.py +4 -0
  270. inspect_ai/dataset/_sources/hf.py +11 -1
  271. inspect_ai/dataset/_sources/json.py +8 -0
  272. inspect_ai/log/_log.py +3 -6
  273. inspect_ai/log/_message.py +1 -1
  274. inspect_ai/log/_recorders/json.py +5 -7
  275. inspect_ai/model/_call_tools.py +2 -1
  276. inspect_ai/model/_providers/anthropic.py +3 -3
  277. inspect_ai/model/_providers/openai_o1.py +3 -5
  278. inspect_ai/model/_providers/openrouter.py +86 -0
  279. inspect_ai/model/_providers/providers.py +11 -0
  280. inspect_ai/scorer/_answer.py +7 -7
  281. inspect_ai/scorer/_classification.py +34 -18
  282. inspect_ai/scorer/_common.py +2 -8
  283. inspect_ai/solver/_multiple_choice.py +24 -9
  284. inspect_ai/tool/__init__.py +2 -0
  285. inspect_ai/tool/{beta → _tools}/_computer/_computer.py +2 -5
  286. inspect_ai/tool/_tools/_computer/_resources/tool/__init__.py +0 -0
  287. inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_x11_client.py +1 -1
  288. inspect_ai/tool/_tools/_computer/_resources/tool/requirements.txt +0 -0
  289. inspect_ai/tool/_tools/_execute.py +8 -2
  290. inspect_ai/tool/beta.py +3 -0
  291. inspect_ai/util/_sandbox/docker/docker.py +32 -85
  292. inspect_ai/util/_sandbox/self_check.py +124 -16
  293. {inspect_ai-0.3.62.dist-info → inspect_ai-0.3.63.dist-info}/METADATA +2 -1
  294. inspect_ai-0.3.63.dist-info/RECORD +618 -0
  295. inspect_ai/_view/www/src/Register.mjs +0 -3
  296. inspect_ai/_view/www/src/Types.mjs +0 -38
  297. inspect_ai/_view/www/src/appearance/Colors.mjs +0 -27
  298. inspect_ai/_view/www/src/appearance/Fonts.mjs +0 -66
  299. inspect_ai/_view/www/src/appearance/Icons.mjs +0 -240
  300. inspect_ai/_view/www/src/components/AnsiDisplay.mjs +0 -184
  301. inspect_ai/_view/www/src/components/AppErrorBoundary.mjs +0 -34
  302. inspect_ai/_view/www/src/components/AsciiCinemaPlayer.mjs +0 -74
  303. inspect_ai/_view/www/src/components/Card.mjs +0 -126
  304. inspect_ai/_view/www/src/components/ChatView.mjs +0 -441
  305. inspect_ai/_view/www/src/components/CopyButton.mjs +0 -48
  306. inspect_ai/_view/www/src/components/Dialog.mjs +0 -61
  307. inspect_ai/_view/www/src/components/DownloadButton.mjs +0 -15
  308. inspect_ai/_view/www/src/components/DownloadPanel.mjs +0 -29
  309. inspect_ai/_view/www/src/components/EmptyPanel.mjs +0 -23
  310. inspect_ai/_view/www/src/components/ErrorPanel.mjs +0 -66
  311. inspect_ai/_view/www/src/components/ExpandablePanel.mjs +0 -136
  312. inspect_ai/_view/www/src/components/FindBand.mjs +0 -157
  313. inspect_ai/_view/www/src/components/HumanBaselineView.mjs +0 -168
  314. inspect_ai/_view/www/src/components/JsonPanel.mjs +0 -61
  315. inspect_ai/_view/www/src/components/LabeledValue.mjs +0 -32
  316. inspect_ai/_view/www/src/components/LargeModal.mjs +0 -190
  317. inspect_ai/_view/www/src/components/LightboxCarousel.mjs +0 -217
  318. inspect_ai/_view/www/src/components/MarkdownDiv.mjs +0 -118
  319. inspect_ai/_view/www/src/components/MessageBand.mjs +0 -48
  320. inspect_ai/_view/www/src/components/MessageContent.mjs +0 -111
  321. inspect_ai/_view/www/src/components/MetaDataGrid.mjs +0 -92
  322. inspect_ai/_view/www/src/components/MetaDataView.mjs +0 -109
  323. inspect_ai/_view/www/src/components/MorePopOver.mjs +0 -50
  324. inspect_ai/_view/www/src/components/NavPills.mjs +0 -63
  325. inspect_ai/_view/www/src/components/ProgressBar.mjs +0 -51
  326. inspect_ai/_view/www/src/components/RenderedContent/ChatMessageRenderer.mjs +0 -54
  327. inspect_ai/_view/www/src/components/RenderedContent/Types.mjs +0 -19
  328. inspect_ai/_view/www/src/components/TabSet.mjs +0 -184
  329. inspect_ai/_view/www/src/components/ToolButton.mjs +0 -16
  330. inspect_ai/_view/www/src/components/Tools.mjs +0 -376
  331. inspect_ai/_view/www/src/components/VirtualList.mjs +0 -280
  332. inspect_ai/_view/www/src/components/ansi-output.js +0 -932
  333. inspect_ai/_view/www/src/json/JsonTab.mjs +0 -48
  334. inspect_ai/_view/www/src/log-reader/Log-Reader.mjs +0 -25
  335. inspect_ai/_view/www/src/log-reader/Native-Log-Reader.mjs +0 -13
  336. inspect_ai/_view/www/src/log-reader/Open-AI-Log-Reader.mjs +0 -263
  337. inspect_ai/_view/www/src/navbar/Navbar.mjs +0 -418
  338. inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +0 -175
  339. inspect_ai/_view/www/src/plan/PlanCard.mjs +0 -418
  340. inspect_ai/_view/www/src/samples/SampleDialog.mjs +0 -123
  341. inspect_ai/_view/www/src/samples/SampleDisplay.mjs +0 -516
  342. inspect_ai/_view/www/src/samples/SampleError.mjs +0 -99
  343. inspect_ai/_view/www/src/samples/SampleList.mjs +0 -427
  344. inspect_ai/_view/www/src/samples/SampleScoreView.mjs +0 -172
  345. inspect_ai/_view/www/src/samples/SampleScores.mjs +0 -34
  346. inspect_ai/_view/www/src/samples/SampleTranscript.mjs +0 -20
  347. inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +0 -771
  348. inspect_ai/_view/www/src/samples/SamplesTab.mjs +0 -399
  349. inspect_ai/_view/www/src/samples/SamplesTools.mjs +0 -64
  350. inspect_ai/_view/www/src/samples/tools/EpochFilter.mjs +0 -38
  351. inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +0 -756
  352. inspect_ai/_view/www/src/samples/tools/SelectScorer.mjs +0 -141
  353. inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +0 -151
  354. inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.mjs +0 -71
  355. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs +0 -44
  356. inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs +0 -271
  357. inspect_ai/_view/www/src/samples/transcript/EventRow.mjs +0 -46
  358. inspect_ai/_view/www/src/samples/transcript/EventSection.mjs +0 -33
  359. inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs +0 -59
  360. inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs +0 -44
  361. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.mjs +0 -32
  362. inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs +0 -216
  363. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.mjs +0 -107
  364. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.mjs +0 -74
  365. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.mjs +0 -100
  366. inspect_ai/_view/www/src/samples/transcript/StepEventView.mjs +0 -187
  367. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.mjs +0 -133
  368. inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs +0 -88
  369. inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs +0 -459
  370. inspect_ai/_view/www/src/samples/transcript/Types.mjs +0 -44
  371. inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.mjs +0 -53
  372. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.mjs +0 -254
  373. inspect_ai/_view/www/src/sidebar/Sidebar.mjs +0 -418
  374. inspect_ai/_view/www/src/usage/ModelTokenTable.mjs +0 -72
  375. inspect_ai/_view/www/src/usage/UsageCard.mjs +0 -159
  376. inspect_ai/_view/www/src/utils/Format.mjs +0 -260
  377. inspect_ai/_view/www/src/utils/Git.mjs +0 -12
  378. inspect_ai/_view/www/src/utils/Html.mjs +0 -21
  379. inspect_ai/_view/www/src/utils/attachments.mjs +0 -31
  380. inspect_ai/_view/www/src/utils/debugging.mjs +0 -23
  381. inspect_ai/_view/www/src/utils/http.mjs +0 -18
  382. inspect_ai/_view/www/src/utils/queue.mjs +0 -67
  383. inspect_ai/_view/www/src/utils/sync.mjs +0 -101
  384. inspect_ai/_view/www/src/workspace/TaskErrorPanel.mjs +0 -17
  385. inspect_ai/_view/www/src/workspace/WorkSpace.mjs +0 -516
  386. inspect_ai/tool/beta/__init__.py +0 -5
  387. inspect_ai-0.3.62.dist-info/RECORD +0 -481
  388. /inspect_ai/{tool/beta/_computer/_resources/tool/__init__.py → _view/www/src/components/MorePopOver.css} +0 -0
  389. /inspect_ai/_view/www/src/{constants.mjs → constants.ts} +0 -0
  390. /inspect_ai/{tool/beta/_computer/_resources/tool/requirements.txt → _view/www/src/workspace/tabs/InfoTab.module.css} +0 -0
  391. /inspect_ai/tool/{beta → _tools}/_computer/__init__.py +0 -0
  392. /inspect_ai/tool/{beta → _tools}/_computer/_common.py +0 -0
  393. /inspect_ai/tool/{beta → _tools}/_computer/_computer_split.py +0 -0
  394. /inspect_ai/tool/{beta → _tools}/_computer/_resources/Dockerfile +0 -0
  395. /inspect_ai/tool/{beta → _tools}/_computer/_resources/README.md +0 -0
  396. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/entrypoint.sh +0 -0
  397. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/novnc_startup.sh +0 -0
  398. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/x11vnc_startup.sh +0 -0
  399. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/xfce_startup.sh +0 -0
  400. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/xvfb_startup.sh +0 -0
  401. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/.config/Code/User/globalStorage/state.vscdb +0 -0
  402. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/.config/Code/User/settings.json +0 -0
  403. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml +0 -0
  404. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml +0 -0
  405. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop +0 -0
  406. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/Desktop/Terminal.desktop +0 -0
  407. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop +0 -0
  408. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_logger.py +0 -0
  409. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_run.py +0 -0
  410. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_tool_result.py +0 -0
  411. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/computer_tool.py +0 -0
  412. {inspect_ai-0.3.62.dist-info → inspect_ai-0.3.63.dist-info}/LICENSE +0 -0
  413. {inspect_ai-0.3.62.dist-info → inspect_ai-0.3.63.dist-info}/WHEEL +0 -0
  414. {inspect_ai-0.3.62.dist-info → inspect_ai-0.3.63.dist-info}/entry_points.txt +0 -0
  415. {inspect_ai-0.3.62.dist-info → inspect_ai-0.3.63.dist-info}/top_level.txt +0 -0
@@ -1,756 +0,0 @@
1
- import { autocompletion, startCompletion } from "@codemirror/autocomplete";
2
- import {
3
- HighlightStyle,
4
- StreamLanguage,
5
- StringStream,
6
- bracketMatching,
7
- syntaxHighlighting,
8
- } from "@codemirror/language";
9
- import { linter } from "@codemirror/lint";
10
- import { Compartment, EditorState } from "@codemirror/state";
11
- import { tags } from "@lezer/highlight";
12
- import { EditorView, minimalSetup } from "codemirror";
13
- import { html } from "htm/preact";
14
- import { useEffect, useMemo, useRef, useState } from "preact/hooks";
15
- import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
16
- import { filterSamples, scoreFilterItems } from "./filters.mjs";
17
- import {
18
- kScoreTypeBoolean,
19
- kScoreTypeCategorical,
20
- kScoreTypeNumeric,
21
- kScoreTypeOther,
22
- kScoreTypePassFail,
23
- } from "../../constants.mjs";
24
-
25
- /**
26
- * @typedef {Object} Token
27
- * @property {string} type
28
- * @property {string} text
29
- * @property {number} from
30
- * @property {number} to
31
- */
32
-
33
- /**
34
- * @typedef {Object} FilteringResult
35
- * @property {number} numSamples - The number of samples that match the filter.
36
- * @property {import("./filters.mjs").FilterError | undefined} error - The error in the filter expression, if any.
37
- */
38
-
39
- const KEYWORDS = ["and", "or", "not", "in", "not in", "mod"];
40
-
41
- const MATH_FUNCTIONS = [
42
- ["min", "Minimum of two or more values"],
43
- ["max", "Maximum of two or more values"],
44
- ["abs", "Absolute value"],
45
- ["round", "Round to the nearest integer"],
46
- ["floor", "Round down to the nearest integer"],
47
- ["ceil", "Round up to the nearest integer"],
48
- ["sqrt", "Square root"],
49
- ["log", "Natural logarithm"],
50
- ["log2", "Base 2 logarithm"],
51
- ["log10", "Base 10 logarithm"],
52
- ];
53
-
54
- const SAMPLE_FUNCTIONS = [
55
- ["input_contains", "Checks if input contains a regular expression"],
56
- ["target_contains", "Checks if target contains a regular expression"],
57
- ];
58
-
59
- /**
60
- * Makes sure that the filter expression is a single line.
61
- * @param {import("@codemirror/state").Transaction} tr - The transaction to join lines in.
62
- * @returns {import("@codemirror/state").TransactionSpec} The transaction with joined lines, if any.
63
- */
64
- function ensureOneLine(tr) {
65
- const newDoc = tr.newDoc.toString();
66
- if (newDoc.includes("\n")) {
67
- if (tr.isUserEvent("input.paste")) {
68
- const newDocAdjusted = newDoc.replace(/\n/g, " ").trim();
69
- return {
70
- changes: {
71
- from: 0,
72
- to: tr.startState.doc.length,
73
- insert: newDocAdjusted,
74
- },
75
- };
76
- } else {
77
- return {};
78
- }
79
- }
80
- return tr;
81
- }
82
-
83
- const highlightStyle = HighlightStyle.define([
84
- { tag: tags.string, class: "token string" },
85
- { tag: tags.number, class: "token number" },
86
- { tag: tags.keyword, class: "token keyword" },
87
- ]);
88
-
89
- /** @param {string} word */
90
- function countSpaces(word) {
91
- return word.split(" ").length - 1;
92
- }
93
-
94
- const nextToken = (() => {
95
- const wordsRe = (words) => new RegExp(`^(${words.join("|")})\\b`);
96
- const keywordsRe = wordsRe(
97
- // Sort to make sure "not in" is matched before "not".
98
- KEYWORDS.sort((a, b) => countSpaces(b) - countSpaces(a)),
99
- );
100
- const mathFunctionsRe = wordsRe(MATH_FUNCTIONS.map(([label]) => label));
101
- const sampleFunctionsRe = wordsRe(SAMPLE_FUNCTIONS.map(([label]) => label));
102
-
103
- /** @param {import("@codemirror/language").StringStream} stream */
104
- return function (stream) {
105
- if (stream.match(/"[^"]*"/)) return "string";
106
- if (stream.match(/"[^"]*/)) return "unterminatedString";
107
- if (stream.match(/^(-|\+)?\d+(\.\d+)?/)) return "number";
108
- if (stream.match(keywordsRe)) return "keyword";
109
- if (stream.match(mathFunctionsRe)) return "mathFunction";
110
- if (stream.match(sampleFunctionsRe)) return "sampleFunction";
111
- if (stream.match(/^[a-zA-Z_][a-zA-Z0-9_]*/)) return "variable";
112
- if (stream.match(/^(==|!=|<=|>=|<|>|~=)/)) return "relation";
113
- if (stream.match(/^(=|!|~)/)) return "miscOperator"; // recognize relations while typing; not valid syntax per se
114
- if (stream.match(/^(\+|-|\*|\/|\^|\(|\)|,|\.)/)) return "miscOperator";
115
- stream.next();
116
- return null;
117
- };
118
- })();
119
-
120
- const language = StreamLanguage.define({
121
- token: nextToken,
122
- tokenTable: {
123
- string: tags.string,
124
- unterminatedString: tags.string,
125
- number: tags.number,
126
- keyword: tags.keyword,
127
- mathFunction: tags.function(tags.variableName),
128
- sampleFunction: tags.function(tags.variableName),
129
- variable: tags.variableName,
130
- relation: tags.operator,
131
- miscOperator: tags.operator,
132
- },
133
- });
134
-
135
- /**
136
- * @param {string} input
137
- * @returns {Token[]}
138
- */
139
- function tokenize(input) {
140
- const tokens = [];
141
- const stream = new StringStream(input, 0, 0);
142
- while (stream.pos < input.length) {
143
- const from = stream.pos;
144
- const type = nextToken(stream);
145
- if (type) {
146
- tokens.push({
147
- type,
148
- text: input.slice(from, stream.pos),
149
- from,
150
- to: stream.pos,
151
- });
152
- }
153
- }
154
- return tokens;
155
- }
156
-
157
- /**
158
- * @param {import("./filters.mjs").ScoreFilterItem[]} filterItems
159
- * @param {string} scorer
160
- * @returns {import("./filters.mjs").ScoreFilterItem[]}
161
- */
162
- function getMemberScoreItems(filterItems, scorer) {
163
- return filterItems.filter((item) =>
164
- item?.qualifiedName?.startsWith(`${scorer}.`),
165
- );
166
- }
167
-
168
- /**
169
- * Generates completions for the filter expression. The main goal is to make the
170
- * sample filter intuitive for beginners and to provide a smooth experience for
171
- * simple cases. To this end, we proactively try to suggest the next step of the
172
- * expression, in a wizard-style fashion. This logic is primarily intended to
173
- * support unsophisticated expressions of the form
174
- * SUBEXPR and/or SUBEXPR or/not SUBEXPR ...
175
- * where each SUBEXPR is
176
- * VARIABLE ==/!=/</>/in/... VALUE
177
- * and VALUE is a literal (string, number, etc.)
178
- * It does support some expressions more complex than that, but the completion
179
- * algorithm is not intended to be comprehensive. This is why we usually add
180
- * default completions to the list in case our guess was off.
181
- *
182
- * @param {import("@codemirror/autocomplete").CompletionContext} context
183
- * @param {import("../../samples/tools/filters.mjs").ScoreFilterItem[]} filterItems
184
- * @returns {import("@codemirror/autocomplete").CompletionResult}
185
- */
186
- function getCompletions(context, filterItems) {
187
- /** @param {Token} token */
188
- const isLiteral = (token) =>
189
- ["string", "unterminatedString", "number"].includes(token?.type);
190
- /** @param {Token} token */
191
- const isLogicalOp = (token) => ["and", "or", "not"].includes(token?.text);
192
-
193
- /**
194
- * With most tokens we complete only after a space, but for sometimes it makes
195
- * sense to start autocompletion as soon as the token was typed.
196
- * @param {Token} token
197
- */
198
- const autocompleteImmediatelyAfter = (token) =>
199
- ["(", "."].includes(token?.text);
200
-
201
- /**
202
- * @param {import("codemirror").EditorView} view
203
- * @param {import("@codemirror/autocomplete").Completion} completion
204
- * @param {number} from
205
- * @param {number} to
206
- */
207
- function applyWithCall(view, completion, from, to) {
208
- view.dispatch({
209
- changes: { from, to, insert: `${completion.label}()` },
210
- selection: { anchor: from + completion.label.length + 1 },
211
- });
212
- }
213
-
214
- /** @type {(k: string) => import("@codemirror/autocomplete").Completion} */
215
- const makeKeywordCompletion = (k) => ({
216
- label: k,
217
- type: "keyword",
218
- boost: -20,
219
- });
220
- /** @type {([label, info]: [string, string]) => import("@codemirror/autocomplete").Completion} */
221
- const makeMathFunctionCompletion = ([label, info]) => ({
222
- label,
223
- type: "function",
224
- info,
225
- apply: applyWithCall,
226
- boost: -10,
227
- });
228
- /** @type {([label, info]: [string, string]) => import("@codemirror/autocomplete").Completion} */
229
- const makeSampleFunctionCompletion = ([label, info]) => ({
230
- label,
231
- type: "function",
232
- info,
233
- apply: applyWithCall,
234
- boost: 0,
235
- });
236
- /** @type {(k: string) => import("@codemirror/autocomplete").Completion} */
237
- const makeLiteralCompletion = (k) => ({
238
- label: k,
239
- type: "text",
240
- boost: 10,
241
- });
242
- /**
243
- * @param {import("./filters.mjs").ScoreFilterItem} item
244
- * @param {Object} [props]
245
- * @param {(item: import("./filters.mjs").ScoreFilterItem) => boolean} [props.autoSpaceIf] - Similar to `autoSpaceAfter`, but conditional.
246
- * @returns {import("@codemirror/autocomplete").Completion}
247
- */
248
- const makeCanonicalNameCompletion = (
249
- item,
250
- { autoSpaceIf = () => false } = {},
251
- ) => ({
252
- label: item.canonicalName + (autoSpaceIf(item) ? " " : ""),
253
- type: "variable",
254
- info: item.tooltip,
255
- boost: 20,
256
- });
257
- /** @type {(item: import("./filters.mjs").ScoreFilterItem) => import("@codemirror/autocomplete").Completion} */
258
- const makeMemberAccessCompletion = (item) => ({
259
- label: item.qualifiedName.split(".")[1],
260
- type: "variable",
261
- info: item.tooltip,
262
- boost: 20,
263
- });
264
-
265
- const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
266
- const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
267
- makeMathFunctionCompletion,
268
- );
269
- const sampleFunctionCompletionItems = SAMPLE_FUNCTIONS.map(
270
- makeSampleFunctionCompletion,
271
- );
272
- const variableCompletionItems = filterItems.map((item) =>
273
- makeCanonicalNameCompletion(item),
274
- );
275
-
276
- const defaultCompletionItems = [
277
- ...keywordCompletionItems,
278
- ...mathFunctionCompletionItems,
279
- ...sampleFunctionCompletionItems,
280
- ...variableCompletionItems,
281
- ];
282
-
283
- const doc = context.state.doc;
284
- const input = doc.toString().slice(0, context.pos);
285
- const tokens = tokenize(input);
286
- const lastToken = tokens[tokens.length - 1];
287
- const isCompletionInsideToken =
288
- lastToken &&
289
- context.pos == lastToken.to &&
290
- !autocompleteImmediatelyAfter(lastToken);
291
- const currentTokenIndex = isCompletionInsideToken
292
- ? tokens.length - 1
293
- : tokens.length; // `currentToken` is undefined when we are not inside a token
294
-
295
- /**
296
- * Returns nth token back away from the current token. Note that `prevToken(0)`
297
- * is always reserved for the current token, whether it exists or not.
298
- * @param {number} index
299
- * @returns {Token | undefined}
300
- */
301
- const prevToken = (index) => tokens[currentTokenIndex - index];
302
-
303
- const currentToken = prevToken(0);
304
- const completionStart = currentToken ? currentToken.from : context.pos;
305
- const completingAtEnd = context.pos == doc.length;
306
-
307
- /**
308
- * @param {number} endIndex
309
- * @returns {import("../../samples/tools/filters.mjs").ScoreFilterItem | undefined}
310
- */
311
- const findFilterItem = (endIndex) => {
312
- if (prevToken(endIndex)?.type == "variable") {
313
- let name = prevToken(endIndex).text;
314
- let i = endIndex;
315
- while (prevToken(i + 1)?.text == ".") {
316
- if (prevToken(i + 2)?.type == "variable") {
317
- name = prevToken(i + 2).text + "." + name;
318
- i += 2;
319
- } else {
320
- break;
321
- }
322
- }
323
- return filterItems.find((item) => item.canonicalName == name);
324
- }
325
- return undefined;
326
- };
327
-
328
- /**
329
- * @param {import("@codemirror/autocomplete").Completion[]} priorityCompletions
330
- * @param {Object} props
331
- * @param {boolean} [props.autocompleteInTheMiddle] - If true, completion would be shown automatically even when editing in the middle of the expression.
332
- * @param {boolean} [props.enforceOrder] - If true, the priorityCompletions are shown in the order they are provided.
333
- * @param {boolean} [props.autoSpaceAfter] - If true, space is inserted after priorityCompletions. When a user accepts a completion with a space, another completion is suggested immediately (see `activateOnCompletion`). Use when fairly certain that the expression continues.
334
- * @param {boolean} [props.includeDefault] - If true, the default completions are included after the priority completions.
335
- * @returns {import("@codemirror/autocomplete").CompletionResult}
336
- */
337
- const makeCompletions = (
338
- priorityCompletions,
339
- {
340
- autocompleteInTheMiddle = false,
341
- enforceOrder = false,
342
- autoSpaceAfter = false,
343
- includeDefault = true,
344
- } = {},
345
- ) => {
346
- if (!autocompleteInTheMiddle && !completingAtEnd && !context.explicit) {
347
- return null;
348
- }
349
- const priorityCompletionsOrdered = enforceOrder
350
- ? priorityCompletions.map((c, idx) => ({
351
- ...c,
352
- boost: -idx,
353
- }))
354
- : priorityCompletions;
355
- const priorityCompletionsAdjusted = autoSpaceAfter
356
- ? priorityCompletionsOrdered.map((c) =>
357
- !c.apply && !c.label.endsWith(" ")
358
- ? { ...c, label: c.label + " " }
359
- : c,
360
- )
361
- : priorityCompletionsOrdered;
362
- if (includeDefault) {
363
- /** @type {import("@codemirror/autocomplete").CompletionSection} */
364
- const miscSection = {
365
- name: "misc",
366
- header: () => {
367
- const element = document.createElement("hr");
368
- element.style.display = "list-item";
369
- element.style.margin = "2px 0";
370
- return element;
371
- },
372
- };
373
- const priorityLabels = new Set(priorityCompletions.map((c) => c.label));
374
- const defaultCompletionAdjusted = priorityCompletions
375
- ? defaultCompletionItems
376
- .filter((c) => !priorityLabels.has(c.label))
377
- .map((c) => ({ ...c, section: miscSection }))
378
- : defaultCompletionItems;
379
- return {
380
- from: completionStart,
381
- options: [...priorityCompletionsAdjusted, ...defaultCompletionAdjusted],
382
- };
383
- } else {
384
- return {
385
- from: completionStart,
386
- options: priorityCompletionsAdjusted,
387
- };
388
- }
389
- };
390
- const defaultCompletions = () => makeCompletions([]);
391
- const noCompletions = () => (context.explicit ? defaultCompletions() : null);
392
- const newExpressionCompletions = () =>
393
- makeCompletions([
394
- ...filterItems.map((item) =>
395
- makeCanonicalNameCompletion(item, {
396
- autoSpaceIf: (item) =>
397
- completingAtEnd && item.scoreType != kScoreTypeBoolean,
398
- }),
399
- ),
400
- ...sampleFunctionCompletionItems,
401
- ]);
402
- const variableCompletions = () => makeCompletions(variableCompletionItems);
403
- /** @param {import("./filters.mjs").ScoreFilterItem[]} items */
404
- const memberAccessCompletions = (items) =>
405
- makeCompletions(items.map(makeMemberAccessCompletion), {
406
- autocompleteInTheMiddle: true,
407
- includeDefault: false,
408
- });
409
- const logicalOpCompletions = () =>
410
- makeCompletions(["and", "or"].map(makeKeywordCompletion), {
411
- enforceOrder: true,
412
- autoSpaceAfter: completingAtEnd,
413
- });
414
- const descreteRelationCompletions = () =>
415
- makeCompletions(["==", "!=", "in", "not in"].map(makeKeywordCompletion), {
416
- enforceOrder: true,
417
- autoSpaceAfter: completingAtEnd,
418
- });
419
- const continuousRelationCompletions = () =>
420
- makeCompletions(
421
- ["<", "<=", ">", ">=", "==", "!="].map(makeKeywordCompletion),
422
- { enforceOrder: true, autoSpaceAfter: completingAtEnd },
423
- );
424
- const customRelationCompletions = () =>
425
- makeCompletions(
426
- ["<", "<=", ">", ">=", "==", "!=", "~="].map(makeKeywordCompletion),
427
- { enforceOrder: true, autoSpaceAfter: completingAtEnd },
428
- );
429
- /** @param {string[]} options */
430
- const rhsCompletions = (options) =>
431
- makeCompletions(options.map(makeLiteralCompletion));
432
-
433
- if (!prevToken(1)) return newExpressionCompletions();
434
-
435
- // Member access
436
- if (prevToken(1)?.text == ".") {
437
- const scorer = prevToken(2)?.text;
438
- if (scorer) {
439
- return memberAccessCompletions(getMemberScoreItems(filterItems, scorer));
440
- }
441
- }
442
-
443
- // Start of a function call or of a bracketed expression
444
- if (prevToken(1)?.text == "(") {
445
- if (prevToken(2)?.type == "mathFunction") return variableCompletions();
446
- if (prevToken(2)?.type == "sampleFunction") {
447
- // All sample functions expect a literal (a string to search for).
448
- return noCompletions();
449
- }
450
- // A grouping parenthesis, not a function call.
451
- return newExpressionCompletions();
452
- }
453
-
454
- // End of a function call or of a bracketed expression
455
- if (prevToken(1)?.text == ")") {
456
- // Don't try to guess: too unpredictable. Could continue with an arithmetic
457
- // operator (if constructing a complex expression), with a comparison (if
458
- // comparing function call result to something) or with a logical connector
459
- // (if a new subexpression is starting). Very hard to figure out what is
460
- // going on without an AST, which we don't have here.
461
- return noCompletions();
462
- }
463
-
464
- // Suggest relation based on variable type
465
- if (prevToken(1)?.type == "variable") {
466
- const scoreType = findFilterItem(1)?.scoreType;
467
- if ([kScoreTypePassFail, kScoreTypeCategorical].includes(scoreType))
468
- return descreteRelationCompletions();
469
- if (scoreType == kScoreTypeNumeric) return continuousRelationCompletions();
470
- if (scoreType == kScoreTypeOther) return customRelationCompletions();
471
- if (scoreType == kScoreTypeBoolean) return logicalOpCompletions();
472
- }
473
-
474
- // Suggest comparison RHS based on the LHS
475
- if (prevToken(1)?.type == "relation") {
476
- const item = findFilterItem(2);
477
- if (item) {
478
- if (item?.categories?.length) {
479
- return rhsCompletions(item.categories);
480
- } else {
481
- // Technically, it's possible to compare two scores, but comparison to a
482
- // constant is much more likely.
483
- return noCompletions();
484
- }
485
- } else {
486
- // Most likely: comparison starting from a constant, perhaps beginning of
487
- // a chain comparison.
488
- return variableCompletions();
489
- }
490
- }
491
-
492
- // Suggest connector to the next subexpression after `VARIABLE OP VALUE` subexpression finished.
493
- if (isLiteral(prevToken(1)) && prevToken(2)?.type == "relation") {
494
- return logicalOpCompletions();
495
- }
496
-
497
- // New subexpression begins after a logical connector.
498
- if (isLogicalOp(prevToken(1))) return newExpressionCompletions();
499
-
500
- // Something unusual is going on. We don't have any good guesses, but the user
501
- // can trigger completion manually with Ctrl+Space if they want.
502
- return noCompletions();
503
- }
504
-
505
- /**
506
- * @param {import("codemirror").EditorView} view
507
- * @param {import("./filters.mjs").FilterError | undefined} filterError
508
- * @returns {import("@codemirror/lint").Diagnostic[]}
509
- */
510
- function getLints(view, filterError) {
511
- if (!filterError) return [];
512
- return [
513
- {
514
- from: filterError.from || 0,
515
- to: filterError.to || view.state.doc.length,
516
- severity: filterError.severity,
517
- message: filterError.message,
518
- },
519
- ];
520
- }
521
-
522
- // Emulate `form-control` style to make it look like a text input.
523
- const editorTheme = EditorView.theme({
524
- "&": {
525
- fontSize: "inherit",
526
- color: "var(--inspect-input-foreground)",
527
- backgroundColor: "var(--inspect-input-background)",
528
- border: "1px solid var(--inspect-input-border)",
529
- borderRadius: "var(--bs-border-radius)",
530
- },
531
- ".cm-cursor.cm-cursor-primary": {
532
- borderLeftColor: "var(--bs-body-color)",
533
- },
534
- ".cm-selectionBackground": {
535
- backgroundColor: "var(--inspect-inactive-selection-background)",
536
- },
537
- "&.cm-focused > .cm-scroller > .cm-selectionLayer > .cm-selectionBackground":
538
- {
539
- backgroundColor: "var(--inspect-active-selection-background)",
540
- },
541
- "&.cm-focused": {
542
- outline: "none",
543
- borderColor: "var(--inspect-focus-border-color)",
544
- boxShadow: "var(--inspect-focus-border-shadow)",
545
- },
546
- ".filter-pending > &.cm-focused": {
547
- borderColor: "var(--inspect-focus-border-gray-color)",
548
- boxShadow: "var(--inspect-focus-border-gray-shadow)",
549
- },
550
- ".cm-tooltip": {
551
- backgroundColor: "var(--bs-light)",
552
- border: "1px solid var(--bs-border-color)",
553
- color: "var(--bs-body-color)",
554
- },
555
- ".cm-tooltip.cm-tooltip-autocomplete > ul > li": {
556
- color: "var(--bs-body-color)",
557
- },
558
- ".cm-tooltip.cm-tooltip-autocomplete > ul > li[aria-selected]": {
559
- backgroundColor: "var(--inspect-active-selection-background)",
560
- color: "var(--bs-body-color)",
561
- },
562
- ".cm-scroller": {
563
- overflow: "hidden",
564
- },
565
- });
566
-
567
- /**
568
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
569
- * @param {string} filterValue
570
- * @returns {FilteringResult}
571
- */
572
- const getFilteringResult = (evalDescriptor, filterValue) => {
573
- const { result, error } = filterSamples(
574
- evalDescriptor,
575
- evalDescriptor.samples,
576
- filterValue,
577
- );
578
- return { numSamples: result.length, error };
579
- };
580
-
581
- /**
582
- * Renders the Sample Filter Control
583
- *
584
- * @param {Object} props - The parameters for the component.
585
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} props.evalDescriptor
586
- * @param {(filter: import("../../Types.mjs").ScoreFilter) => void} props.filterChanged - Filter changed function
587
- * @param {import("../../Types.mjs").ScoreFilter} props.filter - Filter that is currently applied.
588
- * @returns {import("preact").JSX.Element | string} The TranscriptView component.
589
- */
590
- export const SampleFilter = ({ evalDescriptor, filter, filterChanged }) => {
591
- const editorRef = useRef(/** @type {HTMLElement|null} */ (null));
592
- const editorViewRef = useRef(
593
- /** @type {import("codemirror").EditorView|null} */ (null),
594
- );
595
- const linterCompartment = useRef(new Compartment());
596
- const autocompletionCompartment = useRef(new Compartment());
597
- const updateListenerCompartment = useRef(new Compartment());
598
- const filterItems = useMemo(
599
- () => scoreFilterItems(evalDescriptor),
600
- [evalDescriptor],
601
- );
602
- // Result of applying the filter expression in the editor, which might be
603
- // different from the active filter.
604
- const [filteringResultInstant, setFilteringResultInstant] = useState(
605
- /** @type {FilteringResult | null} */ (null),
606
- );
607
-
608
- /**
609
- * @param {FocusEvent} event
610
- * @param {import("codemirror").EditorView} view
611
- */
612
- const handleFocus = (event, view) => {
613
- if (event.isTrusted && view.state.doc.toString() === "") {
614
- setTimeout(() => startCompletion(view), 0);
615
- }
616
- };
617
-
618
- const makeAutocompletion = () =>
619
- autocompletion({
620
- override: [(context) => getCompletions(context, filterItems)],
621
- activateOnCompletion: (c) => c.label.endsWith(" "), // see autoSpaceAfter
622
- });
623
- const makeLinter = () =>
624
- // CodeMirror debounces the linter, so even instant error updates are not annoying
625
- linter((view) => getLints(view, filteringResultInstant?.error));
626
- const makeUpdateListener = () =>
627
- EditorView.updateListener.of((update) => {
628
- if (update.docChanged) {
629
- const newValue = update.state.doc.toString();
630
- const filteringResult = getFilteringResult(evalDescriptor, newValue);
631
- if (!filteringResult.error) {
632
- filterChanged({ value: newValue });
633
- }
634
- setFilteringResultInstant(filteringResult);
635
- }
636
- });
637
-
638
- // Initialize editor when component mounts
639
- useEffect(() => {
640
- editorViewRef.current?.destroy();
641
- editorViewRef.current = new EditorView({
642
- parent: editorRef.current,
643
- state: EditorState.create({
644
- doc: filter.value || "",
645
- extensions: [
646
- minimalSetup,
647
- bracketMatching(),
648
- editorTheme,
649
- EditorState.transactionFilter.of(ensureOneLine),
650
- updateListenerCompartment.current.of(makeUpdateListener()),
651
- EditorView.domEventHandlers({
652
- focus: handleFocus,
653
- }),
654
- language,
655
- syntaxHighlighting(highlightStyle),
656
- autocompletionCompartment.current.of(makeAutocompletion()),
657
- linterCompartment.current.of(makeLinter()),
658
- ],
659
- }),
660
- });
661
- return () => {
662
- editorViewRef.current?.destroy();
663
- };
664
- }, []);
665
-
666
- useEffect(() => {
667
- if (
668
- editorViewRef.current &&
669
- filter.value !== editorViewRef.current.state.doc.toString()
670
- ) {
671
- setFilteringResultInstant(
672
- getFilteringResult(evalDescriptor, filter.value),
673
- );
674
- editorViewRef.current.dispatch({
675
- changes: {
676
- from: 0,
677
- to: editorViewRef.current.state.doc.length,
678
- insert: filter.value || "",
679
- },
680
- });
681
- }
682
- }, [evalDescriptor, filter.value]);
683
-
684
- useEffect(() => {
685
- if (editorViewRef.current) {
686
- editorViewRef.current.dispatch({
687
- effects:
688
- updateListenerCompartment.current.reconfigure(makeUpdateListener()),
689
- });
690
- }
691
- }, [evalDescriptor]);
692
-
693
- useEffect(() => {
694
- if (editorViewRef.current) {
695
- editorViewRef.current.dispatch({
696
- effects:
697
- autocompletionCompartment.current.reconfigure(makeAutocompletion()),
698
- });
699
- }
700
- }, [filterItems]);
701
-
702
- useEffect(() => {
703
- if (editorViewRef.current) {
704
- editorViewRef.current.dispatch({
705
- effects: linterCompartment.current.reconfigure(makeLinter()),
706
- });
707
- }
708
- }, [filteringResultInstant?.error]);
709
-
710
- return html`
711
- <div style=${{ display: "flex" }}>
712
- <span
713
- class="sample-filter-label"
714
- style=${{
715
- alignSelf: "center",
716
- fontSize: FontSize.smaller,
717
- ...TextStyle.label,
718
- ...TextStyle.secondary,
719
- marginRight: "0.3em",
720
- marginLeft: "0.2em",
721
- }}
722
- >Filter:</span
723
- >
724
- <div
725
- ref=${editorRef}
726
- style=${{ width: "300px" }}
727
- class=${filteringResultInstant?.error ? ["filter-pending"] : []}
728
- ></div>
729
- <span
730
- class="bi bi-question-circle"
731
- style=${{
732
- position: "relative",
733
- marginLeft: "0.5em",
734
- cursor: "help",
735
- alignSelf: "center",
736
- }}
737
- data-tooltip=${filterTooltip}
738
- data-tooltip-position="bottom-left"
739
- ></span>
740
- </div>
741
- `;
742
- };
743
-
744
- const filterTooltip = `
745
- Filter samples by:
746
- • Scores
747
- • Input and target regex search: input_contains, target_contains
748
-
749
- Supported expressions:
750
- • Arithmetic: +, -, *, /, mod, ^
751
- • Comparison: <, <=, >, >=, ==, !=, including chain comparisons, e.g. “10 <= x < 20”
752
- • Boolean: and, or, not
753
- • Regex matching: ~= (case-sensitive)
754
- • Set operations: in, not in; e.g. “x in (2, 3, 5)”
755
- • Functions: min, max, abs, round, floor, ceil, sqrt, log, log2, log10
756
- `.trim();