inspect-ai 0.3.61__py3-none-any.whl → 0.3.63__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (430) hide show
  1. inspect_ai/_cli/eval.py +13 -0
  2. inspect_ai/_cli/main.py +1 -1
  3. inspect_ai/_cli/trace.py +8 -0
  4. inspect_ai/_cli/view.py +4 -0
  5. inspect_ai/_display/core/active.py +2 -3
  6. inspect_ai/_display/textual/widgets/transcript.py +15 -9
  7. inspect_ai/_eval/eval.py +4 -4
  8. inspect_ai/_eval/evalset.py +6 -6
  9. inspect_ai/_eval/task/error.py +10 -14
  10. inspect_ai/_eval/task/run.py +13 -8
  11. inspect_ai/_util/hash.py +1 -1
  12. inspect_ai/_util/transcript.py +11 -0
  13. inspect_ai/_view/www/.vscode/extensions.json +3 -0
  14. inspect_ai/_view/www/.vscode/settings.json +8 -0
  15. inspect_ai/_view/www/App.css +92 -29
  16. inspect_ai/_view/www/dist/assets/index.css +16636 -14674
  17. inspect_ai/_view/www/dist/assets/index.js +43585 -36122
  18. inspect_ai/_view/www/dist/index.html +1 -1
  19. inspect_ai/_view/www/index.html +2 -2
  20. inspect_ai/_view/www/log-schema.json +36 -19
  21. inspect_ai/_view/www/package.json +22 -4
  22. inspect_ai/_view/www/postcss.config.cjs +8 -9
  23. inspect_ai/_view/www/src/{App.mjs → App.tsx} +355 -365
  24. inspect_ai/_view/www/src/AppErrorBoundary.tsx +47 -0
  25. inspect_ai/_view/www/src/api/api-browser.ts +2 -2
  26. inspect_ai/_view/www/src/api/api-http.ts +3 -5
  27. inspect_ai/_view/www/src/api/api-vscode.ts +6 -6
  28. inspect_ai/_view/www/src/api/client-api.ts +4 -4
  29. inspect_ai/_view/www/src/api/index.ts +4 -4
  30. inspect_ai/_view/www/src/api/{Types.ts → types.ts} +25 -9
  31. inspect_ai/_view/www/src/appearance/colors.ts +9 -0
  32. inspect_ai/_view/www/src/appearance/fonts.ts +39 -0
  33. inspect_ai/_view/www/src/appearance/icons.ts +100 -0
  34. inspect_ai/_view/www/src/appearance/{Styles.mjs → styles.ts} +2 -32
  35. inspect_ai/_view/www/src/components/AnsiDisplay.tsx +198 -0
  36. inspect_ai/_view/www/src/components/AsciinemaPlayer.tsx +86 -0
  37. inspect_ai/_view/www/src/components/Card.css +60 -0
  38. inspect_ai/_view/www/src/components/Card.tsx +109 -0
  39. inspect_ai/_view/www/src/components/CopyButton.module.css +11 -0
  40. inspect_ai/_view/www/src/components/CopyButton.tsx +58 -0
  41. inspect_ai/_view/www/src/components/DownloadButton.css +4 -0
  42. inspect_ai/_view/www/src/components/DownloadButton.tsx +25 -0
  43. inspect_ai/_view/www/src/components/DownloadPanel.css +10 -0
  44. inspect_ai/_view/www/src/components/DownloadPanel.tsx +30 -0
  45. inspect_ai/_view/www/src/components/EmptyPanel.css +12 -0
  46. inspect_ai/_view/www/src/components/EmptyPanel.tsx +15 -0
  47. inspect_ai/_view/www/src/components/ErrorPanel.css +37 -0
  48. inspect_ai/_view/www/src/components/ErrorPanel.tsx +39 -0
  49. inspect_ai/_view/www/src/components/ExpandablePanel.css +40 -0
  50. inspect_ai/_view/www/src/components/ExpandablePanel.tsx +115 -0
  51. inspect_ai/_view/www/src/components/FindBand.css +49 -0
  52. inspect_ai/_view/www/src/components/FindBand.tsx +130 -0
  53. inspect_ai/_view/www/src/components/HumanBaselineView.css +41 -0
  54. inspect_ai/_view/www/src/components/HumanBaselineView.tsx +162 -0
  55. inspect_ai/_view/www/src/components/JsonPanel.css +20 -0
  56. inspect_ai/_view/www/src/components/JsonPanel.tsx +82 -0
  57. inspect_ai/_view/www/src/components/LabeledValue.css +20 -0
  58. inspect_ai/_view/www/src/components/LabeledValue.tsx +41 -0
  59. inspect_ai/_view/www/src/components/LargeModal.module.css +54 -0
  60. inspect_ai/_view/www/src/components/LargeModal.tsx +199 -0
  61. inspect_ai/_view/www/src/components/LightboxCarousel.css +95 -0
  62. inspect_ai/_view/www/src/components/LightboxCarousel.tsx +132 -0
  63. inspect_ai/_view/www/src/components/MarkdownDiv.css +3 -0
  64. inspect_ai/_view/www/src/components/MarkdownDiv.tsx +133 -0
  65. inspect_ai/_view/www/src/components/MessageBand.css +43 -0
  66. inspect_ai/_view/www/src/components/MessageBand.tsx +39 -0
  67. inspect_ai/_view/www/src/components/MorePopOver.tsx +67 -0
  68. inspect_ai/_view/www/src/components/NavPills.module.css +18 -0
  69. inspect_ai/_view/www/src/components/NavPills.tsx +99 -0
  70. inspect_ai/_view/www/src/components/ProgressBar.module.css +37 -0
  71. inspect_ai/_view/www/src/components/ProgressBar.tsx +22 -0
  72. inspect_ai/_view/www/src/components/TabSet.module.css +40 -0
  73. inspect_ai/_view/www/src/components/TabSet.tsx +200 -0
  74. inspect_ai/_view/www/src/components/ToolButton.css +3 -0
  75. inspect_ai/_view/www/src/components/ToolButton.tsx +27 -0
  76. inspect_ai/_view/www/src/components/VirtualList.module.css +19 -0
  77. inspect_ai/_view/www/src/components/VirtualList.tsx +292 -0
  78. inspect_ai/_view/www/src/{index.js → index.tsx} +45 -19
  79. inspect_ai/_view/www/src/{log → logfile}/remoteLogFile.ts +3 -7
  80. inspect_ai/_view/www/src/{utils/remoteZipFile.mjs → logfile/remoteZipFile.ts} +86 -80
  81. inspect_ai/_view/www/src/metadata/MetaDataGrid.tsx +83 -0
  82. inspect_ai/_view/www/src/metadata/MetaDataView.module.css +35 -0
  83. inspect_ai/_view/www/src/metadata/MetaDataView.tsx +95 -0
  84. inspect_ai/_view/www/src/metadata/MetadataGrid.module.css +15 -0
  85. inspect_ai/_view/www/src/metadata/RenderedContent.module.css +12 -0
  86. inspect_ai/_view/www/src/{components/RenderedContent/RenderedContent.mjs → metadata/RenderedContent.tsx} +92 -73
  87. inspect_ai/_view/www/src/metadata/types.ts +18 -0
  88. inspect_ai/_view/www/src/plan/DatasetDetailView.module.css +3 -0
  89. inspect_ai/_view/www/src/plan/DatasetDetailView.tsx +37 -0
  90. inspect_ai/_view/www/src/plan/DetailStep.module.css +9 -0
  91. inspect_ai/_view/www/src/plan/DetailStep.tsx +31 -0
  92. inspect_ai/_view/www/src/plan/PlanCard.tsx +28 -0
  93. inspect_ai/_view/www/src/plan/PlanDetailView.module.css +48 -0
  94. inspect_ai/_view/www/src/plan/PlanDetailView.tsx +309 -0
  95. inspect_ai/_view/www/src/plan/ScorerDetailView.module.css +3 -0
  96. inspect_ai/_view/www/src/plan/ScorerDetailView.tsx +30 -0
  97. inspect_ai/_view/www/src/plan/SolverDetailView.module.css +15 -0
  98. inspect_ai/_view/www/src/plan/SolverDetailView.tsx +32 -0
  99. inspect_ai/_view/www/src/samples/InlineSampleDisplay.module.css +8 -0
  100. inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx +53 -0
  101. inspect_ai/_view/www/src/samples/SampleDialog.tsx +122 -0
  102. inspect_ai/_view/www/src/samples/SampleDisplay.module.css +29 -0
  103. inspect_ai/_view/www/src/samples/SampleDisplay.tsx +326 -0
  104. inspect_ai/_view/www/src/samples/SampleSummaryView.module.css +24 -0
  105. inspect_ai/_view/www/src/samples/SampleSummaryView.tsx +175 -0
  106. inspect_ai/_view/www/src/samples/SamplesTools.tsx +60 -0
  107. inspect_ai/_view/www/src/samples/chat/ChatMessage.module.css +29 -0
  108. inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx +76 -0
  109. inspect_ai/_view/www/src/samples/chat/ChatMessageRenderer.tsx +60 -0
  110. inspect_ai/_view/www/src/samples/chat/ChatMessageRow.module.css +9 -0
  111. inspect_ai/_view/www/src/samples/chat/ChatMessageRow.tsx +57 -0
  112. inspect_ai/_view/www/src/samples/chat/ChatView.tsx +46 -0
  113. inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.module.css +4 -0
  114. inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.tsx +58 -0
  115. inspect_ai/_view/www/src/samples/chat/MessageContent.module.css +4 -0
  116. inspect_ai/_view/www/src/samples/chat/MessageContent.tsx +143 -0
  117. inspect_ai/_view/www/src/samples/chat/MessageContents.module.css +3 -0
  118. inspect_ai/_view/www/src/samples/chat/MessageContents.tsx +131 -0
  119. inspect_ai/_view/www/src/samples/chat/messages.ts +112 -0
  120. inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.tsx +145 -0
  121. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.module.css +14 -0
  122. inspect_ai/_view/www/src/samples/chat/tools/ToolInput.tsx +86 -0
  123. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.module.css +19 -0
  124. inspect_ai/_view/www/src/samples/chat/tools/ToolOutput.tsx +53 -0
  125. inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.module.css +4 -0
  126. inspect_ai/_view/www/src/samples/chat/tools/ToolTitle.tsx +18 -0
  127. inspect_ai/_view/www/src/samples/chat/tools/tool.ts +107 -0
  128. inspect_ai/_view/www/src/samples/descriptor/samplesDescriptor.tsx +363 -0
  129. inspect_ai/_view/www/src/samples/descriptor/score/BooleanScoreDescriptor.module.css +22 -0
  130. inspect_ai/_view/www/src/samples/descriptor/score/BooleanScoreDescriptor.tsx +26 -0
  131. inspect_ai/_view/www/src/samples/descriptor/score/CategoricalScoreDescriptor.tsx +18 -0
  132. inspect_ai/_view/www/src/samples/descriptor/score/NumericScoreDescriptor.tsx +27 -0
  133. inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.module.css +18 -0
  134. inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.tsx +71 -0
  135. inspect_ai/_view/www/src/samples/descriptor/score/OtherScoreDescriptor.tsx +20 -0
  136. inspect_ai/_view/www/src/samples/descriptor/score/PassFailScoreDescriptor.module.css +28 -0
  137. inspect_ai/_view/www/src/samples/descriptor/score/PassFailScoreDescriptor.tsx +81 -0
  138. inspect_ai/_view/www/src/samples/descriptor/score/ScoreDescriptor.tsx +99 -0
  139. inspect_ai/_view/www/src/samples/descriptor/types.ts +55 -0
  140. inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.module.css +19 -0
  141. inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.tsx +22 -0
  142. inspect_ai/_view/www/src/samples/error/SampleErrorView.module.css +17 -0
  143. inspect_ai/_view/www/src/samples/error/SampleErrorView.tsx +31 -0
  144. inspect_ai/_view/www/src/samples/error/error.ts +15 -0
  145. inspect_ai/_view/www/src/samples/list/SampleFooter.module.css +9 -0
  146. inspect_ai/_view/www/src/samples/list/SampleFooter.tsx +14 -0
  147. inspect_ai/_view/www/src/samples/list/SampleHeader.module.css +13 -0
  148. inspect_ai/_view/www/src/samples/list/SampleHeader.tsx +36 -0
  149. inspect_ai/_view/www/src/samples/list/SampleList.module.css +11 -0
  150. inspect_ai/_view/www/src/samples/list/SampleList.tsx +247 -0
  151. inspect_ai/_view/www/src/samples/list/SampleRow.module.css +33 -0
  152. inspect_ai/_view/www/src/samples/list/SampleRow.tsx +98 -0
  153. inspect_ai/_view/www/src/samples/list/SampleSeparator.module.css +6 -0
  154. inspect_ai/_view/www/src/samples/list/SampleSeparator.tsx +24 -0
  155. inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.module.css +9 -0
  156. inspect_ai/_view/www/src/samples/sample-tools/EpochFilter.tsx +51 -0
  157. inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.module.css +16 -0
  158. inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx +173 -0
  159. inspect_ai/_view/www/src/samples/sample-tools/SortFilter.module.css +9 -0
  160. inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx +182 -0
  161. inspect_ai/_view/www/src/samples/{tools/filters.mjs → sample-tools/filters.ts} +86 -81
  162. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.module.css +16 -0
  163. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx +288 -0
  164. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/completions.ts +346 -0
  165. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/language.ts +19 -0
  166. inspect_ai/_view/www/src/samples/sample-tools/sample-filter/tokenize.ts +97 -0
  167. inspect_ai/_view/www/src/samples/{SampleLimit.mjs → sampleLimit.ts} +3 -6
  168. inspect_ai/_view/www/src/samples/scores/SampleScoreView.module.css +53 -0
  169. inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx +168 -0
  170. inspect_ai/_view/www/src/samples/scores/SampleScores.module.css +5 -0
  171. inspect_ai/_view/www/src/samples/scores/SampleScores.tsx +37 -0
  172. inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.tsx +66 -0
  173. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx +51 -0
  174. inspect_ai/_view/www/src/samples/transcript/InfoEventView.module.css +3 -0
  175. inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx +54 -0
  176. inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx +48 -0
  177. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.module.css +6 -0
  178. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.tsx +36 -0
  179. inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css +43 -0
  180. inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx +223 -0
  181. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.module.css +23 -0
  182. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.tsx +108 -0
  183. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx +75 -0
  184. inspect_ai/_view/www/src/samples/transcript/SampleTranscript.tsx +22 -0
  185. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.module.css +15 -0
  186. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.tsx +100 -0
  187. inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx +171 -0
  188. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.module.css +19 -0
  189. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.tsx +133 -0
  190. inspect_ai/_view/www/src/samples/transcript/ToolEventView.module.css +10 -0
  191. inspect_ai/_view/www/src/samples/transcript/ToolEventView.tsx +91 -0
  192. inspect_ai/_view/www/src/samples/transcript/TranscriptView.module.css +49 -0
  193. inspect_ai/_view/www/src/samples/transcript/TranscriptView.tsx +449 -0
  194. inspect_ai/_view/www/src/samples/transcript/event/EventNav.module.css +5 -0
  195. inspect_ai/_view/www/src/samples/transcript/event/EventNav.tsx +43 -0
  196. inspect_ai/_view/www/src/samples/transcript/event/EventNavs.module.css +3 -0
  197. inspect_ai/_view/www/src/samples/transcript/event/EventNavs.tsx +38 -0
  198. inspect_ai/_view/www/src/samples/transcript/event/EventPanel.module.css +25 -0
  199. inspect_ai/_view/www/src/samples/transcript/event/EventPanel.tsx +190 -0
  200. inspect_ai/_view/www/src/samples/transcript/event/EventRow.module.css +13 -0
  201. inspect_ai/_view/www/src/samples/transcript/event/EventRow.tsx +32 -0
  202. inspect_ai/_view/www/src/samples/transcript/event/EventSection.module.css +8 -0
  203. inspect_ai/_view/www/src/samples/transcript/event/EventSection.tsx +29 -0
  204. inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.tsx +67 -0
  205. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx +274 -0
  206. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenders.module.css +10 -0
  207. inspect_ai/_view/www/src/samples/transcript/state/StateEventView.module.css +9 -0
  208. inspect_ai/_view/www/src/samples/transcript/state/{StateEventView.mjs → StateEventView.tsx} +148 -110
  209. inspect_ai/_view/www/src/samples/transcript/types.ts +58 -0
  210. inspect_ai/_view/www/src/types/log.d.ts +7 -4
  211. inspect_ai/_view/www/src/types/prism.d.ts +11 -0
  212. inspect_ai/_view/www/src/types.ts +71 -0
  213. inspect_ai/_view/www/src/usage/ModelTokenTable.tsx +22 -0
  214. inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css +24 -0
  215. inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx +95 -0
  216. inspect_ai/_view/www/src/usage/TokenTable.module.css +17 -0
  217. inspect_ai/_view/www/src/usage/TokenTable.tsx +91 -0
  218. inspect_ai/_view/www/src/usage/UsageCard.module.css +15 -0
  219. inspect_ai/_view/www/src/usage/UsageCard.tsx +67 -0
  220. inspect_ai/_view/www/src/utils/attachments.ts +42 -0
  221. inspect_ai/_view/www/src/utils/{Base64.mjs → base64.ts} +1 -6
  222. inspect_ai/_view/www/src/{components/Browser.mjs → utils/browser.ts} +0 -1
  223. inspect_ai/_view/www/src/utils/debugging.ts +28 -0
  224. inspect_ai/_view/www/src/utils/dom.ts +30 -0
  225. inspect_ai/_view/www/src/utils/format.ts +194 -0
  226. inspect_ai/_view/www/src/utils/git.ts +7 -0
  227. inspect_ai/_view/www/src/utils/html.ts +6 -0
  228. inspect_ai/_view/www/src/utils/http.ts +14 -0
  229. inspect_ai/_view/www/src/utils/{Path.mjs → path.ts} +2 -9
  230. inspect_ai/_view/www/src/utils/{Print.mjs → print.ts} +34 -26
  231. inspect_ai/_view/www/src/utils/queue.ts +51 -0
  232. inspect_ai/_view/www/src/utils/sync.ts +114 -0
  233. inspect_ai/_view/www/src/utils/{Type.mjs → type.ts} +3 -6
  234. inspect_ai/_view/www/src/utils/vscode.ts +13 -0
  235. inspect_ai/_view/www/src/workspace/WorkSpace.tsx +324 -0
  236. inspect_ai/_view/www/src/workspace/WorkSpaceView.module.css +33 -0
  237. inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx +160 -0
  238. inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.module.css +3 -0
  239. inspect_ai/_view/www/src/workspace/error/TaskErrorPanel.tsx +28 -0
  240. inspect_ai/_view/www/src/workspace/navbar/Navbar.module.css +54 -0
  241. inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx +68 -0
  242. inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.module.css +52 -0
  243. inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx +113 -0
  244. inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.module.css +67 -0
  245. inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx +156 -0
  246. inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.module.css +28 -0
  247. inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx +222 -0
  248. inspect_ai/_view/www/src/workspace/navbar/StatusPanel.module.css +14 -0
  249. inspect_ai/_view/www/src/workspace/navbar/StatusPanel.tsx +61 -0
  250. inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.module.css +15 -0
  251. inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.tsx +71 -0
  252. inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.module.css +5 -0
  253. inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.tsx +56 -0
  254. inspect_ai/_view/www/src/workspace/sidebar/Sidebar.module.css +68 -0
  255. inspect_ai/_view/www/src/workspace/sidebar/Sidebar.tsx +85 -0
  256. inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.module.css +29 -0
  257. inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.tsx +95 -0
  258. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.module.css +23 -0
  259. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx +41 -0
  260. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.module.css +35 -0
  261. inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx +61 -0
  262. inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx +80 -0
  263. inspect_ai/_view/www/src/workspace/tabs/JsonTab.module.css +5 -0
  264. inspect_ai/_view/www/src/workspace/tabs/JsonTab.tsx +46 -0
  265. inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx +204 -0
  266. inspect_ai/_view/www/src/workspace/tabs/grouping.ts +195 -0
  267. inspect_ai/_view/www/src/workspace/tabs/types.ts +19 -0
  268. inspect_ai/_view/www/src/workspace/types.ts +10 -0
  269. inspect_ai/_view/www/tsconfig.json +23 -9
  270. inspect_ai/_view/www/vite.config.js +8 -17
  271. inspect_ai/_view/www/yarn.lock +627 -556
  272. inspect_ai/dataset/_dataset.py +36 -0
  273. inspect_ai/dataset/_sources/csv.py +8 -0
  274. inspect_ai/dataset/_sources/file.py +4 -0
  275. inspect_ai/dataset/_sources/hf.py +11 -1
  276. inspect_ai/dataset/_sources/json.py +8 -0
  277. inspect_ai/log/_log.py +3 -6
  278. inspect_ai/log/_message.py +1 -1
  279. inspect_ai/log/_recorders/eval.py +1 -1
  280. inspect_ai/log/_recorders/json.py +5 -7
  281. inspect_ai/model/_call_tools.py +2 -1
  282. inspect_ai/model/_chat_message.py +27 -0
  283. inspect_ai/model/_conversation.py +10 -3
  284. inspect_ai/model/_generate_config.py +6 -0
  285. inspect_ai/model/_model.py +74 -0
  286. inspect_ai/model/_openai.py +33 -1
  287. inspect_ai/model/_providers/anthropic.py +12 -0
  288. inspect_ai/model/_providers/groq.py +4 -0
  289. inspect_ai/model/_providers/openai.py +21 -9
  290. inspect_ai/model/_providers/openai_o1.py +3 -5
  291. inspect_ai/model/_providers/openrouter.py +86 -0
  292. inspect_ai/model/_providers/providers.py +12 -1
  293. inspect_ai/model/_reasoning.py +17 -0
  294. inspect_ai/scorer/_answer.py +7 -7
  295. inspect_ai/scorer/_classification.py +34 -18
  296. inspect_ai/scorer/_common.py +2 -8
  297. inspect_ai/solver/_basic_agent.py +19 -9
  298. inspect_ai/solver/_multiple_choice.py +24 -9
  299. inspect_ai/tool/__init__.py +2 -0
  300. inspect_ai/tool/{beta → _tools}/_computer/_computer.py +2 -5
  301. inspect_ai/tool/{beta → _tools}/_computer/_resources/Dockerfile +4 -0
  302. inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/Code/User/globalStorage/state.vscdb +0 -0
  303. inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/Code/User/settings.json +3 -0
  304. inspect_ai/tool/_tools/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml +61 -0
  305. inspect_ai/tool/_tools/_computer/_resources/image_home_dir/Desktop/Terminal.desktop +10 -0
  306. inspect_ai/tool/_tools/_computer/_resources/tool/__init__.py +0 -0
  307. inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_x11_client.py +1 -1
  308. inspect_ai/tool/_tools/_computer/_resources/tool/requirements.txt +0 -0
  309. inspect_ai/tool/_tools/_execute.py +8 -2
  310. inspect_ai/tool/beta.py +3 -0
  311. inspect_ai/util/_sandbox/docker/docker.py +32 -85
  312. inspect_ai/util/_sandbox/self_check.py +124 -16
  313. {inspect_ai-0.3.61.dist-info → inspect_ai-0.3.63.dist-info}/METADATA +2 -1
  314. inspect_ai-0.3.63.dist-info/RECORD +618 -0
  315. inspect_ai/_view/www/src/Register.mjs +0 -3
  316. inspect_ai/_view/www/src/Types.mjs +0 -38
  317. inspect_ai/_view/www/src/appearance/Colors.mjs +0 -27
  318. inspect_ai/_view/www/src/appearance/Fonts.mjs +0 -66
  319. inspect_ai/_view/www/src/appearance/Icons.mjs +0 -240
  320. inspect_ai/_view/www/src/components/AnsiDisplay.mjs +0 -184
  321. inspect_ai/_view/www/src/components/AppErrorBoundary.mjs +0 -34
  322. inspect_ai/_view/www/src/components/AsciiCinemaPlayer.mjs +0 -74
  323. inspect_ai/_view/www/src/components/Card.mjs +0 -126
  324. inspect_ai/_view/www/src/components/ChatView.mjs +0 -418
  325. inspect_ai/_view/www/src/components/CopyButton.mjs +0 -48
  326. inspect_ai/_view/www/src/components/Dialog.mjs +0 -61
  327. inspect_ai/_view/www/src/components/DownloadButton.mjs +0 -15
  328. inspect_ai/_view/www/src/components/DownloadPanel.mjs +0 -29
  329. inspect_ai/_view/www/src/components/EmptyPanel.mjs +0 -23
  330. inspect_ai/_view/www/src/components/ErrorPanel.mjs +0 -66
  331. inspect_ai/_view/www/src/components/ExpandablePanel.mjs +0 -136
  332. inspect_ai/_view/www/src/components/FindBand.mjs +0 -157
  333. inspect_ai/_view/www/src/components/HumanBaselineView.mjs +0 -168
  334. inspect_ai/_view/www/src/components/JsonPanel.mjs +0 -61
  335. inspect_ai/_view/www/src/components/LabeledValue.mjs +0 -32
  336. inspect_ai/_view/www/src/components/LargeModal.mjs +0 -190
  337. inspect_ai/_view/www/src/components/LightboxCarousel.mjs +0 -217
  338. inspect_ai/_view/www/src/components/MarkdownDiv.mjs +0 -118
  339. inspect_ai/_view/www/src/components/MessageBand.mjs +0 -48
  340. inspect_ai/_view/www/src/components/MessageContent.mjs +0 -111
  341. inspect_ai/_view/www/src/components/MetaDataGrid.mjs +0 -92
  342. inspect_ai/_view/www/src/components/MetaDataView.mjs +0 -109
  343. inspect_ai/_view/www/src/components/MorePopOver.mjs +0 -50
  344. inspect_ai/_view/www/src/components/NavPills.mjs +0 -63
  345. inspect_ai/_view/www/src/components/ProgressBar.mjs +0 -51
  346. inspect_ai/_view/www/src/components/RenderedContent/ChatMessageRenderer.mjs +0 -54
  347. inspect_ai/_view/www/src/components/RenderedContent/Types.mjs +0 -19
  348. inspect_ai/_view/www/src/components/TabSet.mjs +0 -184
  349. inspect_ai/_view/www/src/components/ToolButton.mjs +0 -16
  350. inspect_ai/_view/www/src/components/Tools.mjs +0 -376
  351. inspect_ai/_view/www/src/components/VirtualList.mjs +0 -280
  352. inspect_ai/_view/www/src/components/ansi-output.js +0 -932
  353. inspect_ai/_view/www/src/json/JsonTab.mjs +0 -48
  354. inspect_ai/_view/www/src/log-reader/Log-Reader.mjs +0 -25
  355. inspect_ai/_view/www/src/log-reader/Native-Log-Reader.mjs +0 -13
  356. inspect_ai/_view/www/src/log-reader/Open-AI-Log-Reader.mjs +0 -263
  357. inspect_ai/_view/www/src/navbar/Navbar.mjs +0 -418
  358. inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +0 -175
  359. inspect_ai/_view/www/src/plan/PlanCard.mjs +0 -418
  360. inspect_ai/_view/www/src/samples/SampleDialog.mjs +0 -123
  361. inspect_ai/_view/www/src/samples/SampleDisplay.mjs +0 -516
  362. inspect_ai/_view/www/src/samples/SampleError.mjs +0 -99
  363. inspect_ai/_view/www/src/samples/SampleList.mjs +0 -427
  364. inspect_ai/_view/www/src/samples/SampleScoreView.mjs +0 -172
  365. inspect_ai/_view/www/src/samples/SampleScores.mjs +0 -34
  366. inspect_ai/_view/www/src/samples/SampleTranscript.mjs +0 -20
  367. inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +0 -771
  368. inspect_ai/_view/www/src/samples/SamplesTab.mjs +0 -399
  369. inspect_ai/_view/www/src/samples/SamplesTools.mjs +0 -64
  370. inspect_ai/_view/www/src/samples/tools/EpochFilter.mjs +0 -38
  371. inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +0 -756
  372. inspect_ai/_view/www/src/samples/tools/SelectScorer.mjs +0 -141
  373. inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +0 -151
  374. inspect_ai/_view/www/src/samples/transcript/ApprovalEventView.mjs +0 -71
  375. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs +0 -44
  376. inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs +0 -271
  377. inspect_ai/_view/www/src/samples/transcript/EventRow.mjs +0 -46
  378. inspect_ai/_view/www/src/samples/transcript/EventSection.mjs +0 -33
  379. inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs +0 -59
  380. inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs +0 -44
  381. inspect_ai/_view/www/src/samples/transcript/LoggerEventView.mjs +0 -32
  382. inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs +0 -216
  383. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.mjs +0 -107
  384. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.mjs +0 -74
  385. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.mjs +0 -100
  386. inspect_ai/_view/www/src/samples/transcript/StepEventView.mjs +0 -187
  387. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.mjs +0 -133
  388. inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs +0 -88
  389. inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs +0 -459
  390. inspect_ai/_view/www/src/samples/transcript/Types.mjs +0 -44
  391. inspect_ai/_view/www/src/samples/transcript/state/StateDiffView.mjs +0 -53
  392. inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.mjs +0 -254
  393. inspect_ai/_view/www/src/sidebar/Sidebar.mjs +0 -418
  394. inspect_ai/_view/www/src/usage/ModelTokenTable.mjs +0 -72
  395. inspect_ai/_view/www/src/usage/UsageCard.mjs +0 -159
  396. inspect_ai/_view/www/src/utils/Format.mjs +0 -260
  397. inspect_ai/_view/www/src/utils/Git.mjs +0 -12
  398. inspect_ai/_view/www/src/utils/Html.mjs +0 -21
  399. inspect_ai/_view/www/src/utils/attachments.mjs +0 -31
  400. inspect_ai/_view/www/src/utils/debugging.mjs +0 -23
  401. inspect_ai/_view/www/src/utils/http.mjs +0 -18
  402. inspect_ai/_view/www/src/utils/queue.mjs +0 -67
  403. inspect_ai/_view/www/src/utils/sync.mjs +0 -101
  404. inspect_ai/_view/www/src/workspace/TaskErrorPanel.mjs +0 -17
  405. inspect_ai/_view/www/src/workspace/WorkSpace.mjs +0 -516
  406. inspect_ai/tool/beta/__init__.py +0 -5
  407. inspect_ai-0.3.61.dist-info/RECORD +0 -476
  408. /inspect_ai/{tool/beta/_computer/_resources/tool/__init__.py → _view/www/src/components/MorePopOver.css} +0 -0
  409. /inspect_ai/_view/www/src/{constants.mjs → constants.ts} +0 -0
  410. /inspect_ai/{tool/beta/_computer/_resources/tool/requirements.txt → _view/www/src/workspace/tabs/InfoTab.module.css} +0 -0
  411. /inspect_ai/tool/{beta → _tools}/_computer/__init__.py +0 -0
  412. /inspect_ai/tool/{beta → _tools}/_computer/_common.py +0 -0
  413. /inspect_ai/tool/{beta → _tools}/_computer/_computer_split.py +0 -0
  414. /inspect_ai/tool/{beta → _tools}/_computer/_resources/README.md +0 -0
  415. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/entrypoint.sh +0 -0
  416. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/novnc_startup.sh +0 -0
  417. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/x11vnc_startup.sh +0 -0
  418. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/xfce_startup.sh +0 -0
  419. /inspect_ai/tool/{beta → _tools}/_computer/_resources/entrypoint/xvfb_startup.sh +0 -0
  420. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml +0 -0
  421. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop +0 -0
  422. /inspect_ai/tool/{beta → _tools}/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop +0 -0
  423. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_logger.py +0 -0
  424. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_run.py +0 -0
  425. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_tool_result.py +0 -0
  426. /inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/computer_tool.py +0 -0
  427. {inspect_ai-0.3.61.dist-info → inspect_ai-0.3.63.dist-info}/LICENSE +0 -0
  428. {inspect_ai-0.3.61.dist-info → inspect_ai-0.3.63.dist-info}/WHEEL +0 -0
  429. {inspect_ai-0.3.61.dist-info → inspect_ai-0.3.63.dist-info}/entry_points.txt +0 -0
  430. {inspect_ai-0.3.61.dist-info → inspect_ai-0.3.63.dist-info}/top_level.txt +0 -0
@@ -1,756 +0,0 @@
1
- import { autocompletion, startCompletion } from "@codemirror/autocomplete";
2
- import {
3
- HighlightStyle,
4
- StreamLanguage,
5
- StringStream,
6
- bracketMatching,
7
- syntaxHighlighting,
8
- } from "@codemirror/language";
9
- import { linter } from "@codemirror/lint";
10
- import { Compartment, EditorState } from "@codemirror/state";
11
- import { tags } from "@lezer/highlight";
12
- import { EditorView, minimalSetup } from "codemirror";
13
- import { html } from "htm/preact";
14
- import { useEffect, useMemo, useRef, useState } from "preact/hooks";
15
- import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
16
- import { filterSamples, scoreFilterItems } from "./filters.mjs";
17
- import {
18
- kScoreTypeBoolean,
19
- kScoreTypeCategorical,
20
- kScoreTypeNumeric,
21
- kScoreTypeOther,
22
- kScoreTypePassFail,
23
- } from "../../constants.mjs";
24
-
25
- /**
26
- * @typedef {Object} Token
27
- * @property {string} type
28
- * @property {string} text
29
- * @property {number} from
30
- * @property {number} to
31
- */
32
-
33
- /**
34
- * @typedef {Object} FilteringResult
35
- * @property {number} numSamples - The number of samples that match the filter.
36
- * @property {import("./filters.mjs").FilterError | undefined} error - The error in the filter expression, if any.
37
- */
38
-
39
- const KEYWORDS = ["and", "or", "not", "in", "not in", "mod"];
40
-
41
- const MATH_FUNCTIONS = [
42
- ["min", "Minimum of two or more values"],
43
- ["max", "Maximum of two or more values"],
44
- ["abs", "Absolute value"],
45
- ["round", "Round to the nearest integer"],
46
- ["floor", "Round down to the nearest integer"],
47
- ["ceil", "Round up to the nearest integer"],
48
- ["sqrt", "Square root"],
49
- ["log", "Natural logarithm"],
50
- ["log2", "Base 2 logarithm"],
51
- ["log10", "Base 10 logarithm"],
52
- ];
53
-
54
- const SAMPLE_FUNCTIONS = [
55
- ["input_contains", "Checks if input contains a regular expression"],
56
- ["target_contains", "Checks if target contains a regular expression"],
57
- ];
58
-
59
- /**
60
- * Makes sure that the filter expression is a single line.
61
- * @param {import("@codemirror/state").Transaction} tr - The transaction to join lines in.
62
- * @returns {import("@codemirror/state").TransactionSpec} The transaction with joined lines, if any.
63
- */
64
- function ensureOneLine(tr) {
65
- const newDoc = tr.newDoc.toString();
66
- if (newDoc.includes("\n")) {
67
- if (tr.isUserEvent("input.paste")) {
68
- const newDocAdjusted = newDoc.replace(/\n/g, " ").trim();
69
- return {
70
- changes: {
71
- from: 0,
72
- to: tr.startState.doc.length,
73
- insert: newDocAdjusted,
74
- },
75
- };
76
- } else {
77
- return {};
78
- }
79
- }
80
- return tr;
81
- }
82
-
83
- const highlightStyle = HighlightStyle.define([
84
- { tag: tags.string, class: "token string" },
85
- { tag: tags.number, class: "token number" },
86
- { tag: tags.keyword, class: "token keyword" },
87
- ]);
88
-
89
- /** @param {string} word */
90
- function countSpaces(word) {
91
- return word.split(" ").length - 1;
92
- }
93
-
94
- const nextToken = (() => {
95
- const wordsRe = (words) => new RegExp(`^(${words.join("|")})\\b`);
96
- const keywordsRe = wordsRe(
97
- // Sort to make sure "not in" is matched before "not".
98
- KEYWORDS.sort((a, b) => countSpaces(b) - countSpaces(a)),
99
- );
100
- const mathFunctionsRe = wordsRe(MATH_FUNCTIONS.map(([label]) => label));
101
- const sampleFunctionsRe = wordsRe(SAMPLE_FUNCTIONS.map(([label]) => label));
102
-
103
- /** @param {import("@codemirror/language").StringStream} stream */
104
- return function (stream) {
105
- if (stream.match(/"[^"]*"/)) return "string";
106
- if (stream.match(/"[^"]*/)) return "unterminatedString";
107
- if (stream.match(/^(-|\+)?\d+(\.\d+)?/)) return "number";
108
- if (stream.match(keywordsRe)) return "keyword";
109
- if (stream.match(mathFunctionsRe)) return "mathFunction";
110
- if (stream.match(sampleFunctionsRe)) return "sampleFunction";
111
- if (stream.match(/^[a-zA-Z_][a-zA-Z0-9_]*/)) return "variable";
112
- if (stream.match(/^(==|!=|<=|>=|<|>|~=)/)) return "relation";
113
- if (stream.match(/^(=|!|~)/)) return "miscOperator"; // recognize relations while typing; not valid syntax per se
114
- if (stream.match(/^(\+|-|\*|\/|\^|\(|\)|,|\.)/)) return "miscOperator";
115
- stream.next();
116
- return null;
117
- };
118
- })();
119
-
120
- const language = StreamLanguage.define({
121
- token: nextToken,
122
- tokenTable: {
123
- string: tags.string,
124
- unterminatedString: tags.string,
125
- number: tags.number,
126
- keyword: tags.keyword,
127
- mathFunction: tags.function(tags.variableName),
128
- sampleFunction: tags.function(tags.variableName),
129
- variable: tags.variableName,
130
- relation: tags.operator,
131
- miscOperator: tags.operator,
132
- },
133
- });
134
-
135
- /**
136
- * @param {string} input
137
- * @returns {Token[]}
138
- */
139
- function tokenize(input) {
140
- const tokens = [];
141
- const stream = new StringStream(input, 0, 0);
142
- while (stream.pos < input.length) {
143
- const from = stream.pos;
144
- const type = nextToken(stream);
145
- if (type) {
146
- tokens.push({
147
- type,
148
- text: input.slice(from, stream.pos),
149
- from,
150
- to: stream.pos,
151
- });
152
- }
153
- }
154
- return tokens;
155
- }
156
-
157
- /**
158
- * @param {import("./filters.mjs").ScoreFilterItem[]} filterItems
159
- * @param {string} scorer
160
- * @returns {import("./filters.mjs").ScoreFilterItem[]}
161
- */
162
- function getMemberScoreItems(filterItems, scorer) {
163
- return filterItems.filter((item) =>
164
- item?.qualifiedName?.startsWith(`${scorer}.`),
165
- );
166
- }
167
-
168
- /**
169
- * Generates completions for the filter expression. The main goal is to make the
170
- * sample filter intuitive for beginners and to provide a smooth experience for
171
- * simple cases. To this end, we proactively try to suggest the next step of the
172
- * expression, in a wizard-style fashion. This logic is primarily intended to
173
- * support unsophisticated expressions of the form
174
- * SUBEXPR and/or SUBEXPR or/not SUBEXPR ...
175
- * where each SUBEXPR is
176
- * VARIABLE ==/!=/</>/in/... VALUE
177
- * and VALUE is a literal (string, number, etc.)
178
- * It does support some expressions more complex than that, but the completion
179
- * algorithm is not intended to be comprehensive. This is why we usually add
180
- * default completions to the list in case our guess was off.
181
- *
182
- * @param {import("@codemirror/autocomplete").CompletionContext} context
183
- * @param {import("../../samples/tools/filters.mjs").ScoreFilterItem[]} filterItems
184
- * @returns {import("@codemirror/autocomplete").CompletionResult}
185
- */
186
- function getCompletions(context, filterItems) {
187
- /** @param {Token} token */
188
- const isLiteral = (token) =>
189
- ["string", "unterminatedString", "number"].includes(token?.type);
190
- /** @param {Token} token */
191
- const isLogicalOp = (token) => ["and", "or", "not"].includes(token?.text);
192
-
193
- /**
194
- * With most tokens we complete only after a space, but for sometimes it makes
195
- * sense to start autocompletion as soon as the token was typed.
196
- * @param {Token} token
197
- */
198
- const autocompleteImmediatelyAfter = (token) =>
199
- ["(", "."].includes(token?.text);
200
-
201
- /**
202
- * @param {import("codemirror").EditorView} view
203
- * @param {import("@codemirror/autocomplete").Completion} completion
204
- * @param {number} from
205
- * @param {number} to
206
- */
207
- function applyWithCall(view, completion, from, to) {
208
- view.dispatch({
209
- changes: { from, to, insert: `${completion.label}()` },
210
- selection: { anchor: from + completion.label.length + 1 },
211
- });
212
- }
213
-
214
- /** @type {(k: string) => import("@codemirror/autocomplete").Completion} */
215
- const makeKeywordCompletion = (k) => ({
216
- label: k,
217
- type: "keyword",
218
- boost: -20,
219
- });
220
- /** @type {([label, info]: [string, string]) => import("@codemirror/autocomplete").Completion} */
221
- const makeMathFunctionCompletion = ([label, info]) => ({
222
- label,
223
- type: "function",
224
- info,
225
- apply: applyWithCall,
226
- boost: -10,
227
- });
228
- /** @type {([label, info]: [string, string]) => import("@codemirror/autocomplete").Completion} */
229
- const makeSampleFunctionCompletion = ([label, info]) => ({
230
- label,
231
- type: "function",
232
- info,
233
- apply: applyWithCall,
234
- boost: 0,
235
- });
236
- /** @type {(k: string) => import("@codemirror/autocomplete").Completion} */
237
- const makeLiteralCompletion = (k) => ({
238
- label: k,
239
- type: "text",
240
- boost: 10,
241
- });
242
- /**
243
- * @param {import("./filters.mjs").ScoreFilterItem} item
244
- * @param {Object} [props]
245
- * @param {(item: import("./filters.mjs").ScoreFilterItem) => boolean} [props.autoSpaceIf] - Similar to `autoSpaceAfter`, but conditional.
246
- * @returns {import("@codemirror/autocomplete").Completion}
247
- */
248
- const makeCanonicalNameCompletion = (
249
- item,
250
- { autoSpaceIf = () => false } = {},
251
- ) => ({
252
- label: item.canonicalName + (autoSpaceIf(item) ? " " : ""),
253
- type: "variable",
254
- info: item.tooltip,
255
- boost: 20,
256
- });
257
- /** @type {(item: import("./filters.mjs").ScoreFilterItem) => import("@codemirror/autocomplete").Completion} */
258
- const makeMemberAccessCompletion = (item) => ({
259
- label: item.qualifiedName.split(".")[1],
260
- type: "variable",
261
- info: item.tooltip,
262
- boost: 20,
263
- });
264
-
265
- const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
266
- const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
267
- makeMathFunctionCompletion,
268
- );
269
- const sampleFunctionCompletionItems = SAMPLE_FUNCTIONS.map(
270
- makeSampleFunctionCompletion,
271
- );
272
- const variableCompletionItems = filterItems.map((item) =>
273
- makeCanonicalNameCompletion(item),
274
- );
275
-
276
- const defaultCompletionItems = [
277
- ...keywordCompletionItems,
278
- ...mathFunctionCompletionItems,
279
- ...sampleFunctionCompletionItems,
280
- ...variableCompletionItems,
281
- ];
282
-
283
- const doc = context.state.doc;
284
- const input = doc.toString().slice(0, context.pos);
285
- const tokens = tokenize(input);
286
- const lastToken = tokens[tokens.length - 1];
287
- const isCompletionInsideToken =
288
- lastToken &&
289
- context.pos == lastToken.to &&
290
- !autocompleteImmediatelyAfter(lastToken);
291
- const currentTokenIndex = isCompletionInsideToken
292
- ? tokens.length - 1
293
- : tokens.length; // `currentToken` is undefined when we are not inside a token
294
-
295
- /**
296
- * Returns nth token back away from the current token. Note that `prevToken(0)`
297
- * is always reserved for the current token, whether it exists or not.
298
- * @param {number} index
299
- * @returns {Token | undefined}
300
- */
301
- const prevToken = (index) => tokens[currentTokenIndex - index];
302
-
303
- const currentToken = prevToken(0);
304
- const completionStart = currentToken ? currentToken.from : context.pos;
305
- const completingAtEnd = context.pos == doc.length;
306
-
307
- /**
308
- * @param {number} endIndex
309
- * @returns {import("../../samples/tools/filters.mjs").ScoreFilterItem | undefined}
310
- */
311
- const findFilterItem = (endIndex) => {
312
- if (prevToken(endIndex)?.type == "variable") {
313
- let name = prevToken(endIndex).text;
314
- let i = endIndex;
315
- while (prevToken(i + 1)?.text == ".") {
316
- if (prevToken(i + 2)?.type == "variable") {
317
- name = prevToken(i + 2).text + "." + name;
318
- i += 2;
319
- } else {
320
- break;
321
- }
322
- }
323
- return filterItems.find((item) => item.canonicalName == name);
324
- }
325
- return undefined;
326
- };
327
-
328
- /**
329
- * @param {import("@codemirror/autocomplete").Completion[]} priorityCompletions
330
- * @param {Object} props
331
- * @param {boolean} [props.autocompleteInTheMiddle] - If true, completion would be shown automatically even when editing in the middle of the expression.
332
- * @param {boolean} [props.enforceOrder] - If true, the priorityCompletions are shown in the order they are provided.
333
- * @param {boolean} [props.autoSpaceAfter] - If true, space is inserted after priorityCompletions. When a user accepts a completion with a space, another completion is suggested immediately (see `activateOnCompletion`). Use when fairly certain that the expression continues.
334
- * @param {boolean} [props.includeDefault] - If true, the default completions are included after the priority completions.
335
- * @returns {import("@codemirror/autocomplete").CompletionResult}
336
- */
337
- const makeCompletions = (
338
- priorityCompletions,
339
- {
340
- autocompleteInTheMiddle = false,
341
- enforceOrder = false,
342
- autoSpaceAfter = false,
343
- includeDefault = true,
344
- } = {},
345
- ) => {
346
- if (!autocompleteInTheMiddle && !completingAtEnd && !context.explicit) {
347
- return null;
348
- }
349
- const priorityCompletionsOrdered = enforceOrder
350
- ? priorityCompletions.map((c, idx) => ({
351
- ...c,
352
- boost: -idx,
353
- }))
354
- : priorityCompletions;
355
- const priorityCompletionsAdjusted = autoSpaceAfter
356
- ? priorityCompletionsOrdered.map((c) =>
357
- !c.apply && !c.label.endsWith(" ")
358
- ? { ...c, label: c.label + " " }
359
- : c,
360
- )
361
- : priorityCompletionsOrdered;
362
- if (includeDefault) {
363
- /** @type {import("@codemirror/autocomplete").CompletionSection} */
364
- const miscSection = {
365
- name: "misc",
366
- header: () => {
367
- const element = document.createElement("hr");
368
- element.style.display = "list-item";
369
- element.style.margin = "2px 0";
370
- return element;
371
- },
372
- };
373
- const priorityLabels = new Set(priorityCompletions.map((c) => c.label));
374
- const defaultCompletionAdjusted = priorityCompletions
375
- ? defaultCompletionItems
376
- .filter((c) => !priorityLabels.has(c.label))
377
- .map((c) => ({ ...c, section: miscSection }))
378
- : defaultCompletionItems;
379
- return {
380
- from: completionStart,
381
- options: [...priorityCompletionsAdjusted, ...defaultCompletionAdjusted],
382
- };
383
- } else {
384
- return {
385
- from: completionStart,
386
- options: priorityCompletionsAdjusted,
387
- };
388
- }
389
- };
390
- const defaultCompletions = () => makeCompletions([]);
391
- const noCompletions = () => (context.explicit ? defaultCompletions() : null);
392
- const newExpressionCompletions = () =>
393
- makeCompletions([
394
- ...filterItems.map((item) =>
395
- makeCanonicalNameCompletion(item, {
396
- autoSpaceIf: (item) =>
397
- completingAtEnd && item.scoreType != kScoreTypeBoolean,
398
- }),
399
- ),
400
- ...sampleFunctionCompletionItems,
401
- ]);
402
- const variableCompletions = () => makeCompletions(variableCompletionItems);
403
- /** @param {import("./filters.mjs").ScoreFilterItem[]} items */
404
- const memberAccessCompletions = (items) =>
405
- makeCompletions(items.map(makeMemberAccessCompletion), {
406
- autocompleteInTheMiddle: true,
407
- includeDefault: false,
408
- });
409
- const logicalOpCompletions = () =>
410
- makeCompletions(["and", "or"].map(makeKeywordCompletion), {
411
- enforceOrder: true,
412
- autoSpaceAfter: completingAtEnd,
413
- });
414
- const descreteRelationCompletions = () =>
415
- makeCompletions(["==", "!=", "in", "not in"].map(makeKeywordCompletion), {
416
- enforceOrder: true,
417
- autoSpaceAfter: completingAtEnd,
418
- });
419
- const continuousRelationCompletions = () =>
420
- makeCompletions(
421
- ["<", "<=", ">", ">=", "==", "!="].map(makeKeywordCompletion),
422
- { enforceOrder: true, autoSpaceAfter: completingAtEnd },
423
- );
424
- const customRelationCompletions = () =>
425
- makeCompletions(
426
- ["<", "<=", ">", ">=", "==", "!=", "~="].map(makeKeywordCompletion),
427
- { enforceOrder: true, autoSpaceAfter: completingAtEnd },
428
- );
429
- /** @param {string[]} options */
430
- const rhsCompletions = (options) =>
431
- makeCompletions(options.map(makeLiteralCompletion));
432
-
433
- if (!prevToken(1)) return newExpressionCompletions();
434
-
435
- // Member access
436
- if (prevToken(1)?.text == ".") {
437
- const scorer = prevToken(2)?.text;
438
- if (scorer) {
439
- return memberAccessCompletions(getMemberScoreItems(filterItems, scorer));
440
- }
441
- }
442
-
443
- // Start of a function call or of a bracketed expression
444
- if (prevToken(1)?.text == "(") {
445
- if (prevToken(2)?.type == "mathFunction") return variableCompletions();
446
- if (prevToken(2)?.type == "sampleFunction") {
447
- // All sample functions expect a literal (a string to search for).
448
- return noCompletions();
449
- }
450
- // A grouping parenthesis, not a function call.
451
- return newExpressionCompletions();
452
- }
453
-
454
- // End of a function call or of a bracketed expression
455
- if (prevToken(1)?.text == ")") {
456
- // Don't try to guess: too unpredictable. Could continue with an arithmetic
457
- // operator (if constructing a complex expression), with a comparison (if
458
- // comparing function call result to something) or with a logical connector
459
- // (if a new subexpression is starting). Very hard to figure out what is
460
- // going on without an AST, which we don't have here.
461
- return noCompletions();
462
- }
463
-
464
- // Suggest relation based on variable type
465
- if (prevToken(1)?.type == "variable") {
466
- const scoreType = findFilterItem(1)?.scoreType;
467
- if ([kScoreTypePassFail, kScoreTypeCategorical].includes(scoreType))
468
- return descreteRelationCompletions();
469
- if (scoreType == kScoreTypeNumeric) return continuousRelationCompletions();
470
- if (scoreType == kScoreTypeOther) return customRelationCompletions();
471
- if (scoreType == kScoreTypeBoolean) return logicalOpCompletions();
472
- }
473
-
474
- // Suggest comparison RHS based on the LHS
475
- if (prevToken(1)?.type == "relation") {
476
- const item = findFilterItem(2);
477
- if (item) {
478
- if (item?.categories?.length) {
479
- return rhsCompletions(item.categories);
480
- } else {
481
- // Technically, it's possible to compare two scores, but comparison to a
482
- // constant is much more likely.
483
- return noCompletions();
484
- }
485
- } else {
486
- // Most likely: comparison starting from a constant, perhaps beginning of
487
- // a chain comparison.
488
- return variableCompletions();
489
- }
490
- }
491
-
492
- // Suggest connector to the next subexpression after `VARIABLE OP VALUE` subexpression finished.
493
- if (isLiteral(prevToken(1)) && prevToken(2)?.type == "relation") {
494
- return logicalOpCompletions();
495
- }
496
-
497
- // New subexpression begins after a logical connector.
498
- if (isLogicalOp(prevToken(1))) return newExpressionCompletions();
499
-
500
- // Something unusual is going on. We don't have any good guesses, but the user
501
- // can trigger completion manually with Ctrl+Space if they want.
502
- return noCompletions();
503
- }
504
-
505
- /**
506
- * @param {import("codemirror").EditorView} view
507
- * @param {import("./filters.mjs").FilterError | undefined} filterError
508
- * @returns {import("@codemirror/lint").Diagnostic[]}
509
- */
510
- function getLints(view, filterError) {
511
- if (!filterError) return [];
512
- return [
513
- {
514
- from: filterError.from || 0,
515
- to: filterError.to || view.state.doc.length,
516
- severity: filterError.severity,
517
- message: filterError.message,
518
- },
519
- ];
520
- }
521
-
522
- // Emulate `form-control` style to make it look like a text input.
523
- const editorTheme = EditorView.theme({
524
- "&": {
525
- fontSize: "inherit",
526
- color: "var(--inspect-input-foreground)",
527
- backgroundColor: "var(--inspect-input-background)",
528
- border: "1px solid var(--inspect-input-border)",
529
- borderRadius: "var(--bs-border-radius)",
530
- },
531
- ".cm-cursor.cm-cursor-primary": {
532
- borderLeftColor: "var(--bs-body-color)",
533
- },
534
- ".cm-selectionBackground": {
535
- backgroundColor: "var(--inspect-inactive-selection-background)",
536
- },
537
- "&.cm-focused > .cm-scroller > .cm-selectionLayer > .cm-selectionBackground":
538
- {
539
- backgroundColor: "var(--inspect-active-selection-background)",
540
- },
541
- "&.cm-focused": {
542
- outline: "none",
543
- borderColor: "var(--inspect-focus-border-color)",
544
- boxShadow: "var(--inspect-focus-border-shadow)",
545
- },
546
- ".filter-pending > &.cm-focused": {
547
- borderColor: "var(--inspect-focus-border-gray-color)",
548
- boxShadow: "var(--inspect-focus-border-gray-shadow)",
549
- },
550
- ".cm-tooltip": {
551
- backgroundColor: "var(--bs-light)",
552
- border: "1px solid var(--bs-border-color)",
553
- color: "var(--bs-body-color)",
554
- },
555
- ".cm-tooltip.cm-tooltip-autocomplete > ul > li": {
556
- color: "var(--bs-body-color)",
557
- },
558
- ".cm-tooltip.cm-tooltip-autocomplete > ul > li[aria-selected]": {
559
- backgroundColor: "var(--inspect-active-selection-background)",
560
- color: "var(--bs-body-color)",
561
- },
562
- ".cm-scroller": {
563
- overflow: "hidden",
564
- },
565
- });
566
-
567
- /**
568
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
569
- * @param {string} filterValue
570
- * @returns {FilteringResult}
571
- */
572
- const getFilteringResult = (evalDescriptor, filterValue) => {
573
- const { result, error } = filterSamples(
574
- evalDescriptor,
575
- evalDescriptor.samples,
576
- filterValue,
577
- );
578
- return { numSamples: result.length, error };
579
- };
580
-
581
- /**
582
- * Renders the Sample Filter Control
583
- *
584
- * @param {Object} props - The parameters for the component.
585
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} props.evalDescriptor
586
- * @param {(filter: import("../../Types.mjs").ScoreFilter) => void} props.filterChanged - Filter changed function
587
- * @param {import("../../Types.mjs").ScoreFilter} props.filter - Filter that is currently applied.
588
- * @returns {import("preact").JSX.Element | string} The TranscriptView component.
589
- */
590
- export const SampleFilter = ({ evalDescriptor, filter, filterChanged }) => {
591
- const editorRef = useRef(/** @type {HTMLElement|null} */ (null));
592
- const editorViewRef = useRef(
593
- /** @type {import("codemirror").EditorView|null} */ (null),
594
- );
595
- const linterCompartment = useRef(new Compartment());
596
- const autocompletionCompartment = useRef(new Compartment());
597
- const updateListenerCompartment = useRef(new Compartment());
598
- const filterItems = useMemo(
599
- () => scoreFilterItems(evalDescriptor),
600
- [evalDescriptor],
601
- );
602
- // Result of applying the filter expression in the editor, which might be
603
- // different from the active filter.
604
- const [filteringResultInstant, setFilteringResultInstant] = useState(
605
- /** @type {FilteringResult | null} */ (null),
606
- );
607
-
608
- /**
609
- * @param {FocusEvent} event
610
- * @param {import("codemirror").EditorView} view
611
- */
612
- const handleFocus = (event, view) => {
613
- if (event.isTrusted && view.state.doc.toString() === "") {
614
- setTimeout(() => startCompletion(view), 0);
615
- }
616
- };
617
-
618
- const makeAutocompletion = () =>
619
- autocompletion({
620
- override: [(context) => getCompletions(context, filterItems)],
621
- activateOnCompletion: (c) => c.label.endsWith(" "), // see autoSpaceAfter
622
- });
623
- const makeLinter = () =>
624
- // CodeMirror debounces the linter, so even instant error updates are not annoying
625
- linter((view) => getLints(view, filteringResultInstant?.error));
626
- const makeUpdateListener = () =>
627
- EditorView.updateListener.of((update) => {
628
- if (update.docChanged) {
629
- const newValue = update.state.doc.toString();
630
- const filteringResult = getFilteringResult(evalDescriptor, newValue);
631
- if (!filteringResult.error) {
632
- filterChanged({ value: newValue });
633
- }
634
- setFilteringResultInstant(filteringResult);
635
- }
636
- });
637
-
638
- // Initialize editor when component mounts
639
- useEffect(() => {
640
- editorViewRef.current?.destroy();
641
- editorViewRef.current = new EditorView({
642
- parent: editorRef.current,
643
- state: EditorState.create({
644
- doc: filter.value || "",
645
- extensions: [
646
- minimalSetup,
647
- bracketMatching(),
648
- editorTheme,
649
- EditorState.transactionFilter.of(ensureOneLine),
650
- updateListenerCompartment.current.of(makeUpdateListener()),
651
- EditorView.domEventHandlers({
652
- focus: handleFocus,
653
- }),
654
- language,
655
- syntaxHighlighting(highlightStyle),
656
- autocompletionCompartment.current.of(makeAutocompletion()),
657
- linterCompartment.current.of(makeLinter()),
658
- ],
659
- }),
660
- });
661
- return () => {
662
- editorViewRef.current?.destroy();
663
- };
664
- }, []);
665
-
666
- useEffect(() => {
667
- if (
668
- editorViewRef.current &&
669
- filter.value !== editorViewRef.current.state.doc.toString()
670
- ) {
671
- setFilteringResultInstant(
672
- getFilteringResult(evalDescriptor, filter.value),
673
- );
674
- editorViewRef.current.dispatch({
675
- changes: {
676
- from: 0,
677
- to: editorViewRef.current.state.doc.length,
678
- insert: filter.value || "",
679
- },
680
- });
681
- }
682
- }, [evalDescriptor, filter.value]);
683
-
684
- useEffect(() => {
685
- if (editorViewRef.current) {
686
- editorViewRef.current.dispatch({
687
- effects:
688
- updateListenerCompartment.current.reconfigure(makeUpdateListener()),
689
- });
690
- }
691
- }, [evalDescriptor]);
692
-
693
- useEffect(() => {
694
- if (editorViewRef.current) {
695
- editorViewRef.current.dispatch({
696
- effects:
697
- autocompletionCompartment.current.reconfigure(makeAutocompletion()),
698
- });
699
- }
700
- }, [filterItems]);
701
-
702
- useEffect(() => {
703
- if (editorViewRef.current) {
704
- editorViewRef.current.dispatch({
705
- effects: linterCompartment.current.reconfigure(makeLinter()),
706
- });
707
- }
708
- }, [filteringResultInstant?.error]);
709
-
710
- return html`
711
- <div style=${{ display: "flex" }}>
712
- <span
713
- class="sample-filter-label"
714
- style=${{
715
- alignSelf: "center",
716
- fontSize: FontSize.smaller,
717
- ...TextStyle.label,
718
- ...TextStyle.secondary,
719
- marginRight: "0.3em",
720
- marginLeft: "0.2em",
721
- }}
722
- >Filter:</span
723
- >
724
- <div
725
- ref=${editorRef}
726
- style=${{ width: "300px" }}
727
- class=${filteringResultInstant?.error ? ["filter-pending"] : []}
728
- ></div>
729
- <span
730
- class="bi bi-question-circle"
731
- style=${{
732
- position: "relative",
733
- marginLeft: "0.5em",
734
- cursor: "help",
735
- alignSelf: "center",
736
- }}
737
- data-tooltip=${filterTooltip}
738
- data-tooltip-position="bottom-left"
739
- ></span>
740
- </div>
741
- `;
742
- };
743
-
744
- const filterTooltip = `
745
- Filter samples by:
746
- • Scores
747
- • Input and target regex search: input_contains, target_contains
748
-
749
- Supported expressions:
750
- • Arithmetic: +, -, *, /, mod, ^
751
- • Comparison: <, <=, >, >=, ==, !=, including chain comparisons, e.g. “10 <= x < 20”
752
- • Boolean: and, or, not
753
- • Regex matching: ~= (case-sensitive)
754
- • Set operations: in, not in; e.g. “x in (2, 3, 5)”
755
- • Functions: min, max, abs, round, floor, ceil, sqrt, log, log2, log10
756
- `.trim();