inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. inspect_ai/_cli/common.py +3 -1
  2. inspect_ai/_cli/eval.py +15 -9
  3. inspect_ai/_display/core/active.py +4 -1
  4. inspect_ai/_display/core/config.py +3 -3
  5. inspect_ai/_display/core/panel.py +7 -3
  6. inspect_ai/_display/plain/__init__.py +0 -0
  7. inspect_ai/_display/plain/display.py +203 -0
  8. inspect_ai/_display/rich/display.py +0 -5
  9. inspect_ai/_display/textual/widgets/port_mappings.py +110 -0
  10. inspect_ai/_display/textual/widgets/samples.py +79 -12
  11. inspect_ai/_display/textual/widgets/sandbox.py +37 -0
  12. inspect_ai/_eval/eval.py +10 -1
  13. inspect_ai/_eval/loader.py +79 -19
  14. inspect_ai/_eval/registry.py +6 -0
  15. inspect_ai/_eval/score.py +3 -1
  16. inspect_ai/_eval/task/results.py +51 -22
  17. inspect_ai/_eval/task/run.py +47 -13
  18. inspect_ai/_eval/task/sandbox.py +10 -5
  19. inspect_ai/_util/constants.py +1 -0
  20. inspect_ai/_util/port_names.py +61 -0
  21. inspect_ai/_util/text.py +23 -0
  22. inspect_ai/_view/www/App.css +31 -1
  23. inspect_ai/_view/www/dist/assets/index.css +31 -1
  24. inspect_ai/_view/www/dist/assets/index.js +25498 -2044
  25. inspect_ai/_view/www/log-schema.json +32 -2
  26. inspect_ai/_view/www/package.json +2 -0
  27. inspect_ai/_view/www/src/App.mjs +14 -16
  28. inspect_ai/_view/www/src/Types.mjs +1 -2
  29. inspect_ai/_view/www/src/api/Types.ts +133 -0
  30. inspect_ai/_view/www/src/api/{api-browser.mjs → api-browser.ts} +25 -13
  31. inspect_ai/_view/www/src/api/api-http.ts +219 -0
  32. inspect_ai/_view/www/src/api/api-shared.ts +47 -0
  33. inspect_ai/_view/www/src/api/{api-vscode.mjs → api-vscode.ts} +22 -19
  34. inspect_ai/_view/www/src/api/{client-api.mjs → client-api.ts} +93 -53
  35. inspect_ai/_view/www/src/api/index.ts +51 -0
  36. inspect_ai/_view/www/src/api/jsonrpc.ts +225 -0
  37. inspect_ai/_view/www/src/components/ChatView.mjs +133 -43
  38. inspect_ai/_view/www/src/components/DownloadButton.mjs +1 -1
  39. inspect_ai/_view/www/src/components/ExpandablePanel.mjs +0 -4
  40. inspect_ai/_view/www/src/components/LargeModal.mjs +19 -20
  41. inspect_ai/_view/www/src/components/TabSet.mjs +3 -1
  42. inspect_ai/_view/www/src/components/VirtualList.mjs +266 -84
  43. inspect_ai/_view/www/src/index.js +77 -4
  44. inspect_ai/_view/www/src/log/{remoteLogFile.mjs → remoteLogFile.ts} +62 -46
  45. inspect_ai/_view/www/src/navbar/Navbar.mjs +4 -1
  46. inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +19 -10
  47. inspect_ai/_view/www/src/samples/SampleDialog.mjs +5 -1
  48. inspect_ai/_view/www/src/samples/SampleDisplay.mjs +23 -15
  49. inspect_ai/_view/www/src/samples/SampleList.mjs +19 -49
  50. inspect_ai/_view/www/src/samples/SampleScores.mjs +1 -1
  51. inspect_ai/_view/www/src/samples/SampleTranscript.mjs +8 -3
  52. inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +38 -26
  53. inspect_ai/_view/www/src/samples/SamplesTab.mjs +14 -11
  54. inspect_ai/_view/www/src/samples/SamplesTools.mjs +8 -8
  55. inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +712 -89
  56. inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +2 -2
  57. inspect_ai/_view/www/src/samples/tools/filters.mjs +260 -87
  58. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs +24 -2
  59. inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs +29 -24
  60. inspect_ai/_view/www/src/samples/transcript/EventRow.mjs +1 -1
  61. inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs +24 -2
  62. inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs +24 -2
  63. inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs +31 -10
  64. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.mjs +24 -2
  65. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.mjs +23 -2
  66. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.mjs +24 -2
  67. inspect_ai/_view/www/src/samples/transcript/StepEventView.mjs +33 -3
  68. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.mjs +25 -2
  69. inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs +25 -2
  70. inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs +193 -11
  71. inspect_ai/_view/www/src/samples/transcript/Types.mjs +10 -0
  72. inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs +26 -2
  73. inspect_ai/_view/www/src/types/log.d.ts +13 -2
  74. inspect_ai/_view/www/src/utils/Format.mjs +10 -3
  75. inspect_ai/_view/www/src/utils/{Json.mjs → json-worker.ts} +13 -9
  76. inspect_ai/_view/www/src/utils/vscode.ts +36 -0
  77. inspect_ai/_view/www/src/workspace/WorkSpace.mjs +11 -5
  78. inspect_ai/_view/www/vite.config.js +7 -0
  79. inspect_ai/_view/www/yarn.lock +116 -0
  80. inspect_ai/approval/_human/__init__.py +0 -0
  81. inspect_ai/approval/_human/manager.py +1 -1
  82. inspect_ai/approval/_policy.py +12 -6
  83. inspect_ai/log/_log.py +1 -1
  84. inspect_ai/log/_samples.py +16 -0
  85. inspect_ai/log/_transcript.py +4 -1
  86. inspect_ai/model/_call_tools.py +59 -0
  87. inspect_ai/model/_conversation.py +16 -7
  88. inspect_ai/model/_generate_config.py +12 -12
  89. inspect_ai/model/_model.py +117 -18
  90. inspect_ai/model/_model_output.py +22 -2
  91. inspect_ai/model/_openai.py +383 -0
  92. inspect_ai/model/_providers/anthropic.py +152 -55
  93. inspect_ai/model/_providers/azureai.py +21 -21
  94. inspect_ai/model/_providers/bedrock.py +37 -40
  95. inspect_ai/model/_providers/goodfire.py +248 -0
  96. inspect_ai/model/_providers/google.py +46 -54
  97. inspect_ai/model/_providers/groq.py +7 -3
  98. inspect_ai/model/_providers/hf.py +6 -0
  99. inspect_ai/model/_providers/mistral.py +13 -12
  100. inspect_ai/model/_providers/openai.py +51 -218
  101. inspect_ai/model/_providers/openai_o1.py +11 -12
  102. inspect_ai/model/_providers/providers.py +23 -1
  103. inspect_ai/model/_providers/together.py +12 -12
  104. inspect_ai/model/_providers/util/__init__.py +2 -3
  105. inspect_ai/model/_providers/util/hf_handler.py +1 -1
  106. inspect_ai/model/_providers/util/llama31.py +1 -1
  107. inspect_ai/model/_providers/util/util.py +0 -76
  108. inspect_ai/model/_providers/vertex.py +1 -4
  109. inspect_ai/scorer/_metric.py +3 -0
  110. inspect_ai/scorer/_reducer/reducer.py +1 -1
  111. inspect_ai/scorer/_scorer.py +4 -3
  112. inspect_ai/solver/__init__.py +4 -5
  113. inspect_ai/solver/_basic_agent.py +1 -1
  114. inspect_ai/solver/_bridge/__init__.py +3 -0
  115. inspect_ai/solver/_bridge/bridge.py +100 -0
  116. inspect_ai/solver/_bridge/patch.py +170 -0
  117. inspect_ai/solver/_prompt.py +35 -5
  118. inspect_ai/solver/_solver.py +6 -0
  119. inspect_ai/solver/_task_state.py +80 -38
  120. inspect_ai/tool/__init__.py +2 -0
  121. inspect_ai/tool/_tool.py +12 -1
  122. inspect_ai/tool/_tool_call.py +10 -0
  123. inspect_ai/tool/_tool_def.py +16 -5
  124. inspect_ai/tool/_tool_with.py +21 -4
  125. inspect_ai/tool/beta/__init__.py +5 -0
  126. inspect_ai/tool/beta/_computer/__init__.py +3 -0
  127. inspect_ai/tool/beta/_computer/_common.py +133 -0
  128. inspect_ai/tool/beta/_computer/_computer.py +155 -0
  129. inspect_ai/tool/beta/_computer/_computer_split.py +198 -0
  130. inspect_ai/tool/beta/_computer/_resources/Dockerfile +100 -0
  131. inspect_ai/tool/beta/_computer/_resources/README.md +30 -0
  132. inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh +18 -0
  133. inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh +20 -0
  134. inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh +48 -0
  135. inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh +13 -0
  136. inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh +48 -0
  137. inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop +10 -0
  138. inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop +10 -0
  139. inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop +10 -0
  140. inspect_ai/tool/beta/_computer/_resources/tool/__init__.py +0 -0
  141. inspect_ai/tool/beta/_computer/_resources/tool/_logger.py +22 -0
  142. inspect_ai/tool/beta/_computer/_resources/tool/_run.py +42 -0
  143. inspect_ai/tool/beta/_computer/_resources/tool/_tool_result.py +33 -0
  144. inspect_ai/tool/beta/_computer/_resources/tool/_x11_client.py +262 -0
  145. inspect_ai/tool/beta/_computer/_resources/tool/computer_tool.py +85 -0
  146. inspect_ai/tool/beta/_computer/_resources/tool/requirements.txt +0 -0
  147. inspect_ai/util/__init__.py +2 -0
  148. inspect_ai/util/_display.py +5 -0
  149. inspect_ai/util/_limit.py +26 -0
  150. inspect_ai/util/_sandbox/docker/docker.py +64 -1
  151. inspect_ai/util/_sandbox/docker/internal.py +3 -1
  152. inspect_ai/util/_sandbox/docker/prereqs.py +1 -1
  153. inspect_ai/util/_sandbox/environment.py +14 -0
  154. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/METADATA +3 -2
  155. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/RECORD +159 -126
  156. inspect_ai/_view/www/src/api/Types.mjs +0 -117
  157. inspect_ai/_view/www/src/api/api-http.mjs +0 -300
  158. inspect_ai/_view/www/src/api/api-shared.mjs +0 -10
  159. inspect_ai/_view/www/src/api/index.mjs +0 -49
  160. inspect_ai/_view/www/src/api/jsonrpc.mjs +0 -208
  161. inspect_ai/_view/www/src/samples/transcript/TranscriptState.mjs +0 -70
  162. inspect_ai/_view/www/src/utils/vscode.mjs +0 -16
  163. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/LICENSE +0 -0
  164. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/WHEEL +0 -0
  165. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/entry_points.txt +0 -0
  166. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/top_level.txt +0 -0
@@ -1,112 +1,716 @@
1
+ import { autocompletion, startCompletion } from "@codemirror/autocomplete";
2
+ import {
3
+ HighlightStyle,
4
+ StreamLanguage,
5
+ StringStream,
6
+ bracketMatching,
7
+ syntaxHighlighting,
8
+ } from "@codemirror/language";
9
+ import { linter } from "@codemirror/lint";
10
+ import { Compartment, EditorState } from "@codemirror/state";
11
+ import { tags } from "@lezer/highlight";
12
+ import { EditorView, minimalSetup } from "codemirror";
1
13
  import { html } from "htm/preact";
14
+ import { useEffect, useMemo, useRef, useState } from "preact/hooks";
2
15
  import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
3
-
16
+ import { filterSamples, scoreFilterItems } from "./filters.mjs";
4
17
  import {
18
+ kScoreTypeBoolean,
5
19
  kScoreTypeCategorical,
6
20
  kScoreTypeNumeric,
7
- kScoreTypeObject,
21
+ kScoreTypeOther,
8
22
  kScoreTypePassFail,
9
23
  } from "../../constants.mjs";
10
24
 
11
25
  /**
12
- * Renders the Sample Filter Control
13
- *
14
- * @param {Object} props - The parameters for the component.
15
- * @param {import("../SamplesDescriptor.mjs").SamplesDescriptor} props.descriptor - The sample descriptor
16
- * @param {(filter: import("../../Types.mjs").ScoreFilter) => void} props.filterChanged - Filter changed function
17
- * @param {import("../../Types.mjs").ScoreFilter} props.filter - Capabilities of the application host
18
- * @returns {import("preact").JSX.Element | string} The TranscriptView component.
26
+ * @typedef {Object} Token
27
+ * @property {string} type
28
+ * @property {string} text
29
+ * @property {number} from
30
+ * @property {number} to
19
31
  */
20
- export const SampleFilter = ({ descriptor, filter, filterChanged }) => {
21
- const updateCategoryValue = (e) => {
22
- const val = e.currentTarget.value;
23
- if (val === "all") {
24
- filterChanged({});
32
+
33
+ /**
34
+ * @typedef {Object} FilteringResult
35
+ * @property {number} numSamples - The number of samples that match the filter.
36
+ * @property {import("./filters.mjs").FilterError | undefined} error - The error in the filter expression, if any.
37
+ */
38
+
39
+ const KEYWORDS = ["and", "or", "not", "in", "not in", "mod"];
40
+
41
+ const MATH_FUNCTIONS = [
42
+ ["min", "Minimum of two or more values"],
43
+ ["max", "Maximum of two or more values"],
44
+ ["abs", "Absolute value"],
45
+ ["round", "Round to the nearest integer"],
46
+ ["floor", "Round down to the nearest integer"],
47
+ ["ceil", "Round up to the nearest integer"],
48
+ ["sqrt", "Square root"],
49
+ ["log", "Natural logarithm"],
50
+ ["log2", "Base 2 logarithm"],
51
+ ["log10", "Base 10 logarithm"],
52
+ ];
53
+
54
+ const SAMPLE_FUNCTIONS = [
55
+ ["input_contains", "Checks if input contains a regular expression"],
56
+ ["target_contains", "Checks if target contains a regular expression"],
57
+ ];
58
+
59
+ /**
60
+ * Makes sure that the filter expression is a single line.
61
+ * @param {import("@codemirror/state").Transaction} tr - The transaction to join lines in.
62
+ * @returns {import("@codemirror/state").TransactionSpec} The transaction with joined lines, if any.
63
+ */
64
+ function ensureOneLine(tr) {
65
+ const newDoc = tr.newDoc.toString();
66
+ if (newDoc.includes("\n")) {
67
+ if (tr.isUserEvent("input.paste")) {
68
+ const newDocAdjusted = newDoc.replace(/\n/g, " ").trim();
69
+ return {
70
+ changes: {
71
+ from: 0,
72
+ to: tr.startState.doc.length,
73
+ insert: newDocAdjusted,
74
+ },
75
+ };
25
76
  } else {
26
- filterChanged({
27
- value: val,
28
- type: kScoreTypeCategorical,
77
+ return {};
78
+ }
79
+ }
80
+ return tr;
81
+ }
82
+
83
+ const highlightStyle = HighlightStyle.define([
84
+ { tag: tags.string, class: "token string" },
85
+ { tag: tags.number, class: "token number" },
86
+ { tag: tags.keyword, class: "token keyword" },
87
+ ]);
88
+
89
+ /** @param {string} word */
90
+ function countSpaces(word) {
91
+ return word.split(" ").length - 1;
92
+ }
93
+
94
+ const nextToken = (() => {
95
+ const wordsRe = (words) => new RegExp(`^(${words.join("|")})\\b`);
96
+ const keywordsRe = wordsRe(
97
+ // Sort to make sure "not in" is matched before "not".
98
+ KEYWORDS.sort((a, b) => countSpaces(b) - countSpaces(a)),
99
+ );
100
+ const mathFunctionsRe = wordsRe(MATH_FUNCTIONS.map(([label]) => label));
101
+ const sampleFunctionsRe = wordsRe(SAMPLE_FUNCTIONS.map(([label]) => label));
102
+
103
+ /** @param {import("@codemirror/language").StringStream} stream */
104
+ return function (stream) {
105
+ if (stream.match(/"[^"]*"/)) return "string";
106
+ if (stream.match(/"[^"]*/)) return "unterminatedString";
107
+ if (stream.match(/^(-|\+)?\d+(\.\d+)?/)) return "number";
108
+ if (stream.match(keywordsRe)) return "keyword";
109
+ if (stream.match(mathFunctionsRe)) return "mathFunction";
110
+ if (stream.match(sampleFunctionsRe)) return "sampleFunction";
111
+ if (stream.match(/^[a-zA-Z_][a-zA-Z0-9_]*/)) return "variable";
112
+ if (stream.match(/^(==|!=|<=|>=|<|>|~=)/)) return "relation";
113
+ if (stream.match(/^(=|!|~)/)) return "miscOperator"; // recognize relations while typing; not valid syntax per se
114
+ if (stream.match(/^(\+|-|\*|\/|\^|\(|\)|,|\.)/)) return "miscOperator";
115
+ stream.next();
116
+ return null;
117
+ };
118
+ })();
119
+
120
+ const language = StreamLanguage.define({
121
+ token: nextToken,
122
+ tokenTable: {
123
+ string: tags.string,
124
+ unterminatedString: tags.string,
125
+ number: tags.number,
126
+ keyword: tags.keyword,
127
+ mathFunction: tags.function(tags.variableName),
128
+ sampleFunction: tags.function(tags.variableName),
129
+ variable: tags.variableName,
130
+ relation: tags.operator,
131
+ miscOperator: tags.operator,
132
+ },
133
+ });
134
+
135
+ /**
136
+ * @param {string} input
137
+ * @returns {Token[]}
138
+ */
139
+ function tokenize(input) {
140
+ const tokens = [];
141
+ const stream = new StringStream(input, 0, 0);
142
+ while (stream.pos < input.length) {
143
+ const from = stream.pos;
144
+ const type = nextToken(stream);
145
+ if (type) {
146
+ tokens.push({
147
+ type,
148
+ text: input.slice(from, stream.pos),
149
+ from,
150
+ to: stream.pos,
29
151
  });
30
152
  }
153
+ }
154
+ return tokens;
155
+ }
156
+
157
+ /**
158
+ * @param {import("./filters.mjs").ScoreFilterItem[]} filterItems
159
+ * @param {string} scorer
160
+ * @returns {import("./filters.mjs").ScoreFilterItem[]}
161
+ */
162
+ function getMemberScoreItems(filterItems, scorer) {
163
+ return filterItems.filter((item) =>
164
+ item?.qualifiedName?.startsWith(`${scorer}.`),
165
+ );
166
+ }
167
+
168
+ /**
169
+ * Generates completions for the filter expression. The main goal is to make the
170
+ * sample filter intuitive for beginners and to provide a smooth experience for
171
+ * simple cases. To this end, we proactively try to suggest the next step of the
172
+ * expression, in a wizard-style fashion. This logic is primarily intended to
173
+ * support unsophisticated expressions of the form
174
+ * SUBEXPR and/or SUBEXPR or/not SUBEXPR ...
175
+ * where each SUBEXPR is
176
+ * VARIABLE ==/!=/</>/in/... VALUE
177
+ * and VALUE is a literal (string, number, etc.)
178
+ * It does support some expressions more complex than that, but the completion
179
+ * algorithm is not intended to be comprehensive. This is why we usually add
180
+ * default completions to the list in case our guess was off.
181
+ *
182
+ * @param {import("@codemirror/autocomplete").CompletionContext} context
183
+ * @param {import("../../samples/tools/filters.mjs").ScoreFilterItem[]} filterItems
184
+ * @returns {import("@codemirror/autocomplete").CompletionResult}
185
+ */
186
+ function getCompletions(context, filterItems) {
187
+ /** @param {Token} token */
188
+ const isLiteral = (token) =>
189
+ ["string", "unterminatedString", "number"].includes(token?.type);
190
+ /** @param {Token} token */
191
+ const isLogicalOp = (token) => ["and", "or", "not"].includes(token?.text);
192
+
193
+ /**
194
+ * With most tokens we complete only after a space, but for sometimes it makes
195
+ * sense to start autocompletion as soon as the token was typed.
196
+ * @param {Token} token
197
+ */
198
+ const autocompleteImmediatelyAfter = (token) =>
199
+ ["(", "."].includes(token?.text);
200
+
201
+ /**
202
+ * @param {import("codemirror").EditorView} view
203
+ * @param {import("@codemirror/autocomplete").Completion} completion
204
+ * @param {number} from
205
+ * @param {number} to
206
+ */
207
+ function applyWithCall(view, completion, from, to) {
208
+ view.dispatch({
209
+ changes: { from, to, insert: `${completion.label}()` },
210
+ selection: { anchor: from + completion.label.length + 1 },
211
+ });
212
+ }
213
+
214
+ /** @type {(k: string) => import("@codemirror/autocomplete").Completion} */
215
+ const makeKeywordCompletion = (k) => ({
216
+ label: k,
217
+ type: "keyword",
218
+ boost: -20,
219
+ });
220
+ /** @type {([label, info]: [string, string]) => import("@codemirror/autocomplete").Completion} */
221
+ const makeMathFunctionCompletion = ([label, info]) => ({
222
+ label,
223
+ type: "function",
224
+ info,
225
+ apply: applyWithCall,
226
+ boost: -10,
227
+ });
228
+ /** @type {([label, info]: [string, string]) => import("@codemirror/autocomplete").Completion} */
229
+ const makeSampleFunctionCompletion = ([label, info]) => ({
230
+ label,
231
+ type: "function",
232
+ info,
233
+ apply: applyWithCall,
234
+ boost: 0,
235
+ });
236
+ /** @type {(k: string) => import("@codemirror/autocomplete").Completion} */
237
+ const makeLiteralCompletion = (k) => ({
238
+ label: k,
239
+ type: "text",
240
+ boost: 10,
241
+ });
242
+ /**
243
+ * @param {import("./filters.mjs").ScoreFilterItem} item
244
+ * @param {Object} [props]
245
+ * @param {(item: import("./filters.mjs").ScoreFilterItem) => boolean} [props.autoSpaceIf] - Similar to `autoSpaceAfter`, but conditional.
246
+ * @returns {import("@codemirror/autocomplete").Completion}
247
+ */
248
+ const makeCanonicalNameCompletion = (
249
+ item,
250
+ { autoSpaceIf = () => false } = {},
251
+ ) => ({
252
+ label: item.canonicalName + (autoSpaceIf(item) ? " " : ""),
253
+ type: "variable",
254
+ info: item.tooltip,
255
+ boost: 20,
256
+ });
257
+ /** @type {(item: import("./filters.mjs").ScoreFilterItem) => import("@codemirror/autocomplete").Completion} */
258
+ const makeMemberAccessCompletion = (item) => ({
259
+ label: item.qualifiedName.split(".")[1],
260
+ type: "variable",
261
+ info: item.tooltip,
262
+ boost: 20,
263
+ });
264
+
265
+ const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
266
+ const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
267
+ makeMathFunctionCompletion,
268
+ );
269
+ const sampleFunctionCompletionItems = SAMPLE_FUNCTIONS.map(
270
+ makeSampleFunctionCompletion,
271
+ );
272
+ const variableCompletionItems = filterItems.map((item) =>
273
+ makeCanonicalNameCompletion(item),
274
+ );
275
+
276
+ const defaultCompletionItems = [
277
+ ...keywordCompletionItems,
278
+ ...mathFunctionCompletionItems,
279
+ ...sampleFunctionCompletionItems,
280
+ ...variableCompletionItems,
281
+ ];
282
+
283
+ const doc = context.state.doc;
284
+ const input = doc.toString().slice(0, context.pos);
285
+ const tokens = tokenize(input);
286
+ const lastToken = tokens[tokens.length - 1];
287
+ const isCompletionInsideToken =
288
+ lastToken &&
289
+ context.pos == lastToken.to &&
290
+ !autocompleteImmediatelyAfter(lastToken);
291
+ const currentTokenIndex = isCompletionInsideToken
292
+ ? tokens.length - 1
293
+ : tokens.length; // `currentToken` is undefined when we are not inside a token
294
+
295
+ /**
296
+ * Returns nth token back away from the current token. Note that `prevToken(0)`
297
+ * is always reserved for the current token, whether it exists or not.
298
+ * @param {number} index
299
+ * @returns {Token | undefined}
300
+ */
301
+ const prevToken = (index) => tokens[currentTokenIndex - index];
302
+
303
+ const currentToken = prevToken(0);
304
+ const completionStart = currentToken ? currentToken.from : context.pos;
305
+ const completingAtEnd = context.pos == doc.length;
306
+
307
+ /**
308
+ * @param {number} endIndex
309
+ * @returns {import("../../samples/tools/filters.mjs").ScoreFilterItem | undefined}
310
+ */
311
+ const findFilterItem = (endIndex) => {
312
+ if (prevToken(endIndex)?.type == "variable") {
313
+ let name = prevToken(endIndex).text;
314
+ let i = endIndex;
315
+ while (prevToken(i + 1)?.text == ".") {
316
+ if (prevToken(i + 2)?.type == "variable") {
317
+ name = prevToken(i + 2).text + "." + name;
318
+ i += 2;
319
+ } else {
320
+ break;
321
+ }
322
+ }
323
+ return filterItems.find((item) => item.canonicalName == name);
324
+ }
325
+ return undefined;
31
326
  };
32
327
 
33
- switch (descriptor?.selectedScoreDescriptor?.scoreType) {
34
- case kScoreTypePassFail: {
35
- const options = [{ text: "All", value: "all" }];
36
- options.push(
37
- ...descriptor.selectedScoreDescriptor.categories.map((cat) => {
38
- return { text: cat.text, value: cat.val };
328
+ /**
329
+ * @param {import("@codemirror/autocomplete").Completion[]} priorityCompletions
330
+ * @param {Object} props
331
+ * @param {boolean} [props.autocompleteInTheMiddle] - If true, completion would be shown automatically even when editing in the middle of the expression.
332
+ * @param {boolean} [props.enforceOrder] - If true, the priorityCompletions are shown in the order they are provided.
333
+ * @param {boolean} [props.autoSpaceAfter] - If true, space is inserted after priorityCompletions. When a user accepts a completion with a space, another completion is suggested immediately (see `activateOnCompletion`). Use when fairly certain that the expression continues.
334
+ * @param {boolean} [props.includeDefault] - If true, the default completions are included after the priority completions.
335
+ * @returns {import("@codemirror/autocomplete").CompletionResult}
336
+ */
337
+ const makeCompletions = (
338
+ priorityCompletions,
339
+ {
340
+ autocompleteInTheMiddle = false,
341
+ enforceOrder = false,
342
+ autoSpaceAfter = false,
343
+ includeDefault = true,
344
+ } = {},
345
+ ) => {
346
+ if (!autocompleteInTheMiddle && !completingAtEnd && !context.explicit) {
347
+ return null;
348
+ }
349
+ const priorityCompletionsOrdered = enforceOrder
350
+ ? priorityCompletions.map((c, idx) => ({
351
+ ...c,
352
+ boost: -idx,
353
+ }))
354
+ : priorityCompletions;
355
+ const priorityCompletionsAdjusted = autoSpaceAfter
356
+ ? priorityCompletionsOrdered.map((c) =>
357
+ !c.apply && !c.label.endsWith(" ")
358
+ ? { ...c, label: c.label + " " }
359
+ : c,
360
+ )
361
+ : priorityCompletionsOrdered;
362
+ if (includeDefault) {
363
+ /** @type {import("@codemirror/autocomplete").CompletionSection} */
364
+ const miscSection = {
365
+ name: "misc",
366
+ header: () => {
367
+ const element = document.createElement("hr");
368
+ element.style.display = "list-item";
369
+ element.style.margin = "2px 0";
370
+ return element;
371
+ },
372
+ };
373
+ const priorityLabels = new Set(priorityCompletions.map((c) => c.label));
374
+ const defaultCompletionAdjusted = priorityCompletions
375
+ ? defaultCompletionItems
376
+ .filter((c) => !priorityLabels.has(c.label))
377
+ .map((c) => ({ ...c, section: miscSection }))
378
+ : defaultCompletionItems;
379
+ return {
380
+ from: completionStart,
381
+ options: [...priorityCompletionsAdjusted, ...defaultCompletionAdjusted],
382
+ };
383
+ } else {
384
+ return {
385
+ from: completionStart,
386
+ options: priorityCompletionsAdjusted,
387
+ };
388
+ }
389
+ };
390
+ const defaultCompletions = () => makeCompletions([]);
391
+ const noCompletions = () => (context.explicit ? defaultCompletions() : null);
392
+ const newExpressionCompletions = () =>
393
+ makeCompletions([
394
+ ...filterItems.map((item) =>
395
+ makeCanonicalNameCompletion(item, {
396
+ autoSpaceIf: (item) =>
397
+ completingAtEnd && item.scoreType != kScoreTypeBoolean,
39
398
  }),
40
- );
41
- return html`<${SelectFilter}
42
- value=${filter.value || "all"}
43
- options=${options}
44
- onChange=${updateCategoryValue}
45
- />`;
399
+ ),
400
+ ...sampleFunctionCompletionItems,
401
+ ]);
402
+ const variableCompletions = () => makeCompletions(variableCompletionItems);
403
+ /** @param {import("./filters.mjs").ScoreFilterItem[]} items */
404
+ const memberAccessCompletions = (items) =>
405
+ makeCompletions(items.map(makeMemberAccessCompletion), {
406
+ autocompleteInTheMiddle: true,
407
+ includeDefault: false,
408
+ });
409
+ const logicalOpCompletions = () =>
410
+ makeCompletions(["and", "or"].map(makeKeywordCompletion), {
411
+ enforceOrder: true,
412
+ autoSpaceAfter: completingAtEnd,
413
+ });
414
+ const descreteRelationCompletions = () =>
415
+ makeCompletions(["==", "!=", "in", "not in"].map(makeKeywordCompletion), {
416
+ enforceOrder: true,
417
+ autoSpaceAfter: completingAtEnd,
418
+ });
419
+ const continuousRelationCompletions = () =>
420
+ makeCompletions(
421
+ ["<", "<=", ">", ">=", "==", "!="].map(makeKeywordCompletion),
422
+ { enforceOrder: true, autoSpaceAfter: completingAtEnd },
423
+ );
424
+ const customRelationCompletions = () =>
425
+ makeCompletions(
426
+ ["<", "<=", ">", ">=", "==", "!=", "~="].map(makeKeywordCompletion),
427
+ { enforceOrder: true, autoSpaceAfter: completingAtEnd },
428
+ );
429
+ /** @param {string[]} options */
430
+ const rhsCompletions = (options) =>
431
+ makeCompletions(options.map(makeLiteralCompletion));
432
+
433
+ if (!prevToken(1)) return newExpressionCompletions();
434
+
435
+ // Member access
436
+ if (prevToken(1)?.text == ".") {
437
+ const scorer = prevToken(2)?.text;
438
+ if (scorer) {
439
+ return memberAccessCompletions(getMemberScoreItems(filterItems, scorer));
46
440
  }
441
+ }
47
442
 
48
- case kScoreTypeCategorical: {
49
- const options = [{ text: "All", value: "all" }];
50
- options.push(
51
- ...descriptor.selectedScoreDescriptor.categories.map((cat) => {
52
- return { text: cat, value: cat };
53
- }),
54
- );
55
- return html`<${SelectFilter}
56
- value=${filter.value || "all"}
57
- options=${options}
58
- onChange=${updateCategoryValue}
59
- />`;
443
+ // Start of a function call or of a bracketed expression
444
+ if (prevToken(1)?.text == "(") {
445
+ if (prevToken(2)?.type == "mathFunction") return variableCompletions();
446
+ if (prevToken(2)?.type == "sampleFunction") {
447
+ // All sample functions expect a literal (a string to search for).
448
+ return noCompletions();
60
449
  }
450
+ // A grouping parenthesis, not a function call.
451
+ return newExpressionCompletions();
452
+ }
453
+
454
+ // End of a function call or of a bracketed expression
455
+ if (prevToken(1)?.text == ")") {
456
+ // Don't try to guess: too unpredictable. Could continue with an arithmetic
457
+ // operator (if constructing a complex expression), with a comparison (if
458
+ // comparing function call result to something) or with a logical connector
459
+ // (if a new subexpression is starting). Very hard to figure out what is
460
+ // going on without an AST, which we don't have here.
461
+ return noCompletions();
462
+ }
463
+
464
+ // Suggest relation based on variable type
465
+ if (prevToken(1)?.type == "variable") {
466
+ const scoreType = findFilterItem(1)?.scoreType;
467
+ if ([kScoreTypePassFail, kScoreTypeCategorical].includes(scoreType))
468
+ return descreteRelationCompletions();
469
+ if (scoreType == kScoreTypeNumeric) return continuousRelationCompletions();
470
+ if (scoreType == kScoreTypeOther) return customRelationCompletions();
471
+ if (scoreType == kScoreTypeBoolean) return logicalOpCompletions();
472
+ }
61
473
 
62
- case kScoreTypeNumeric: {
63
- // TODO: Create a real numeric slider control of some kind
64
- return html`
65
- <input
66
- type="text"
67
- class="form-control"
68
- value=${filter.value}
69
- placeholder="Filter Samples (score)"
70
- style=${{ width: "150px" }}
71
- onInput=${(e) => {
72
- filterChanged({
73
- value: e.currentTarget.value,
74
- type: kScoreTypeNumeric,
75
- });
76
- }}
77
- />
78
- `;
474
+ // Suggest comparison RHS based on the LHS
475
+ if (prevToken(1)?.type == "relation") {
476
+ const item = findFilterItem(2);
477
+ if (item) {
478
+ if (item?.categories?.length) {
479
+ return rhsCompletions(item.categories);
480
+ } else {
481
+ // Technically, it's possible to compare two scores, but comparison to a
482
+ // constant is much more likely.
483
+ return noCompletions();
484
+ }
485
+ } else {
486
+ // Most likely: comparison starting from a constant, perhaps beginning of
487
+ // a chain comparison.
488
+ return variableCompletions();
79
489
  }
490
+ }
491
+
492
+ // Suggest connector to the next subexpression after `VARIABLE OP VALUE` subexpression finished.
493
+ if (isLiteral(prevToken(1)) && prevToken(2)?.type == "relation") {
494
+ return logicalOpCompletions();
495
+ }
496
+
497
+ // New subexpression begins after a logical connector.
498
+ if (isLogicalOp(prevToken(1))) return newExpressionCompletions();
80
499
 
81
- case kScoreTypeObject: {
82
- if (!descriptor.selectedScoreDescriptor.categories) {
83
- return "";
500
+ // Something unusual is going on. We don't have any good guesses, but the user
501
+ // can trigger completion manually with Ctrl+Space if they want.
502
+ return noCompletions();
503
+ }
504
+
505
+ /**
506
+ * @param {import("codemirror").EditorView} view
507
+ * @param {import("./filters.mjs").FilterError | undefined} filterError
508
+ * @returns {import("@codemirror/lint").Diagnostic[]}
509
+ */
510
+ function getLints(view, filterError) {
511
+ if (!filterError) return [];
512
+ return [
513
+ {
514
+ from: filterError.from || 0,
515
+ to: filterError.to || view.state.doc.length,
516
+ severity: filterError.severity,
517
+ message: filterError.message,
518
+ },
519
+ ];
520
+ }
521
+
522
+ // Emulate `form-control` style to make it look like a text input.
523
+ const editorTheme = EditorView.theme({
524
+ "&": {
525
+ fontSize: "inherit",
526
+ color: "var(--inspect-input-foreground)",
527
+ backgroundColor: "var(--inspect-input-background)",
528
+ border: "1px solid var(--inspect-input-border)",
529
+ borderRadius: "var(--bs-border-radius)",
530
+ },
531
+ ".cm-cursor.cm-cursor-primary": {
532
+ borderLeftColor: "var(--bs-body-color)",
533
+ },
534
+ ".cm-selectionBackground": {
535
+ backgroundColor: "var(--inspect-inactive-selection-background)",
536
+ },
537
+ "&.cm-focused > .cm-scroller > .cm-selectionLayer > .cm-selectionBackground":
538
+ {
539
+ backgroundColor: "var(--inspect-active-selection-background)",
540
+ },
541
+ "&.cm-focused": {
542
+ outline: "none",
543
+ borderColor: "var(--inspect-focus-border-color)",
544
+ boxShadow: "var(--inspect-focus-border-shadow)",
545
+ },
546
+ ".filter-pending > &.cm-focused": {
547
+ borderColor: "var(--inspect-focus-border-gray-color)",
548
+ boxShadow: "var(--inspect-focus-border-gray-shadow)",
549
+ },
550
+ ".cm-tooltip": {
551
+ backgroundColor: "var(--bs-light)",
552
+ border: "1px solid var(--bs-border-color)",
553
+ color: "var(--bs-body-color)",
554
+ },
555
+ ".cm-tooltip.cm-tooltip-autocomplete > ul > li": {
556
+ color: "var(--bs-body-color)",
557
+ },
558
+ ".cm-tooltip.cm-tooltip-autocomplete > ul > li[aria-selected]": {
559
+ backgroundColor: "var(--inspect-active-selection-background)",
560
+ color: "var(--bs-body-color)",
561
+ },
562
+ ".cm-scroller": {
563
+ overflow: "hidden",
564
+ },
565
+ });
566
+
567
+ /**
568
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
569
+ * @param {string} filterValue
570
+ * @returns {FilteringResult}
571
+ */
572
+ const getFilteringResult = (evalDescriptor, filterValue) => {
573
+ const { result, error } = filterSamples(
574
+ evalDescriptor,
575
+ evalDescriptor.samples,
576
+ filterValue,
577
+ );
578
+ return { numSamples: result.length, error };
579
+ };
580
+
581
+ /**
582
+ * Renders the Sample Filter Control
583
+ *
584
+ * @param {Object} props - The parameters for the component.
585
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} props.evalDescriptor
586
+ * @param {(filter: import("../../Types.mjs").ScoreFilter) => void} props.filterChanged - Filter changed function
587
+ * @param {import("../../Types.mjs").ScoreFilter} props.filter - Filter that is currently applied.
588
+ * @returns {import("preact").JSX.Element | string} The TranscriptView component.
589
+ */
590
+ export const SampleFilter = ({ evalDescriptor, filter, filterChanged }) => {
591
+ const editorRef = useRef(/** @type {HTMLElement|null} */ (null));
592
+ const editorViewRef = useRef(
593
+ /** @type {import("codemirror").EditorView|null} */ (null),
594
+ );
595
+ const linterCompartment = useRef(new Compartment());
596
+ const autocompletionCompartment = useRef(new Compartment());
597
+ const updateListenerCompartment = useRef(new Compartment());
598
+ const filterItems = useMemo(
599
+ () => scoreFilterItems(evalDescriptor),
600
+ [evalDescriptor],
601
+ );
602
+ // Result of applying the filter expression in the editor, which might be
603
+ // different from the active filter.
604
+ const [filteringResultInstant, setFilteringResultInstant] = useState(
605
+ /** @type {FilteringResult | null} */ (null),
606
+ );
607
+
608
+ /**
609
+ * @param {FocusEvent} event
610
+ * @param {import("codemirror").EditorView} view
611
+ */
612
+ const handleFocus = (event, view) => {
613
+ if (event.isTrusted && view.state.doc.toString() === "") {
614
+ setTimeout(() => startCompletion(view), 0);
615
+ }
616
+ };
617
+
618
+ const makeAutocompletion = () =>
619
+ autocompletion({
620
+ override: [(context) => getCompletions(context, filterItems)],
621
+ activateOnCompletion: (c) => c.label.endsWith(" "), // see autoSpaceAfter
622
+ });
623
+ const makeLinter = () =>
624
+ // CodeMirror debounces the linter, so even instant error updates are not annoying
625
+ linter((view) => getLints(view, filteringResultInstant?.error));
626
+ const makeUpdateListener = () =>
627
+ EditorView.updateListener.of((update) => {
628
+ if (update.docChanged) {
629
+ const newValue = update.state.doc.toString();
630
+ const filteringResult = getFilteringResult(evalDescriptor, newValue);
631
+ if (!filteringResult.error) {
632
+ filterChanged({ value: newValue });
633
+ }
634
+ setFilteringResultInstant(filteringResult);
84
635
  }
85
- const options = [{ text: "All", value: "all" }];
86
- options.push(
87
- ...descriptor.selectedScoreDescriptor.categories.map((cat) => {
88
- return { text: cat.text, value: cat.value };
89
- }),
636
+ });
637
+
638
+ // Initialize editor when component mounts
639
+ useEffect(() => {
640
+ editorViewRef.current?.destroy();
641
+ editorViewRef.current = new EditorView({
642
+ parent: editorRef.current,
643
+ state: EditorState.create({
644
+ doc: filter.value || "",
645
+ extensions: [
646
+ minimalSetup,
647
+ bracketMatching(),
648
+ editorTheme,
649
+ EditorState.transactionFilter.of(ensureOneLine),
650
+ updateListenerCompartment.current.of(makeUpdateListener()),
651
+ EditorView.domEventHandlers({
652
+ focus: handleFocus,
653
+ }),
654
+ language,
655
+ syntaxHighlighting(highlightStyle),
656
+ autocompletionCompartment.current.of(makeAutocompletion()),
657
+ linterCompartment.current.of(makeLinter()),
658
+ ],
659
+ }),
660
+ });
661
+ return () => {
662
+ editorViewRef.current?.destroy();
663
+ };
664
+ }, []);
665
+
666
+ useEffect(() => {
667
+ if (
668
+ editorViewRef.current &&
669
+ filter.value !== editorViewRef.current.state.doc.toString()
670
+ ) {
671
+ setFilteringResultInstant(
672
+ getFilteringResult(evalDescriptor, filter.value),
90
673
  );
674
+ editorViewRef.current.dispatch({
675
+ changes: {
676
+ from: 0,
677
+ to: editorViewRef.current.state.doc.length,
678
+ insert: filter.value || "",
679
+ },
680
+ });
681
+ }
682
+ }, [evalDescriptor, filter.value]);
683
+
684
+ useEffect(() => {
685
+ if (editorViewRef.current) {
686
+ editorViewRef.current.dispatch({
687
+ effects:
688
+ updateListenerCompartment.current.reconfigure(makeUpdateListener()),
689
+ });
690
+ }
691
+ }, [evalDescriptor]);
91
692
 
92
- return html`<${SelectFilter}
93
- value=${filter.value || "all"}
94
- options=${options}
95
- onChange=${updateCategoryValue}
96
- />`;
693
+ useEffect(() => {
694
+ if (editorViewRef.current) {
695
+ editorViewRef.current.dispatch({
696
+ effects:
697
+ autocompletionCompartment.current.reconfigure(makeAutocompletion()),
698
+ });
97
699
  }
700
+ }, [filterItems]);
98
701
 
99
- default: {
100
- return undefined;
702
+ useEffect(() => {
703
+ if (editorViewRef.current) {
704
+ editorViewRef.current.dispatch({
705
+ effects: linterCompartment.current.reconfigure(makeLinter()),
706
+ });
101
707
  }
102
- }
103
- };
708
+ }, [filteringResultInstant?.error]);
104
709
 
105
- const SelectFilter = ({ value, options, onChange }) => {
106
710
  return html`
107
711
  <div style=${{ display: "flex" }}>
108
712
  <span
109
- class="sample-label"
713
+ class="sample-filter-label"
110
714
  style=${{
111
715
  alignSelf: "center",
112
716
  fontSize: FontSize.smaller,
@@ -115,19 +719,38 @@ const SelectFilter = ({ value, options, onChange }) => {
115
719
  marginRight: "0.3em",
116
720
  marginLeft: "0.2em",
117
721
  }}
118
- >Scores:</span
119
- >
120
- <select
121
- class="form-select form-select-sm"
122
- aria-label=".sample-label"
123
- style=${{ fontSize: FontSize.smaller }}
124
- value=${value}
125
- onChange=${onChange}
722
+ >Filter:</span
126
723
  >
127
- ${options.map((option) => {
128
- return html`<option value="${option.value}">${option.text}</option>`;
129
- })}
130
- </select>
724
+ <div
725
+ ref=${editorRef}
726
+ style=${{ width: "300px" }}
727
+ class=${filteringResultInstant?.error ? ["filter-pending"] : []}
728
+ ></div>
729
+ <span
730
+ class="bi bi-question-circle"
731
+ style=${{
732
+ position: "relative",
733
+ marginLeft: "0.5em",
734
+ cursor: "help",
735
+ alignSelf: "center",
736
+ }}
737
+ data-tooltip=${filterTooltip}
738
+ data-tooltip-position="bottom-left"
739
+ ></span>
131
740
  </div>
132
741
  `;
133
742
  };
743
+
744
+ const filterTooltip = `
745
+ Filter samples by:
746
+ • Scores
747
+ • Input and target regex search: input_contains, target_contains
748
+
749
+ Supported expressions:
750
+ • Arithmetic: +, -, *, /, mod, ^
751
+ • Comparison: <, <=, >, >=, ==, !=, including chain comparisons, e.g. “10 <= x < 20”
752
+ • Boolean: and, or, not
753
+ • Regex matching: ~= (case-sensitive)
754
+ • Set operations: in, not in; e.g. “x in (2, 3, 5)”
755
+ • Functions: min, max, abs, round, floor, ceil, sqrt, log, log2, log10
756
+ `.trim();