inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. inspect_ai/_cli/common.py +3 -1
  2. inspect_ai/_cli/eval.py +15 -9
  3. inspect_ai/_display/core/active.py +4 -1
  4. inspect_ai/_display/core/config.py +3 -3
  5. inspect_ai/_display/core/panel.py +7 -3
  6. inspect_ai/_display/plain/__init__.py +0 -0
  7. inspect_ai/_display/plain/display.py +203 -0
  8. inspect_ai/_display/rich/display.py +0 -5
  9. inspect_ai/_display/textual/widgets/port_mappings.py +110 -0
  10. inspect_ai/_display/textual/widgets/samples.py +79 -12
  11. inspect_ai/_display/textual/widgets/sandbox.py +37 -0
  12. inspect_ai/_eval/eval.py +10 -1
  13. inspect_ai/_eval/loader.py +79 -19
  14. inspect_ai/_eval/registry.py +6 -0
  15. inspect_ai/_eval/score.py +3 -1
  16. inspect_ai/_eval/task/results.py +51 -22
  17. inspect_ai/_eval/task/run.py +47 -13
  18. inspect_ai/_eval/task/sandbox.py +10 -5
  19. inspect_ai/_util/constants.py +1 -0
  20. inspect_ai/_util/port_names.py +61 -0
  21. inspect_ai/_util/text.py +23 -0
  22. inspect_ai/_view/www/App.css +31 -1
  23. inspect_ai/_view/www/dist/assets/index.css +31 -1
  24. inspect_ai/_view/www/dist/assets/index.js +25498 -2044
  25. inspect_ai/_view/www/log-schema.json +32 -2
  26. inspect_ai/_view/www/package.json +2 -0
  27. inspect_ai/_view/www/src/App.mjs +14 -16
  28. inspect_ai/_view/www/src/Types.mjs +1 -2
  29. inspect_ai/_view/www/src/api/Types.ts +133 -0
  30. inspect_ai/_view/www/src/api/{api-browser.mjs → api-browser.ts} +25 -13
  31. inspect_ai/_view/www/src/api/api-http.ts +219 -0
  32. inspect_ai/_view/www/src/api/api-shared.ts +47 -0
  33. inspect_ai/_view/www/src/api/{api-vscode.mjs → api-vscode.ts} +22 -19
  34. inspect_ai/_view/www/src/api/{client-api.mjs → client-api.ts} +93 -53
  35. inspect_ai/_view/www/src/api/index.ts +51 -0
  36. inspect_ai/_view/www/src/api/jsonrpc.ts +225 -0
  37. inspect_ai/_view/www/src/components/ChatView.mjs +133 -43
  38. inspect_ai/_view/www/src/components/DownloadButton.mjs +1 -1
  39. inspect_ai/_view/www/src/components/ExpandablePanel.mjs +0 -4
  40. inspect_ai/_view/www/src/components/LargeModal.mjs +19 -20
  41. inspect_ai/_view/www/src/components/TabSet.mjs +3 -1
  42. inspect_ai/_view/www/src/components/VirtualList.mjs +266 -84
  43. inspect_ai/_view/www/src/index.js +77 -4
  44. inspect_ai/_view/www/src/log/{remoteLogFile.mjs → remoteLogFile.ts} +62 -46
  45. inspect_ai/_view/www/src/navbar/Navbar.mjs +4 -1
  46. inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +19 -10
  47. inspect_ai/_view/www/src/samples/SampleDialog.mjs +5 -1
  48. inspect_ai/_view/www/src/samples/SampleDisplay.mjs +23 -15
  49. inspect_ai/_view/www/src/samples/SampleList.mjs +19 -49
  50. inspect_ai/_view/www/src/samples/SampleScores.mjs +1 -1
  51. inspect_ai/_view/www/src/samples/SampleTranscript.mjs +8 -3
  52. inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +38 -26
  53. inspect_ai/_view/www/src/samples/SamplesTab.mjs +14 -11
  54. inspect_ai/_view/www/src/samples/SamplesTools.mjs +8 -8
  55. inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +712 -89
  56. inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +2 -2
  57. inspect_ai/_view/www/src/samples/tools/filters.mjs +260 -87
  58. inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs +24 -2
  59. inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs +29 -24
  60. inspect_ai/_view/www/src/samples/transcript/EventRow.mjs +1 -1
  61. inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs +24 -2
  62. inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs +24 -2
  63. inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs +31 -10
  64. inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.mjs +24 -2
  65. inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.mjs +23 -2
  66. inspect_ai/_view/www/src/samples/transcript/ScoreEventView.mjs +24 -2
  67. inspect_ai/_view/www/src/samples/transcript/StepEventView.mjs +33 -3
  68. inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.mjs +25 -2
  69. inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs +25 -2
  70. inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs +193 -11
  71. inspect_ai/_view/www/src/samples/transcript/Types.mjs +10 -0
  72. inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs +26 -2
  73. inspect_ai/_view/www/src/types/log.d.ts +13 -2
  74. inspect_ai/_view/www/src/utils/Format.mjs +10 -3
  75. inspect_ai/_view/www/src/utils/{Json.mjs → json-worker.ts} +13 -9
  76. inspect_ai/_view/www/src/utils/vscode.ts +36 -0
  77. inspect_ai/_view/www/src/workspace/WorkSpace.mjs +11 -5
  78. inspect_ai/_view/www/vite.config.js +7 -0
  79. inspect_ai/_view/www/yarn.lock +116 -0
  80. inspect_ai/approval/_human/__init__.py +0 -0
  81. inspect_ai/approval/_human/manager.py +1 -1
  82. inspect_ai/approval/_policy.py +12 -6
  83. inspect_ai/log/_log.py +1 -1
  84. inspect_ai/log/_samples.py +16 -0
  85. inspect_ai/log/_transcript.py +4 -1
  86. inspect_ai/model/_call_tools.py +59 -0
  87. inspect_ai/model/_conversation.py +16 -7
  88. inspect_ai/model/_generate_config.py +12 -12
  89. inspect_ai/model/_model.py +117 -18
  90. inspect_ai/model/_model_output.py +22 -2
  91. inspect_ai/model/_openai.py +383 -0
  92. inspect_ai/model/_providers/anthropic.py +152 -55
  93. inspect_ai/model/_providers/azureai.py +21 -21
  94. inspect_ai/model/_providers/bedrock.py +37 -40
  95. inspect_ai/model/_providers/goodfire.py +248 -0
  96. inspect_ai/model/_providers/google.py +46 -54
  97. inspect_ai/model/_providers/groq.py +7 -3
  98. inspect_ai/model/_providers/hf.py +6 -0
  99. inspect_ai/model/_providers/mistral.py +13 -12
  100. inspect_ai/model/_providers/openai.py +51 -218
  101. inspect_ai/model/_providers/openai_o1.py +11 -12
  102. inspect_ai/model/_providers/providers.py +23 -1
  103. inspect_ai/model/_providers/together.py +12 -12
  104. inspect_ai/model/_providers/util/__init__.py +2 -3
  105. inspect_ai/model/_providers/util/hf_handler.py +1 -1
  106. inspect_ai/model/_providers/util/llama31.py +1 -1
  107. inspect_ai/model/_providers/util/util.py +0 -76
  108. inspect_ai/model/_providers/vertex.py +1 -4
  109. inspect_ai/scorer/_metric.py +3 -0
  110. inspect_ai/scorer/_reducer/reducer.py +1 -1
  111. inspect_ai/scorer/_scorer.py +4 -3
  112. inspect_ai/solver/__init__.py +4 -5
  113. inspect_ai/solver/_basic_agent.py +1 -1
  114. inspect_ai/solver/_bridge/__init__.py +3 -0
  115. inspect_ai/solver/_bridge/bridge.py +100 -0
  116. inspect_ai/solver/_bridge/patch.py +170 -0
  117. inspect_ai/solver/_prompt.py +35 -5
  118. inspect_ai/solver/_solver.py +6 -0
  119. inspect_ai/solver/_task_state.py +80 -38
  120. inspect_ai/tool/__init__.py +2 -0
  121. inspect_ai/tool/_tool.py +12 -1
  122. inspect_ai/tool/_tool_call.py +10 -0
  123. inspect_ai/tool/_tool_def.py +16 -5
  124. inspect_ai/tool/_tool_with.py +21 -4
  125. inspect_ai/tool/beta/__init__.py +5 -0
  126. inspect_ai/tool/beta/_computer/__init__.py +3 -0
  127. inspect_ai/tool/beta/_computer/_common.py +133 -0
  128. inspect_ai/tool/beta/_computer/_computer.py +155 -0
  129. inspect_ai/tool/beta/_computer/_computer_split.py +198 -0
  130. inspect_ai/tool/beta/_computer/_resources/Dockerfile +100 -0
  131. inspect_ai/tool/beta/_computer/_resources/README.md +30 -0
  132. inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh +18 -0
  133. inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh +20 -0
  134. inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh +48 -0
  135. inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh +13 -0
  136. inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh +48 -0
  137. inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop +10 -0
  138. inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop +10 -0
  139. inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop +10 -0
  140. inspect_ai/tool/beta/_computer/_resources/tool/__init__.py +0 -0
  141. inspect_ai/tool/beta/_computer/_resources/tool/_logger.py +22 -0
  142. inspect_ai/tool/beta/_computer/_resources/tool/_run.py +42 -0
  143. inspect_ai/tool/beta/_computer/_resources/tool/_tool_result.py +33 -0
  144. inspect_ai/tool/beta/_computer/_resources/tool/_x11_client.py +262 -0
  145. inspect_ai/tool/beta/_computer/_resources/tool/computer_tool.py +85 -0
  146. inspect_ai/tool/beta/_computer/_resources/tool/requirements.txt +0 -0
  147. inspect_ai/util/__init__.py +2 -0
  148. inspect_ai/util/_display.py +5 -0
  149. inspect_ai/util/_limit.py +26 -0
  150. inspect_ai/util/_sandbox/docker/docker.py +64 -1
  151. inspect_ai/util/_sandbox/docker/internal.py +3 -1
  152. inspect_ai/util/_sandbox/docker/prereqs.py +1 -1
  153. inspect_ai/util/_sandbox/environment.py +14 -0
  154. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/METADATA +3 -2
  155. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/RECORD +159 -126
  156. inspect_ai/_view/www/src/api/Types.mjs +0 -117
  157. inspect_ai/_view/www/src/api/api-http.mjs +0 -300
  158. inspect_ai/_view/www/src/api/api-shared.mjs +0 -10
  159. inspect_ai/_view/www/src/api/index.mjs +0 -49
  160. inspect_ai/_view/www/src/api/jsonrpc.mjs +0 -208
  161. inspect_ai/_view/www/src/samples/transcript/TranscriptState.mjs +0 -70
  162. inspect_ai/_view/www/src/utils/vscode.mjs +0 -16
  163. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/LICENSE +0 -0
  164. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/WHEEL +0 -0
  165. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/entry_points.txt +0 -0
  166. {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/top_level.txt +0 -0
@@ -89,9 +89,9 @@ const sortId = (a, b) => {
89
89
  * Sorts a list of samples
90
90
  *
91
91
  * @param {string} sort - The sort direction
92
- * @param {import("../../api/Types.mjs").SampleSummary[]} samples - The samples
92
+ * @param {import("../../api/Types.ts").SampleSummary[]} samples - The samples
93
93
  * @param {import("../SamplesDescriptor.mjs").SamplesDescriptor} samplesDescriptor - The samples descriptor
94
- * @returns {{ sorted: import("../../api/Types.mjs").SampleSummary[], order: 'asc' | 'desc' }} An object with sorted samples and the sort order.
94
+ * @returns {{ sorted: import("../../api/Types.ts").SampleSummary[], order: 'asc' | 'desc' }} An object with sorted samples and the sort order.
95
95
  */
96
96
  export const sortSamples = (sort, samples, samplesDescriptor) => {
97
97
  const sortedSamples = samples.sort((a, b) => {
@@ -1,114 +1,287 @@
1
- import { kScoreTypeCategorical, kScoreTypeNumeric } from "../../constants.mjs";
2
- import { isNumeric } from "../../utils/Type.mjs";
1
+ import { compileExpression } from "filtrex";
2
+ import { kScoreTypeBoolean } from "../../constants.mjs";
3
+ import { inputString } from "../../utils/Format.mjs";
3
4
 
4
5
  /**
5
- * Gets a filter function for the specified type
6
+ * @typedef {Object} FilterError
7
+ * @property {number=} from - The start of the error.
8
+ * @property {number=} to - The end of the error.
9
+ * @property {string} message - The error message.
10
+ * @property {"warning" | "error"} severity - The severity of the error.
11
+ */
12
+
13
+ /**
14
+ * Coerces a value to the type expected by the score.
6
15
  *
7
- * @param {import("../../Types.mjs").ScoreFilter} filter - The parameters for the component.
8
- * @returns {(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean | undefined} the function
16
+ * @param {any} value
17
+ * @param {import("../../samples/SamplesDescriptor.mjs").ScoreDescriptor} descriptor
18
+ * @returns {any}
9
19
  */
10
- export const filterFnForType = (filter) => {
11
- if (filter.type) {
12
- return filterFnsForType[filter.type];
20
+ const coerceValue = (value, descriptor) => {
21
+ if (descriptor && descriptor.scoreType === kScoreTypeBoolean) {
22
+ return Boolean(value);
13
23
  } else {
14
- return undefined;
24
+ return value;
15
25
  }
16
26
  };
17
27
 
18
28
  /**
19
- * @type{(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean}
29
+ * @param {any} value
30
+ * @returns {boolean}
20
31
  */
21
- const filterCategory = (descriptor, sample, value) => {
22
- const score = descriptor.selectedScore(sample);
23
- if (typeof score.value === "string") {
24
- return score.value.toLowerCase() === value?.toLowerCase();
25
- } else if (typeof score.value === "object") {
26
- return JSON.stringify(score.value) == value;
27
- } else {
28
- return String(score.value) === value;
32
+ const isFilteringSupportedForValue = (value) =>
33
+ ["string", "number", "boolean"].includes(typeof value);
34
+
35
+ /**
36
+ * Returns the names of scores that are not allowed to be used as short names in
37
+ * filter expressions because they are not unique. This should be applied only to
38
+ * the nested scores, not to the top-level scorer names.
39
+ *
40
+ * @param {import("../../Types.mjs").ScoreLabel[]} scores
41
+ * @returns {Set<string>}
42
+ */
43
+ const bannedShortScoreNames = (scores) => {
44
+ const used = new Set();
45
+ const banned = new Set();
46
+ for (const { scorer, name } of scores) {
47
+ banned.add(scorer);
48
+ if (used.has(name)) {
49
+ banned.add(name);
50
+ } else {
51
+ used.add(name);
52
+ }
29
53
  }
54
+ return banned;
30
55
  };
31
56
 
32
57
  /**
33
- * @type{(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean}
58
+ * Generates a dictionary of variables that can be used in the filter expression.
59
+ * High-level scorer metrics can be accessed by name directly.
60
+ * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
61
+ * directly by name when it is unique.
62
+ *
63
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
64
+ * @param {import("../../types/log").Scores1} sampleScores
65
+ * @returns {Object<string, any>}
34
66
  */
35
- const filterText = (descriptor, sample, value) => {
36
- const score = descriptor.selectedScore(sample);
37
- if (!value) {
38
- return true;
39
- } else {
40
- if (isNumeric(value)) {
41
- if (typeof score.value === "number") {
42
- return score.value === Number(value);
43
- } else {
44
- return Number(score.value) === Number(value);
45
- }
46
- } else {
47
- const filters = [
48
- {
49
- prefix: ">=",
50
- fn: (score, val) => {
51
- return score >= val;
52
- },
53
- },
54
- {
55
- prefix: "<=",
56
- fn: (score, val) => {
57
- return score <= val;
58
- },
59
- },
60
- {
61
- prefix: ">",
62
- fn: (score, val) => {
63
- return score > val;
64
- },
65
- },
66
- {
67
- prefix: "<",
68
- fn: (score, val) => {
69
- return score < val;
70
- },
71
- },
72
- {
73
- prefix: "=",
74
- fn: (score, val) => {
75
- return score === val;
76
- },
77
- },
78
- {
79
- prefix: "!=",
80
- fn: (score, val) => {
81
- return score !== val;
82
- },
83
- },
84
- ];
67
+ const scoreVariables = (evalDescriptor, sampleScores) => {
68
+ const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
69
+ const variables = {};
85
70
 
86
- for (const filter of filters) {
87
- if (value?.startsWith(filter.prefix)) {
88
- const val = value.slice(filter.prefix.length).trim();
89
- if (!val) {
90
- return true;
91
- }
71
+ /**
72
+ * @param {import("../../Types.mjs").ScoreLabel} scoreLabel
73
+ * @param {any} value
74
+ */
75
+ const addScore = (variableName, scoreLabel, value) => {
76
+ const coercedValue = coerceValue(
77
+ value,
78
+ evalDescriptor.scoreDescriptor(scoreLabel),
79
+ );
80
+ if (isFilteringSupportedForValue(coercedValue)) {
81
+ variables[variableName] = coercedValue;
82
+ }
83
+ };
92
84
 
93
- const num = Number(val);
94
- return filter.fn(score.value, num);
85
+ for (const [scorer, score] of Object.entries(sampleScores)) {
86
+ addScore(scorer, { scorer, name: scorer }, score.value);
87
+ if (typeof score.value === "object") {
88
+ for (const [name, value] of Object.entries(score.value)) {
89
+ addScore(`${scorer}.${name}`, { scorer, name }, value);
90
+ if (!bannedShortNames.has(name)) {
91
+ addScore(name, { scorer, name }, value);
95
92
  }
96
93
  }
97
- if (typeof score.value === "string") {
98
- return score.value.toLowerCase() === value?.toLowerCase();
99
- } else {
100
- return String(score.value) === value;
101
- }
102
94
  }
103
95
  }
96
+ return variables;
97
+ };
98
+
99
+ /**
100
+ * @typedef {Object} ScoreFilterItem
101
+ * @property {string | undefined} shortName - The short name of the score, if doesn't conflict with other short names.
102
+ * @property {string | undefined} qualifiedName - The `scorer.score` name for children of complex scorers.
103
+ * @property {string} canonicalName - The canonical name: either `shortName` or `qualifiedName` (at least one must exist).
104
+ * @property {string} tooltip - The informational tooltip for the score.
105
+ * @property {string[]} categories - Category values for categorical scores.
106
+ * @property {string} scoreType - The type of the score (e.g., 'numeric', 'categorical', 'boolean').
107
+ */
108
+
109
+ /**
110
+ * Generates a dictionary of variables that can be used in the filter expression.
111
+ * High-level scorer metrics can be accessed by name directly.
112
+ * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
113
+ * directly by name when it is unique.
114
+ *
115
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
116
+ * @returns {ScoreFilterItem[]}
117
+ */
118
+ export const scoreFilterItems = (evalDescriptor) => {
119
+ /** @type {ScoreFilterItem[]} */
120
+ const items = [];
121
+ const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
122
+ const valueToString = (value) =>
123
+ typeof value === "string" ? `"${value}"` : String(value);
124
+
125
+ /**
126
+ * @param {string | undefined} shortName
127
+ * @param {string | undefined} qualifiedName
128
+ * @param {import("../../Types.mjs").ScoreLabel} scoreLabel
129
+ */
130
+ const addScore = (shortName, qualifiedName, scoreLabel) => {
131
+ const canonicalName = shortName || qualifiedName;
132
+ const descriptor = evalDescriptor.scoreDescriptor(scoreLabel);
133
+ const scoreType = descriptor?.scoreType;
134
+ if (!descriptor) {
135
+ items.push({
136
+ shortName,
137
+ qualifiedName,
138
+ canonicalName,
139
+ tooltip: undefined,
140
+ categories: [],
141
+ scoreType,
142
+ });
143
+ return;
144
+ }
145
+ var tooltip = `${canonicalName}: ${descriptor.scoreType}`;
146
+ var categories = [];
147
+ if (descriptor.min !== undefined || descriptor.max !== undefined) {
148
+ const rounded = (num) => {
149
+ // Additional round-trip to remove trailing zeros.
150
+ return parseFloat(num.toPrecision(3)).toString();
151
+ };
152
+ tooltip += `\nrange: ${rounded(descriptor.min)} to ${rounded(descriptor.max)}`;
153
+ }
154
+ if (descriptor.categories) {
155
+ tooltip += `\ncategories: ${descriptor.categories.map((cat) => cat.val).join(", ")}`;
156
+ categories = descriptor.categories.map((cat) => valueToString(cat.val));
157
+ }
158
+ items.push({
159
+ shortName,
160
+ qualifiedName,
161
+ canonicalName,
162
+ tooltip,
163
+ categories,
164
+ scoreType,
165
+ });
166
+ };
167
+
168
+ for (const { name, scorer } of evalDescriptor.scores) {
169
+ const hasShortName = name === scorer || !bannedShortNames.has(name);
170
+ const hasQualifiedName = name !== scorer;
171
+ const shortName = hasShortName ? name : undefined;
172
+ const qualifiedName = hasQualifiedName ? `${scorer}.${name}` : undefined;
173
+ addScore(shortName, qualifiedName, { name, scorer });
174
+ }
175
+ return items;
104
176
  };
105
177
 
106
178
  /**
107
- * A dictionary that maps filter types to their respective filter functions.
179
+ * TODO: Add case-insensitive string comparison.
108
180
  *
109
- * @type {Record<string, (descriptor, sample, value) => boolean>}
181
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
182
+ * @param {import("../../api/Types.mjs").SampleSummary} sample
183
+ * @param {string} filterValue
184
+ * @returns {{matches: boolean, error: FilterError | undefined}}
185
+ */
186
+ export const filterExpression = (evalDescriptor, sample, filterValue) => {
187
+ try {
188
+ /** @type {(regex: string) => boolean} */
189
+ const inputContains = (regex) => {
190
+ return inputString(sample.input).some((msg) =>
191
+ msg.match(new RegExp(regex, "i")),
192
+ );
193
+ };
194
+ /** @type {(regex: string) => boolean} */
195
+ const targetContains = (regex) => {
196
+ let targets = Array.isArray(sample.target)
197
+ ? sample.target
198
+ : [sample.target];
199
+ return targets.some((target) => target.match(new RegExp(regex, "i")));
200
+ };
201
+
202
+ const extraFunctions = {
203
+ input_contains: inputContains,
204
+ target_contains: targetContains,
205
+ };
206
+ const expression = compileExpression(filterValue, { extraFunctions });
207
+ const vars = scoreVariables(evalDescriptor, sample.scores);
208
+ const result = expression(vars);
209
+ if (typeof result === "boolean") {
210
+ return { matches: result, error: undefined };
211
+ } else if (result instanceof Error) {
212
+ throw result;
213
+ } else {
214
+ throw new TypeError(
215
+ `Filter expression returned a non-boolean value: ${result}`,
216
+ );
217
+ }
218
+ } catch (error) {
219
+ if (error instanceof ReferenceError) {
220
+ const propertyName = error["propertyName"];
221
+ if (propertyName) {
222
+ const regex = new RegExp(`\\b${propertyName}\\b`);
223
+ const match = regex.exec(filterValue);
224
+ if (match) {
225
+ return {
226
+ matches: false,
227
+ error: {
228
+ from: match.index,
229
+ to: match.index + propertyName.length,
230
+ message: error.message,
231
+ severity: "warning",
232
+ },
233
+ };
234
+ }
235
+ }
236
+ }
237
+ if (
238
+ error.message.startsWith("Parse error") ||
239
+ error.message.startsWith("Lexical error")
240
+ ) {
241
+ // Filterex uses formatting like this:
242
+ // foo and
243
+ // ----^
244
+ const from = error.message.match(/^(-*)\^$/m)?.[1]?.length;
245
+ return {
246
+ matches: false,
247
+ error: {
248
+ from: from,
249
+ message: "Syntax error",
250
+ severity: "error",
251
+ },
252
+ };
253
+ }
254
+
255
+ return {
256
+ matches: false,
257
+ error: {
258
+ message: error.message,
259
+ severity: "error",
260
+ },
261
+ };
262
+ }
263
+ };
264
+
265
+ /**
266
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
267
+ * @param {import("../../api/Types.mjs").SampleSummary[]} samples
268
+ * @param {string} filterValue
269
+ * @returns {{result: import("../../api/Types.mjs").SampleSummary[], error: FilterError | undefined}}
110
270
  */
111
- const filterFnsForType = {
112
- [kScoreTypeCategorical]: filterCategory,
113
- [kScoreTypeNumeric]: filterText,
271
+ export const filterSamples = (evalDescriptor, samples, filterValue) => {
272
+ var error = undefined;
273
+ const result = samples.filter((sample) => {
274
+ if (filterValue) {
275
+ const { matches, error: sampleError } = filterExpression(
276
+ evalDescriptor,
277
+ sample,
278
+ filterValue,
279
+ );
280
+ error ||= sampleError;
281
+ return matches;
282
+ } else {
283
+ return true;
284
+ }
285
+ });
286
+ return { result, error };
114
287
  };
@@ -12,11 +12,33 @@ import { formatDateTime } from "../../utils/Format.mjs";
12
12
  * @param { string } props.id - The id of this event.
13
13
  * @param {import("../../types/log").ErrorEvent} props.event - The event object to display.
14
14
  * @param { Object } props.style - The style of this event.
15
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
16
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
15
17
  * @returns {import("preact").JSX.Element} The component.
16
18
  */
17
- export const ErrorEventView = ({ id, event, style }) => {
19
+ export const ErrorEventView = ({
20
+ id,
21
+ event,
22
+ style,
23
+ eventState,
24
+ setEventState,
25
+ }) => {
18
26
  return html`
19
- <${EventPanel} id=${id} title="Error" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.error} style=${style}>
27
+ <${EventPanel}
28
+ id=${id}
29
+ title="Error"
30
+ subTitle=${formatDateTime(new Date(event.timestamp))}
31
+ icon=${ApplicationIcons.error}
32
+ style=${style}
33
+ selectedNav=${eventState.selectedNav || ""}
34
+ onSelectedNav=${(selectedNav) => {
35
+ setEventState({ ...eventState, selectedNav });
36
+ }}
37
+ collapsed=${eventState.collapsed}
38
+ onCollapsed=${(collapsed) => {
39
+ setEventState({ ...eventState, collapsed });
40
+ }}
41
+ >
20
42
  <${ANSIDisplay} output=${event.error.traceback_ansi} style=${{ fontSize: "clamp(0.5rem, calc(0.25em + 1vw), 0.8rem)", margin: "0.5em 0" }}/>
21
43
  </${EventPanel}>`;
22
44
  };
@@ -1,6 +1,5 @@
1
1
  // @ts-check
2
2
  import { html } from "htm/preact";
3
- import { useEffect, useMemo, useState } from "preact/hooks";
4
3
  import { ApplicationIcons } from "../../appearance/Icons.mjs";
5
4
  import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
6
5
 
@@ -16,8 +15,12 @@ import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
16
15
  * @param {string | undefined} props.icon - The icon of the event
17
16
  * @param {number | undefined} props.titleColor - The title color of this item
18
17
  * @param {boolean | undefined} props.collapse - Default collapse behavior for card. If omitted, not collapsible.
18
+ * @param {(collapse: boolean) => void} props.onCollapsed - handler for when collapsing
19
+ * @param {boolean} props.collapsed - handler for when collapsing
19
20
  * @param {Object} props.style - The style properties passed to the component.
20
21
  * @param {Object} props.titleStyle - The style properties passed to the title component.
22
+ * @param {(nav: string) => void} props.onSelectedNav - Handler for selected nav changed
23
+ * @param {string} props.selectedNav - The selected nav for this panel
21
24
  * @param {import("preact").ComponentChildren} props.children - The rendered event.
22
25
  * @returns {import("preact").JSX.Element} The component.
23
26
  */
@@ -30,22 +33,16 @@ export const EventPanel = ({
30
33
  icon,
31
34
  titleColor,
32
35
  collapse,
36
+ collapsed,
37
+ onCollapsed,
33
38
  style,
34
39
  titleStyle,
35
40
  children,
41
+ onSelectedNav,
42
+ selectedNav,
36
43
  }) => {
37
44
  const hasCollapse = collapse !== undefined;
38
- const [collapsed, setCollapsed] = useState(!!collapse);
39
- const [selectedNav, setSelectedNav] = useState("");
40
-
41
- const filteredArrChildren = useMemo(() => {
42
- const arrChildren = Array.isArray(children) ? children : [children];
43
- return arrChildren.filter((child) => !!child);
44
- }, [children]);
45
-
46
- useEffect(() => {
47
- setSelectedNav(pillId(0));
48
- }, [filteredArrChildren]);
45
+ const isCollapsed = collapsed === undefined ? collapse : collapsed;
49
46
 
50
47
  /**
51
48
  * Generates the id for the navigation pill.
@@ -57,6 +54,11 @@ export const EventPanel = ({
57
54
  return `${id}-nav-pill-${index}`;
58
55
  };
59
56
 
57
+ const filteredArrChildren = (
58
+ Array.isArray(children) ? children : [children]
59
+ ).filter((child) => !!child);
60
+ const defaultPillId = pillId(0);
61
+
60
62
  const gridColumns = [];
61
63
  if (hasCollapse) {
62
64
  gridColumns.push("minmax(0, max-content)");
@@ -87,9 +89,9 @@ export const EventPanel = ({
87
89
  ${hasCollapse
88
90
  ? html`<i
89
91
  onclick=${() => {
90
- setCollapsed(!collapsed);
92
+ onCollapsed(!isCollapsed);
91
93
  }}
92
- class=${collapsed
94
+ class=${isCollapsed
93
95
  ? ApplicationIcons.chevron.right
94
96
  : ApplicationIcons.chevron.down}
95
97
  />`
@@ -103,7 +105,7 @@ export const EventPanel = ({
103
105
  ...titleStyle,
104
106
  }}
105
107
  onclick=${() => {
106
- setCollapsed(!collapsed);
108
+ onCollapsed(!isCollapsed);
107
109
  }}
108
110
  />`
109
111
  : ""}
@@ -115,14 +117,14 @@ export const EventPanel = ({
115
117
  ...titleStyle,
116
118
  }}
117
119
  onclick=${() => {
118
- setCollapsed(!collapsed);
120
+ onCollapsed(!isCollapsed);
119
121
  }}
120
122
  >
121
123
  ${title}
122
124
  </div>
123
125
  <div
124
126
  onclick=${() => {
125
- setCollapsed(!collapsed);
127
+ onCollapsed(!isCollapsed);
126
128
  }}
127
129
  ></div>
128
130
  <div
@@ -132,7 +134,7 @@ export const EventPanel = ({
132
134
  marginRight: "0.2em",
133
135
  }}
134
136
  onclick=${() => {
135
- setCollapsed(!collapsed);
137
+ onCollapsed(!isCollapsed);
136
138
  }}
137
139
  >
138
140
  ${collapsed ? text : ""}
@@ -144,7 +146,7 @@ export const EventPanel = ({
144
146
  flexDirection: "columns",
145
147
  }}
146
148
  >
147
- ${(!hasCollapse || !collapsed) &&
149
+ ${(!hasCollapse || !isCollapsed) &&
148
150
  filteredArrChildren &&
149
151
  filteredArrChildren.length > 1
150
152
  ? html` <${EventNavs}
@@ -160,8 +162,8 @@ export const EventPanel = ({
160
162
  target: pillId(index),
161
163
  };
162
164
  })}
163
- selectedNav=${selectedNav}
164
- setSelectedNav=${setSelectedNav}
165
+ selectedNav=${selectedNav || defaultPillId}
166
+ setSelectedNav=${onSelectedNav}
165
167
  />`
166
168
  : ""}
167
169
  </div>
@@ -172,7 +174,6 @@ export const EventPanel = ({
172
174
  id=${id}
173
175
  style=${{
174
176
  padding: "0.625rem",
175
- marginBottom: "0.625rem",
176
177
  border: "solid 1px var(--bs-light-border-subtle)",
177
178
  borderRadius: "var(--bs-border-radius)",
178
179
  ...style,
@@ -184,14 +185,18 @@ export const EventPanel = ({
184
185
  class="tab-content"
185
186
  style=${{
186
187
  padding: "0",
187
- display: !hasCollapse || !collapsed ? "inherit" : "none",
188
+ display: !hasCollapse || !isCollapsed ? "inherit" : "none",
188
189
  }}
189
190
  >
190
191
  ${filteredArrChildren?.map((child, index) => {
191
192
  const id = pillId(index);
193
+ const isSelected = selectedNav
194
+ ? id === selectedNav
195
+ : id === defaultPillId;
196
+
192
197
  return html`<div
193
198
  id=${id}
194
- class="tab-pane show ${id === selectedNav ? "active" : ""}"
199
+ class="tab-pane show ${isSelected ? "active" : ""}"
195
200
  >
196
201
  ${child}
197
202
  </div>`;
@@ -34,7 +34,7 @@ export const EventRow = ({ title, icon, style, children }) => {
34
34
  class="card"
35
35
  style=${{
36
36
  padding: "0.4em",
37
- marginBottom: "0.4em",
37
+ marginBottom: "0",
38
38
  border: "solid 1px var(--bs-light-border-subtle)",
39
39
  borderRadius: "var(--bs-border-radius)",
40
40
  ...style,
@@ -13,9 +13,17 @@ import { formatDateTime } from "../../utils/Format.mjs";
13
13
  * @param { string } props.id - The id of this event.
14
14
  * @param { Object } props.style - The depth of this event.
15
15
  * @param {import("../../types/log").InfoEvent} props.event - The event object to display.
16
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
17
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
16
18
  * @returns {import("preact").JSX.Element} The component.
17
19
  */
18
- export const InfoEventView = ({ id, event, style }) => {
20
+ export const InfoEventView = ({
21
+ id,
22
+ event,
23
+ style,
24
+ eventState,
25
+ setEventState,
26
+ }) => {
19
27
  const panels = [];
20
28
  if (typeof event.data === "string") {
21
29
  panels.push(
@@ -31,7 +39,21 @@ export const InfoEventView = ({ id, event, style }) => {
31
39
  }
32
40
 
33
41
  return html`
34
- <${EventPanel} id=${id} title="Info" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.info} style=${style}>
42
+ <${EventPanel}
43
+ id=${id}
44
+ title="Info"
45
+ subTitle=${formatDateTime(new Date(event.timestamp))}
46
+ icon=${ApplicationIcons.info}
47
+ style=${style}
48
+ selectedNav=${eventState.selectedNav || ""}
49
+ onSelectedNav=${(selectedNav) => {
50
+ setEventState({ ...eventState, selectedNav });
51
+ }}
52
+ collapsed=${eventState.collapsed}
53
+ onCollapsed=${(collapsed) => {
54
+ setEventState({ ...eventState, collapsed });
55
+ }}
56
+ >
35
57
  ${panels}
36
58
  </${EventPanel}>`;
37
59
  };
@@ -12,11 +12,33 @@ import { formatDateTime } from "../../utils/Format.mjs";
12
12
  * @param { string } props.id - The id of this event.
13
13
  * @param {import("../../types/log").InputEvent} props.event - The event object to display.
14
14
  * @param { Object } props.style - The style of this event.
15
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
16
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
15
17
  * @returns {import("preact").JSX.Element} The component.
16
18
  */
17
- export const InputEventView = ({ id, event, style }) => {
19
+ export const InputEventView = ({
20
+ id,
21
+ event,
22
+ style,
23
+ eventState,
24
+ setEventState,
25
+ }) => {
18
26
  return html`
19
- <${EventPanel} id=${id} title="Input" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.input} style=${style}>
27
+ <${EventPanel}
28
+ id=${id}
29
+ title="Input"
30
+ subTitle=${formatDateTime(new Date(event.timestamp))}
31
+ icon=${ApplicationIcons.input}
32
+ style=${style}
33
+ selectedNav=${eventState.selectedNav || ""}
34
+ onSelectedNav=${(selectedNav) => {
35
+ setEventState({ ...eventState, selectedNav });
36
+ }}
37
+ collapsed=${eventState.collapsed}
38
+ onCollapsed=${(collapsed) => {
39
+ setEventState({ ...eventState, collapsed });
40
+ }}
41
+ >
20
42
  <${ANSIDisplay} output=${event.input_ansi} style=${{ fontSize: "clamp(0.4rem, 1.15vw, 0.9rem)", ...style }}/>
21
43
  </${EventPanel}>`;
22
44
  };