inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inspect_ai/_cli/common.py +3 -1
- inspect_ai/_cli/eval.py +15 -9
- inspect_ai/_display/core/active.py +4 -1
- inspect_ai/_display/core/config.py +3 -3
- inspect_ai/_display/core/panel.py +7 -3
- inspect_ai/_display/plain/__init__.py +0 -0
- inspect_ai/_display/plain/display.py +203 -0
- inspect_ai/_display/rich/display.py +0 -5
- inspect_ai/_display/textual/widgets/port_mappings.py +110 -0
- inspect_ai/_display/textual/widgets/samples.py +79 -12
- inspect_ai/_display/textual/widgets/sandbox.py +37 -0
- inspect_ai/_eval/eval.py +10 -1
- inspect_ai/_eval/loader.py +79 -19
- inspect_ai/_eval/registry.py +6 -0
- inspect_ai/_eval/score.py +3 -1
- inspect_ai/_eval/task/results.py +51 -22
- inspect_ai/_eval/task/run.py +47 -13
- inspect_ai/_eval/task/sandbox.py +10 -5
- inspect_ai/_util/constants.py +1 -0
- inspect_ai/_util/port_names.py +61 -0
- inspect_ai/_util/text.py +23 -0
- inspect_ai/_view/www/App.css +31 -1
- inspect_ai/_view/www/dist/assets/index.css +31 -1
- inspect_ai/_view/www/dist/assets/index.js +25498 -2044
- inspect_ai/_view/www/log-schema.json +32 -2
- inspect_ai/_view/www/package.json +2 -0
- inspect_ai/_view/www/src/App.mjs +14 -16
- inspect_ai/_view/www/src/Types.mjs +1 -2
- inspect_ai/_view/www/src/api/Types.ts +133 -0
- inspect_ai/_view/www/src/api/{api-browser.mjs → api-browser.ts} +25 -13
- inspect_ai/_view/www/src/api/api-http.ts +219 -0
- inspect_ai/_view/www/src/api/api-shared.ts +47 -0
- inspect_ai/_view/www/src/api/{api-vscode.mjs → api-vscode.ts} +22 -19
- inspect_ai/_view/www/src/api/{client-api.mjs → client-api.ts} +93 -53
- inspect_ai/_view/www/src/api/index.ts +51 -0
- inspect_ai/_view/www/src/api/jsonrpc.ts +225 -0
- inspect_ai/_view/www/src/components/ChatView.mjs +133 -43
- inspect_ai/_view/www/src/components/DownloadButton.mjs +1 -1
- inspect_ai/_view/www/src/components/ExpandablePanel.mjs +0 -4
- inspect_ai/_view/www/src/components/LargeModal.mjs +19 -20
- inspect_ai/_view/www/src/components/TabSet.mjs +3 -1
- inspect_ai/_view/www/src/components/VirtualList.mjs +266 -84
- inspect_ai/_view/www/src/index.js +77 -4
- inspect_ai/_view/www/src/log/{remoteLogFile.mjs → remoteLogFile.ts} +62 -46
- inspect_ai/_view/www/src/navbar/Navbar.mjs +4 -1
- inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +19 -10
- inspect_ai/_view/www/src/samples/SampleDialog.mjs +5 -1
- inspect_ai/_view/www/src/samples/SampleDisplay.mjs +23 -15
- inspect_ai/_view/www/src/samples/SampleList.mjs +19 -49
- inspect_ai/_view/www/src/samples/SampleScores.mjs +1 -1
- inspect_ai/_view/www/src/samples/SampleTranscript.mjs +8 -3
- inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +38 -26
- inspect_ai/_view/www/src/samples/SamplesTab.mjs +14 -11
- inspect_ai/_view/www/src/samples/SamplesTools.mjs +8 -8
- inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +712 -89
- inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +2 -2
- inspect_ai/_view/www/src/samples/tools/filters.mjs +260 -87
- inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs +29 -24
- inspect_ai/_view/www/src/samples/transcript/EventRow.mjs +1 -1
- inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs +31 -10
- inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.mjs +23 -2
- inspect_ai/_view/www/src/samples/transcript/ScoreEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/StepEventView.mjs +33 -3
- inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.mjs +25 -2
- inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs +25 -2
- inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs +193 -11
- inspect_ai/_view/www/src/samples/transcript/Types.mjs +10 -0
- inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs +26 -2
- inspect_ai/_view/www/src/types/log.d.ts +13 -2
- inspect_ai/_view/www/src/utils/Format.mjs +10 -3
- inspect_ai/_view/www/src/utils/{Json.mjs → json-worker.ts} +13 -9
- inspect_ai/_view/www/src/utils/vscode.ts +36 -0
- inspect_ai/_view/www/src/workspace/WorkSpace.mjs +11 -5
- inspect_ai/_view/www/vite.config.js +7 -0
- inspect_ai/_view/www/yarn.lock +116 -0
- inspect_ai/approval/_human/__init__.py +0 -0
- inspect_ai/approval/_human/manager.py +1 -1
- inspect_ai/approval/_policy.py +12 -6
- inspect_ai/log/_log.py +1 -1
- inspect_ai/log/_samples.py +16 -0
- inspect_ai/log/_transcript.py +4 -1
- inspect_ai/model/_call_tools.py +59 -0
- inspect_ai/model/_conversation.py +16 -7
- inspect_ai/model/_generate_config.py +12 -12
- inspect_ai/model/_model.py +117 -18
- inspect_ai/model/_model_output.py +22 -2
- inspect_ai/model/_openai.py +383 -0
- inspect_ai/model/_providers/anthropic.py +152 -55
- inspect_ai/model/_providers/azureai.py +21 -21
- inspect_ai/model/_providers/bedrock.py +37 -40
- inspect_ai/model/_providers/goodfire.py +248 -0
- inspect_ai/model/_providers/google.py +46 -54
- inspect_ai/model/_providers/groq.py +7 -3
- inspect_ai/model/_providers/hf.py +6 -0
- inspect_ai/model/_providers/mistral.py +13 -12
- inspect_ai/model/_providers/openai.py +51 -218
- inspect_ai/model/_providers/openai_o1.py +11 -12
- inspect_ai/model/_providers/providers.py +23 -1
- inspect_ai/model/_providers/together.py +12 -12
- inspect_ai/model/_providers/util/__init__.py +2 -3
- inspect_ai/model/_providers/util/hf_handler.py +1 -1
- inspect_ai/model/_providers/util/llama31.py +1 -1
- inspect_ai/model/_providers/util/util.py +0 -76
- inspect_ai/model/_providers/vertex.py +1 -4
- inspect_ai/scorer/_metric.py +3 -0
- inspect_ai/scorer/_reducer/reducer.py +1 -1
- inspect_ai/scorer/_scorer.py +4 -3
- inspect_ai/solver/__init__.py +4 -5
- inspect_ai/solver/_basic_agent.py +1 -1
- inspect_ai/solver/_bridge/__init__.py +3 -0
- inspect_ai/solver/_bridge/bridge.py +100 -0
- inspect_ai/solver/_bridge/patch.py +170 -0
- inspect_ai/solver/_prompt.py +35 -5
- inspect_ai/solver/_solver.py +6 -0
- inspect_ai/solver/_task_state.py +80 -38
- inspect_ai/tool/__init__.py +2 -0
- inspect_ai/tool/_tool.py +12 -1
- inspect_ai/tool/_tool_call.py +10 -0
- inspect_ai/tool/_tool_def.py +16 -5
- inspect_ai/tool/_tool_with.py +21 -4
- inspect_ai/tool/beta/__init__.py +5 -0
- inspect_ai/tool/beta/_computer/__init__.py +3 -0
- inspect_ai/tool/beta/_computer/_common.py +133 -0
- inspect_ai/tool/beta/_computer/_computer.py +155 -0
- inspect_ai/tool/beta/_computer/_computer_split.py +198 -0
- inspect_ai/tool/beta/_computer/_resources/Dockerfile +100 -0
- inspect_ai/tool/beta/_computer/_resources/README.md +30 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh +18 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh +20 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh +48 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh +13 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh +48 -0
- inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop +10 -0
- inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop +10 -0
- inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop +10 -0
- inspect_ai/tool/beta/_computer/_resources/tool/__init__.py +0 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_logger.py +22 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_run.py +42 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_tool_result.py +33 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_x11_client.py +262 -0
- inspect_ai/tool/beta/_computer/_resources/tool/computer_tool.py +85 -0
- inspect_ai/tool/beta/_computer/_resources/tool/requirements.txt +0 -0
- inspect_ai/util/__init__.py +2 -0
- inspect_ai/util/_display.py +5 -0
- inspect_ai/util/_limit.py +26 -0
- inspect_ai/util/_sandbox/docker/docker.py +64 -1
- inspect_ai/util/_sandbox/docker/internal.py +3 -1
- inspect_ai/util/_sandbox/docker/prereqs.py +1 -1
- inspect_ai/util/_sandbox/environment.py +14 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/METADATA +3 -2
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/RECORD +159 -126
- inspect_ai/_view/www/src/api/Types.mjs +0 -117
- inspect_ai/_view/www/src/api/api-http.mjs +0 -300
- inspect_ai/_view/www/src/api/api-shared.mjs +0 -10
- inspect_ai/_view/www/src/api/index.mjs +0 -49
- inspect_ai/_view/www/src/api/jsonrpc.mjs +0 -208
- inspect_ai/_view/www/src/samples/transcript/TranscriptState.mjs +0 -70
- inspect_ai/_view/www/src/utils/vscode.mjs +0 -16
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/LICENSE +0 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/WHEEL +0 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/entry_points.txt +0 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/top_level.txt +0 -0
@@ -89,9 +89,9 @@ const sortId = (a, b) => {
|
|
89
89
|
* Sorts a list of samples
|
90
90
|
*
|
91
91
|
* @param {string} sort - The sort direction
|
92
|
-
* @param {import("../../api/Types.
|
92
|
+
* @param {import("../../api/Types.ts").SampleSummary[]} samples - The samples
|
93
93
|
* @param {import("../SamplesDescriptor.mjs").SamplesDescriptor} samplesDescriptor - The samples descriptor
|
94
|
-
* @returns {{ sorted: import("../../api/Types.
|
94
|
+
* @returns {{ sorted: import("../../api/Types.ts").SampleSummary[], order: 'asc' | 'desc' }} An object with sorted samples and the sort order.
|
95
95
|
*/
|
96
96
|
export const sortSamples = (sort, samples, samplesDescriptor) => {
|
97
97
|
const sortedSamples = samples.sort((a, b) => {
|
@@ -1,114 +1,287 @@
|
|
1
|
-
import {
|
2
|
-
import {
|
1
|
+
import { compileExpression } from "filtrex";
|
2
|
+
import { kScoreTypeBoolean } from "../../constants.mjs";
|
3
|
+
import { inputString } from "../../utils/Format.mjs";
|
3
4
|
|
4
5
|
/**
|
5
|
-
*
|
6
|
+
* @typedef {Object} FilterError
|
7
|
+
* @property {number=} from - The start of the error.
|
8
|
+
* @property {number=} to - The end of the error.
|
9
|
+
* @property {string} message - The error message.
|
10
|
+
* @property {"warning" | "error"} severity - The severity of the error.
|
11
|
+
*/
|
12
|
+
|
13
|
+
/**
|
14
|
+
* Coerces a value to the type expected by the score.
|
6
15
|
*
|
7
|
-
* @param {
|
8
|
-
* @
|
16
|
+
* @param {any} value
|
17
|
+
* @param {import("../../samples/SamplesDescriptor.mjs").ScoreDescriptor} descriptor
|
18
|
+
* @returns {any}
|
9
19
|
*/
|
10
|
-
|
11
|
-
if (
|
12
|
-
return
|
20
|
+
const coerceValue = (value, descriptor) => {
|
21
|
+
if (descriptor && descriptor.scoreType === kScoreTypeBoolean) {
|
22
|
+
return Boolean(value);
|
13
23
|
} else {
|
14
|
-
return
|
24
|
+
return value;
|
15
25
|
}
|
16
26
|
};
|
17
27
|
|
18
28
|
/**
|
19
|
-
* @
|
29
|
+
* @param {any} value
|
30
|
+
* @returns {boolean}
|
20
31
|
*/
|
21
|
-
const
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
32
|
+
const isFilteringSupportedForValue = (value) =>
|
33
|
+
["string", "number", "boolean"].includes(typeof value);
|
34
|
+
|
35
|
+
/**
|
36
|
+
* Returns the names of scores that are not allowed to be used as short names in
|
37
|
+
* filter expressions because they are not unique. This should be applied only to
|
38
|
+
* the nested scores, not to the top-level scorer names.
|
39
|
+
*
|
40
|
+
* @param {import("../../Types.mjs").ScoreLabel[]} scores
|
41
|
+
* @returns {Set<string>}
|
42
|
+
*/
|
43
|
+
const bannedShortScoreNames = (scores) => {
|
44
|
+
const used = new Set();
|
45
|
+
const banned = new Set();
|
46
|
+
for (const { scorer, name } of scores) {
|
47
|
+
banned.add(scorer);
|
48
|
+
if (used.has(name)) {
|
49
|
+
banned.add(name);
|
50
|
+
} else {
|
51
|
+
used.add(name);
|
52
|
+
}
|
29
53
|
}
|
54
|
+
return banned;
|
30
55
|
};
|
31
56
|
|
32
57
|
/**
|
33
|
-
*
|
58
|
+
* Generates a dictionary of variables that can be used in the filter expression.
|
59
|
+
* High-level scorer metrics can be accessed by name directly.
|
60
|
+
* Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
|
61
|
+
* directly by name when it is unique.
|
62
|
+
*
|
63
|
+
* @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
|
64
|
+
* @param {import("../../types/log").Scores1} sampleScores
|
65
|
+
* @returns {Object<string, any>}
|
34
66
|
*/
|
35
|
-
const
|
36
|
-
const
|
37
|
-
|
38
|
-
return true;
|
39
|
-
} else {
|
40
|
-
if (isNumeric(value)) {
|
41
|
-
if (typeof score.value === "number") {
|
42
|
-
return score.value === Number(value);
|
43
|
-
} else {
|
44
|
-
return Number(score.value) === Number(value);
|
45
|
-
}
|
46
|
-
} else {
|
47
|
-
const filters = [
|
48
|
-
{
|
49
|
-
prefix: ">=",
|
50
|
-
fn: (score, val) => {
|
51
|
-
return score >= val;
|
52
|
-
},
|
53
|
-
},
|
54
|
-
{
|
55
|
-
prefix: "<=",
|
56
|
-
fn: (score, val) => {
|
57
|
-
return score <= val;
|
58
|
-
},
|
59
|
-
},
|
60
|
-
{
|
61
|
-
prefix: ">",
|
62
|
-
fn: (score, val) => {
|
63
|
-
return score > val;
|
64
|
-
},
|
65
|
-
},
|
66
|
-
{
|
67
|
-
prefix: "<",
|
68
|
-
fn: (score, val) => {
|
69
|
-
return score < val;
|
70
|
-
},
|
71
|
-
},
|
72
|
-
{
|
73
|
-
prefix: "=",
|
74
|
-
fn: (score, val) => {
|
75
|
-
return score === val;
|
76
|
-
},
|
77
|
-
},
|
78
|
-
{
|
79
|
-
prefix: "!=",
|
80
|
-
fn: (score, val) => {
|
81
|
-
return score !== val;
|
82
|
-
},
|
83
|
-
},
|
84
|
-
];
|
67
|
+
const scoreVariables = (evalDescriptor, sampleScores) => {
|
68
|
+
const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
|
69
|
+
const variables = {};
|
85
70
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
71
|
+
/**
|
72
|
+
* @param {import("../../Types.mjs").ScoreLabel} scoreLabel
|
73
|
+
* @param {any} value
|
74
|
+
*/
|
75
|
+
const addScore = (variableName, scoreLabel, value) => {
|
76
|
+
const coercedValue = coerceValue(
|
77
|
+
value,
|
78
|
+
evalDescriptor.scoreDescriptor(scoreLabel),
|
79
|
+
);
|
80
|
+
if (isFilteringSupportedForValue(coercedValue)) {
|
81
|
+
variables[variableName] = coercedValue;
|
82
|
+
}
|
83
|
+
};
|
92
84
|
|
93
|
-
|
94
|
-
|
85
|
+
for (const [scorer, score] of Object.entries(sampleScores)) {
|
86
|
+
addScore(scorer, { scorer, name: scorer }, score.value);
|
87
|
+
if (typeof score.value === "object") {
|
88
|
+
for (const [name, value] of Object.entries(score.value)) {
|
89
|
+
addScore(`${scorer}.${name}`, { scorer, name }, value);
|
90
|
+
if (!bannedShortNames.has(name)) {
|
91
|
+
addScore(name, { scorer, name }, value);
|
95
92
|
}
|
96
93
|
}
|
97
|
-
if (typeof score.value === "string") {
|
98
|
-
return score.value.toLowerCase() === value?.toLowerCase();
|
99
|
-
} else {
|
100
|
-
return String(score.value) === value;
|
101
|
-
}
|
102
94
|
}
|
103
95
|
}
|
96
|
+
return variables;
|
97
|
+
};
|
98
|
+
|
99
|
+
/**
|
100
|
+
* @typedef {Object} ScoreFilterItem
|
101
|
+
* @property {string | undefined} shortName - The short name of the score, if doesn't conflict with other short names.
|
102
|
+
* @property {string | undefined} qualifiedName - The `scorer.score` name for children of complex scorers.
|
103
|
+
* @property {string} canonicalName - The canonical name: either `shortName` or `qualifiedName` (at least one must exist).
|
104
|
+
* @property {string} tooltip - The informational tooltip for the score.
|
105
|
+
* @property {string[]} categories - Category values for categorical scores.
|
106
|
+
* @property {string} scoreType - The type of the score (e.g., 'numeric', 'categorical', 'boolean').
|
107
|
+
*/
|
108
|
+
|
109
|
+
/**
|
110
|
+
* Generates a dictionary of variables that can be used in the filter expression.
|
111
|
+
* High-level scorer metrics can be accessed by name directly.
|
112
|
+
* Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
|
113
|
+
* directly by name when it is unique.
|
114
|
+
*
|
115
|
+
* @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
|
116
|
+
* @returns {ScoreFilterItem[]}
|
117
|
+
*/
|
118
|
+
export const scoreFilterItems = (evalDescriptor) => {
|
119
|
+
/** @type {ScoreFilterItem[]} */
|
120
|
+
const items = [];
|
121
|
+
const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
|
122
|
+
const valueToString = (value) =>
|
123
|
+
typeof value === "string" ? `"${value}"` : String(value);
|
124
|
+
|
125
|
+
/**
|
126
|
+
* @param {string | undefined} shortName
|
127
|
+
* @param {string | undefined} qualifiedName
|
128
|
+
* @param {import("../../Types.mjs").ScoreLabel} scoreLabel
|
129
|
+
*/
|
130
|
+
const addScore = (shortName, qualifiedName, scoreLabel) => {
|
131
|
+
const canonicalName = shortName || qualifiedName;
|
132
|
+
const descriptor = evalDescriptor.scoreDescriptor(scoreLabel);
|
133
|
+
const scoreType = descriptor?.scoreType;
|
134
|
+
if (!descriptor) {
|
135
|
+
items.push({
|
136
|
+
shortName,
|
137
|
+
qualifiedName,
|
138
|
+
canonicalName,
|
139
|
+
tooltip: undefined,
|
140
|
+
categories: [],
|
141
|
+
scoreType,
|
142
|
+
});
|
143
|
+
return;
|
144
|
+
}
|
145
|
+
var tooltip = `${canonicalName}: ${descriptor.scoreType}`;
|
146
|
+
var categories = [];
|
147
|
+
if (descriptor.min !== undefined || descriptor.max !== undefined) {
|
148
|
+
const rounded = (num) => {
|
149
|
+
// Additional round-trip to remove trailing zeros.
|
150
|
+
return parseFloat(num.toPrecision(3)).toString();
|
151
|
+
};
|
152
|
+
tooltip += `\nrange: ${rounded(descriptor.min)} to ${rounded(descriptor.max)}`;
|
153
|
+
}
|
154
|
+
if (descriptor.categories) {
|
155
|
+
tooltip += `\ncategories: ${descriptor.categories.map((cat) => cat.val).join(", ")}`;
|
156
|
+
categories = descriptor.categories.map((cat) => valueToString(cat.val));
|
157
|
+
}
|
158
|
+
items.push({
|
159
|
+
shortName,
|
160
|
+
qualifiedName,
|
161
|
+
canonicalName,
|
162
|
+
tooltip,
|
163
|
+
categories,
|
164
|
+
scoreType,
|
165
|
+
});
|
166
|
+
};
|
167
|
+
|
168
|
+
for (const { name, scorer } of evalDescriptor.scores) {
|
169
|
+
const hasShortName = name === scorer || !bannedShortNames.has(name);
|
170
|
+
const hasQualifiedName = name !== scorer;
|
171
|
+
const shortName = hasShortName ? name : undefined;
|
172
|
+
const qualifiedName = hasQualifiedName ? `${scorer}.${name}` : undefined;
|
173
|
+
addScore(shortName, qualifiedName, { name, scorer });
|
174
|
+
}
|
175
|
+
return items;
|
104
176
|
};
|
105
177
|
|
106
178
|
/**
|
107
|
-
*
|
179
|
+
* TODO: Add case-insensitive string comparison.
|
108
180
|
*
|
109
|
-
* @
|
181
|
+
* @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
|
182
|
+
* @param {import("../../api/Types.mjs").SampleSummary} sample
|
183
|
+
* @param {string} filterValue
|
184
|
+
* @returns {{matches: boolean, error: FilterError | undefined}}
|
185
|
+
*/
|
186
|
+
export const filterExpression = (evalDescriptor, sample, filterValue) => {
|
187
|
+
try {
|
188
|
+
/** @type {(regex: string) => boolean} */
|
189
|
+
const inputContains = (regex) => {
|
190
|
+
return inputString(sample.input).some((msg) =>
|
191
|
+
msg.match(new RegExp(regex, "i")),
|
192
|
+
);
|
193
|
+
};
|
194
|
+
/** @type {(regex: string) => boolean} */
|
195
|
+
const targetContains = (regex) => {
|
196
|
+
let targets = Array.isArray(sample.target)
|
197
|
+
? sample.target
|
198
|
+
: [sample.target];
|
199
|
+
return targets.some((target) => target.match(new RegExp(regex, "i")));
|
200
|
+
};
|
201
|
+
|
202
|
+
const extraFunctions = {
|
203
|
+
input_contains: inputContains,
|
204
|
+
target_contains: targetContains,
|
205
|
+
};
|
206
|
+
const expression = compileExpression(filterValue, { extraFunctions });
|
207
|
+
const vars = scoreVariables(evalDescriptor, sample.scores);
|
208
|
+
const result = expression(vars);
|
209
|
+
if (typeof result === "boolean") {
|
210
|
+
return { matches: result, error: undefined };
|
211
|
+
} else if (result instanceof Error) {
|
212
|
+
throw result;
|
213
|
+
} else {
|
214
|
+
throw new TypeError(
|
215
|
+
`Filter expression returned a non-boolean value: ${result}`,
|
216
|
+
);
|
217
|
+
}
|
218
|
+
} catch (error) {
|
219
|
+
if (error instanceof ReferenceError) {
|
220
|
+
const propertyName = error["propertyName"];
|
221
|
+
if (propertyName) {
|
222
|
+
const regex = new RegExp(`\\b${propertyName}\\b`);
|
223
|
+
const match = regex.exec(filterValue);
|
224
|
+
if (match) {
|
225
|
+
return {
|
226
|
+
matches: false,
|
227
|
+
error: {
|
228
|
+
from: match.index,
|
229
|
+
to: match.index + propertyName.length,
|
230
|
+
message: error.message,
|
231
|
+
severity: "warning",
|
232
|
+
},
|
233
|
+
};
|
234
|
+
}
|
235
|
+
}
|
236
|
+
}
|
237
|
+
if (
|
238
|
+
error.message.startsWith("Parse error") ||
|
239
|
+
error.message.startsWith("Lexical error")
|
240
|
+
) {
|
241
|
+
// Filterex uses formatting like this:
|
242
|
+
// foo and
|
243
|
+
// ----^
|
244
|
+
const from = error.message.match(/^(-*)\^$/m)?.[1]?.length;
|
245
|
+
return {
|
246
|
+
matches: false,
|
247
|
+
error: {
|
248
|
+
from: from,
|
249
|
+
message: "Syntax error",
|
250
|
+
severity: "error",
|
251
|
+
},
|
252
|
+
};
|
253
|
+
}
|
254
|
+
|
255
|
+
return {
|
256
|
+
matches: false,
|
257
|
+
error: {
|
258
|
+
message: error.message,
|
259
|
+
severity: "error",
|
260
|
+
},
|
261
|
+
};
|
262
|
+
}
|
263
|
+
};
|
264
|
+
|
265
|
+
/**
|
266
|
+
* @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
|
267
|
+
* @param {import("../../api/Types.mjs").SampleSummary[]} samples
|
268
|
+
* @param {string} filterValue
|
269
|
+
* @returns {{result: import("../../api/Types.mjs").SampleSummary[], error: FilterError | undefined}}
|
110
270
|
*/
|
111
|
-
const
|
112
|
-
|
113
|
-
|
271
|
+
export const filterSamples = (evalDescriptor, samples, filterValue) => {
|
272
|
+
var error = undefined;
|
273
|
+
const result = samples.filter((sample) => {
|
274
|
+
if (filterValue) {
|
275
|
+
const { matches, error: sampleError } = filterExpression(
|
276
|
+
evalDescriptor,
|
277
|
+
sample,
|
278
|
+
filterValue,
|
279
|
+
);
|
280
|
+
error ||= sampleError;
|
281
|
+
return matches;
|
282
|
+
} else {
|
283
|
+
return true;
|
284
|
+
}
|
285
|
+
});
|
286
|
+
return { result, error };
|
114
287
|
};
|
@@ -12,11 +12,33 @@ import { formatDateTime } from "../../utils/Format.mjs";
|
|
12
12
|
* @param { string } props.id - The id of this event.
|
13
13
|
* @param {import("../../types/log").ErrorEvent} props.event - The event object to display.
|
14
14
|
* @param { Object } props.style - The style of this event.
|
15
|
+
* @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
|
16
|
+
* @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
|
15
17
|
* @returns {import("preact").JSX.Element} The component.
|
16
18
|
*/
|
17
|
-
export const ErrorEventView = ({
|
19
|
+
export const ErrorEventView = ({
|
20
|
+
id,
|
21
|
+
event,
|
22
|
+
style,
|
23
|
+
eventState,
|
24
|
+
setEventState,
|
25
|
+
}) => {
|
18
26
|
return html`
|
19
|
-
<${EventPanel}
|
27
|
+
<${EventPanel}
|
28
|
+
id=${id}
|
29
|
+
title="Error"
|
30
|
+
subTitle=${formatDateTime(new Date(event.timestamp))}
|
31
|
+
icon=${ApplicationIcons.error}
|
32
|
+
style=${style}
|
33
|
+
selectedNav=${eventState.selectedNav || ""}
|
34
|
+
onSelectedNav=${(selectedNav) => {
|
35
|
+
setEventState({ ...eventState, selectedNav });
|
36
|
+
}}
|
37
|
+
collapsed=${eventState.collapsed}
|
38
|
+
onCollapsed=${(collapsed) => {
|
39
|
+
setEventState({ ...eventState, collapsed });
|
40
|
+
}}
|
41
|
+
>
|
20
42
|
<${ANSIDisplay} output=${event.error.traceback_ansi} style=${{ fontSize: "clamp(0.5rem, calc(0.25em + 1vw), 0.8rem)", margin: "0.5em 0" }}/>
|
21
43
|
</${EventPanel}>`;
|
22
44
|
};
|
@@ -1,6 +1,5 @@
|
|
1
1
|
// @ts-check
|
2
2
|
import { html } from "htm/preact";
|
3
|
-
import { useEffect, useMemo, useState } from "preact/hooks";
|
4
3
|
import { ApplicationIcons } from "../../appearance/Icons.mjs";
|
5
4
|
import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
|
6
5
|
|
@@ -16,8 +15,12 @@ import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
|
|
16
15
|
* @param {string | undefined} props.icon - The icon of the event
|
17
16
|
* @param {number | undefined} props.titleColor - The title color of this item
|
18
17
|
* @param {boolean | undefined} props.collapse - Default collapse behavior for card. If omitted, not collapsible.
|
18
|
+
* @param {(collapse: boolean) => void} props.onCollapsed - handler for when collapsing
|
19
|
+
* @param {boolean} props.collapsed - handler for when collapsing
|
19
20
|
* @param {Object} props.style - The style properties passed to the component.
|
20
21
|
* @param {Object} props.titleStyle - The style properties passed to the title component.
|
22
|
+
* @param {(nav: string) => void} props.onSelectedNav - Handler for selected nav changed
|
23
|
+
* @param {string} props.selectedNav - The selected nav for this panel
|
21
24
|
* @param {import("preact").ComponentChildren} props.children - The rendered event.
|
22
25
|
* @returns {import("preact").JSX.Element} The component.
|
23
26
|
*/
|
@@ -30,22 +33,16 @@ export const EventPanel = ({
|
|
30
33
|
icon,
|
31
34
|
titleColor,
|
32
35
|
collapse,
|
36
|
+
collapsed,
|
37
|
+
onCollapsed,
|
33
38
|
style,
|
34
39
|
titleStyle,
|
35
40
|
children,
|
41
|
+
onSelectedNav,
|
42
|
+
selectedNav,
|
36
43
|
}) => {
|
37
44
|
const hasCollapse = collapse !== undefined;
|
38
|
-
const
|
39
|
-
const [selectedNav, setSelectedNav] = useState("");
|
40
|
-
|
41
|
-
const filteredArrChildren = useMemo(() => {
|
42
|
-
const arrChildren = Array.isArray(children) ? children : [children];
|
43
|
-
return arrChildren.filter((child) => !!child);
|
44
|
-
}, [children]);
|
45
|
-
|
46
|
-
useEffect(() => {
|
47
|
-
setSelectedNav(pillId(0));
|
48
|
-
}, [filteredArrChildren]);
|
45
|
+
const isCollapsed = collapsed === undefined ? collapse : collapsed;
|
49
46
|
|
50
47
|
/**
|
51
48
|
* Generates the id for the navigation pill.
|
@@ -57,6 +54,11 @@ export const EventPanel = ({
|
|
57
54
|
return `${id}-nav-pill-${index}`;
|
58
55
|
};
|
59
56
|
|
57
|
+
const filteredArrChildren = (
|
58
|
+
Array.isArray(children) ? children : [children]
|
59
|
+
).filter((child) => !!child);
|
60
|
+
const defaultPillId = pillId(0);
|
61
|
+
|
60
62
|
const gridColumns = [];
|
61
63
|
if (hasCollapse) {
|
62
64
|
gridColumns.push("minmax(0, max-content)");
|
@@ -87,9 +89,9 @@ export const EventPanel = ({
|
|
87
89
|
${hasCollapse
|
88
90
|
? html`<i
|
89
91
|
onclick=${() => {
|
90
|
-
|
92
|
+
onCollapsed(!isCollapsed);
|
91
93
|
}}
|
92
|
-
class=${
|
94
|
+
class=${isCollapsed
|
93
95
|
? ApplicationIcons.chevron.right
|
94
96
|
: ApplicationIcons.chevron.down}
|
95
97
|
/>`
|
@@ -103,7 +105,7 @@ export const EventPanel = ({
|
|
103
105
|
...titleStyle,
|
104
106
|
}}
|
105
107
|
onclick=${() => {
|
106
|
-
|
108
|
+
onCollapsed(!isCollapsed);
|
107
109
|
}}
|
108
110
|
/>`
|
109
111
|
: ""}
|
@@ -115,14 +117,14 @@ export const EventPanel = ({
|
|
115
117
|
...titleStyle,
|
116
118
|
}}
|
117
119
|
onclick=${() => {
|
118
|
-
|
120
|
+
onCollapsed(!isCollapsed);
|
119
121
|
}}
|
120
122
|
>
|
121
123
|
${title}
|
122
124
|
</div>
|
123
125
|
<div
|
124
126
|
onclick=${() => {
|
125
|
-
|
127
|
+
onCollapsed(!isCollapsed);
|
126
128
|
}}
|
127
129
|
></div>
|
128
130
|
<div
|
@@ -132,7 +134,7 @@ export const EventPanel = ({
|
|
132
134
|
marginRight: "0.2em",
|
133
135
|
}}
|
134
136
|
onclick=${() => {
|
135
|
-
|
137
|
+
onCollapsed(!isCollapsed);
|
136
138
|
}}
|
137
139
|
>
|
138
140
|
${collapsed ? text : ""}
|
@@ -144,7 +146,7 @@ export const EventPanel = ({
|
|
144
146
|
flexDirection: "columns",
|
145
147
|
}}
|
146
148
|
>
|
147
|
-
${(!hasCollapse || !
|
149
|
+
${(!hasCollapse || !isCollapsed) &&
|
148
150
|
filteredArrChildren &&
|
149
151
|
filteredArrChildren.length > 1
|
150
152
|
? html` <${EventNavs}
|
@@ -160,8 +162,8 @@ export const EventPanel = ({
|
|
160
162
|
target: pillId(index),
|
161
163
|
};
|
162
164
|
})}
|
163
|
-
selectedNav=${selectedNav}
|
164
|
-
setSelectedNav=${
|
165
|
+
selectedNav=${selectedNav || defaultPillId}
|
166
|
+
setSelectedNav=${onSelectedNav}
|
165
167
|
/>`
|
166
168
|
: ""}
|
167
169
|
</div>
|
@@ -172,7 +174,6 @@ export const EventPanel = ({
|
|
172
174
|
id=${id}
|
173
175
|
style=${{
|
174
176
|
padding: "0.625rem",
|
175
|
-
marginBottom: "0.625rem",
|
176
177
|
border: "solid 1px var(--bs-light-border-subtle)",
|
177
178
|
borderRadius: "var(--bs-border-radius)",
|
178
179
|
...style,
|
@@ -184,14 +185,18 @@ export const EventPanel = ({
|
|
184
185
|
class="tab-content"
|
185
186
|
style=${{
|
186
187
|
padding: "0",
|
187
|
-
display: !hasCollapse || !
|
188
|
+
display: !hasCollapse || !isCollapsed ? "inherit" : "none",
|
188
189
|
}}
|
189
190
|
>
|
190
191
|
${filteredArrChildren?.map((child, index) => {
|
191
192
|
const id = pillId(index);
|
193
|
+
const isSelected = selectedNav
|
194
|
+
? id === selectedNav
|
195
|
+
: id === defaultPillId;
|
196
|
+
|
192
197
|
return html`<div
|
193
198
|
id=${id}
|
194
|
-
class="tab-pane show ${
|
199
|
+
class="tab-pane show ${isSelected ? "active" : ""}"
|
195
200
|
>
|
196
201
|
${child}
|
197
202
|
</div>`;
|
@@ -34,7 +34,7 @@ export const EventRow = ({ title, icon, style, children }) => {
|
|
34
34
|
class="card"
|
35
35
|
style=${{
|
36
36
|
padding: "0.4em",
|
37
|
-
marginBottom: "0
|
37
|
+
marginBottom: "0",
|
38
38
|
border: "solid 1px var(--bs-light-border-subtle)",
|
39
39
|
borderRadius: "var(--bs-border-radius)",
|
40
40
|
...style,
|
@@ -13,9 +13,17 @@ import { formatDateTime } from "../../utils/Format.mjs";
|
|
13
13
|
* @param { string } props.id - The id of this event.
|
14
14
|
* @param { Object } props.style - The depth of this event.
|
15
15
|
* @param {import("../../types/log").InfoEvent} props.event - The event object to display.
|
16
|
+
* @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
|
17
|
+
* @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
|
16
18
|
* @returns {import("preact").JSX.Element} The component.
|
17
19
|
*/
|
18
|
-
export const InfoEventView = ({
|
20
|
+
export const InfoEventView = ({
|
21
|
+
id,
|
22
|
+
event,
|
23
|
+
style,
|
24
|
+
eventState,
|
25
|
+
setEventState,
|
26
|
+
}) => {
|
19
27
|
const panels = [];
|
20
28
|
if (typeof event.data === "string") {
|
21
29
|
panels.push(
|
@@ -31,7 +39,21 @@ export const InfoEventView = ({ id, event, style }) => {
|
|
31
39
|
}
|
32
40
|
|
33
41
|
return html`
|
34
|
-
<${EventPanel}
|
42
|
+
<${EventPanel}
|
43
|
+
id=${id}
|
44
|
+
title="Info"
|
45
|
+
subTitle=${formatDateTime(new Date(event.timestamp))}
|
46
|
+
icon=${ApplicationIcons.info}
|
47
|
+
style=${style}
|
48
|
+
selectedNav=${eventState.selectedNav || ""}
|
49
|
+
onSelectedNav=${(selectedNav) => {
|
50
|
+
setEventState({ ...eventState, selectedNav });
|
51
|
+
}}
|
52
|
+
collapsed=${eventState.collapsed}
|
53
|
+
onCollapsed=${(collapsed) => {
|
54
|
+
setEventState({ ...eventState, collapsed });
|
55
|
+
}}
|
56
|
+
>
|
35
57
|
${panels}
|
36
58
|
</${EventPanel}>`;
|
37
59
|
};
|
@@ -12,11 +12,33 @@ import { formatDateTime } from "../../utils/Format.mjs";
|
|
12
12
|
* @param { string } props.id - The id of this event.
|
13
13
|
* @param {import("../../types/log").InputEvent} props.event - The event object to display.
|
14
14
|
* @param { Object } props.style - The style of this event.
|
15
|
+
* @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
|
16
|
+
* @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
|
15
17
|
* @returns {import("preact").JSX.Element} The component.
|
16
18
|
*/
|
17
|
-
export const InputEventView = ({
|
19
|
+
export const InputEventView = ({
|
20
|
+
id,
|
21
|
+
event,
|
22
|
+
style,
|
23
|
+
eventState,
|
24
|
+
setEventState,
|
25
|
+
}) => {
|
18
26
|
return html`
|
19
|
-
<${EventPanel}
|
27
|
+
<${EventPanel}
|
28
|
+
id=${id}
|
29
|
+
title="Input"
|
30
|
+
subTitle=${formatDateTime(new Date(event.timestamp))}
|
31
|
+
icon=${ApplicationIcons.input}
|
32
|
+
style=${style}
|
33
|
+
selectedNav=${eventState.selectedNav || ""}
|
34
|
+
onSelectedNav=${(selectedNav) => {
|
35
|
+
setEventState({ ...eventState, selectedNav });
|
36
|
+
}}
|
37
|
+
collapsed=${eventState.collapsed}
|
38
|
+
onCollapsed=${(collapsed) => {
|
39
|
+
setEventState({ ...eventState, collapsed });
|
40
|
+
}}
|
41
|
+
>
|
20
42
|
<${ANSIDisplay} output=${event.input_ansi} style=${{ fontSize: "clamp(0.4rem, 1.15vw, 0.9rem)", ...style }}/>
|
21
43
|
</${EventPanel}>`;
|
22
44
|
};
|