inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inspect_ai/_cli/common.py +3 -1
- inspect_ai/_cli/eval.py +15 -9
- inspect_ai/_display/core/active.py +4 -1
- inspect_ai/_display/core/config.py +3 -3
- inspect_ai/_display/core/panel.py +7 -3
- inspect_ai/_display/plain/__init__.py +0 -0
- inspect_ai/_display/plain/display.py +203 -0
- inspect_ai/_display/rich/display.py +0 -5
- inspect_ai/_display/textual/widgets/port_mappings.py +110 -0
- inspect_ai/_display/textual/widgets/samples.py +79 -12
- inspect_ai/_display/textual/widgets/sandbox.py +37 -0
- inspect_ai/_eval/eval.py +10 -1
- inspect_ai/_eval/loader.py +79 -19
- inspect_ai/_eval/registry.py +6 -0
- inspect_ai/_eval/score.py +3 -1
- inspect_ai/_eval/task/results.py +51 -22
- inspect_ai/_eval/task/run.py +47 -13
- inspect_ai/_eval/task/sandbox.py +10 -5
- inspect_ai/_util/constants.py +1 -0
- inspect_ai/_util/port_names.py +61 -0
- inspect_ai/_util/text.py +23 -0
- inspect_ai/_view/www/App.css +31 -1
- inspect_ai/_view/www/dist/assets/index.css +31 -1
- inspect_ai/_view/www/dist/assets/index.js +25498 -2044
- inspect_ai/_view/www/log-schema.json +32 -2
- inspect_ai/_view/www/package.json +2 -0
- inspect_ai/_view/www/src/App.mjs +14 -16
- inspect_ai/_view/www/src/Types.mjs +1 -2
- inspect_ai/_view/www/src/api/Types.ts +133 -0
- inspect_ai/_view/www/src/api/{api-browser.mjs → api-browser.ts} +25 -13
- inspect_ai/_view/www/src/api/api-http.ts +219 -0
- inspect_ai/_view/www/src/api/api-shared.ts +47 -0
- inspect_ai/_view/www/src/api/{api-vscode.mjs → api-vscode.ts} +22 -19
- inspect_ai/_view/www/src/api/{client-api.mjs → client-api.ts} +93 -53
- inspect_ai/_view/www/src/api/index.ts +51 -0
- inspect_ai/_view/www/src/api/jsonrpc.ts +225 -0
- inspect_ai/_view/www/src/components/ChatView.mjs +133 -43
- inspect_ai/_view/www/src/components/DownloadButton.mjs +1 -1
- inspect_ai/_view/www/src/components/ExpandablePanel.mjs +0 -4
- inspect_ai/_view/www/src/components/LargeModal.mjs +19 -20
- inspect_ai/_view/www/src/components/TabSet.mjs +3 -1
- inspect_ai/_view/www/src/components/VirtualList.mjs +266 -84
- inspect_ai/_view/www/src/index.js +77 -4
- inspect_ai/_view/www/src/log/{remoteLogFile.mjs → remoteLogFile.ts} +62 -46
- inspect_ai/_view/www/src/navbar/Navbar.mjs +4 -1
- inspect_ai/_view/www/src/navbar/SecondaryBar.mjs +19 -10
- inspect_ai/_view/www/src/samples/SampleDialog.mjs +5 -1
- inspect_ai/_view/www/src/samples/SampleDisplay.mjs +23 -15
- inspect_ai/_view/www/src/samples/SampleList.mjs +19 -49
- inspect_ai/_view/www/src/samples/SampleScores.mjs +1 -1
- inspect_ai/_view/www/src/samples/SampleTranscript.mjs +8 -3
- inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs +38 -26
- inspect_ai/_view/www/src/samples/SamplesTab.mjs +14 -11
- inspect_ai/_view/www/src/samples/SamplesTools.mjs +8 -8
- inspect_ai/_view/www/src/samples/tools/SampleFilter.mjs +712 -89
- inspect_ai/_view/www/src/samples/tools/SortFilter.mjs +2 -2
- inspect_ai/_view/www/src/samples/tools/filters.mjs +260 -87
- inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs +29 -24
- inspect_ai/_view/www/src/samples/transcript/EventRow.mjs +1 -1
- inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs +31 -10
- inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.mjs +23 -2
- inspect_ai/_view/www/src/samples/transcript/ScoreEventView.mjs +24 -2
- inspect_ai/_view/www/src/samples/transcript/StepEventView.mjs +33 -3
- inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.mjs +25 -2
- inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs +25 -2
- inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs +193 -11
- inspect_ai/_view/www/src/samples/transcript/Types.mjs +10 -0
- inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs +26 -2
- inspect_ai/_view/www/src/types/log.d.ts +13 -2
- inspect_ai/_view/www/src/utils/Format.mjs +10 -3
- inspect_ai/_view/www/src/utils/{Json.mjs → json-worker.ts} +13 -9
- inspect_ai/_view/www/src/utils/vscode.ts +36 -0
- inspect_ai/_view/www/src/workspace/WorkSpace.mjs +11 -5
- inspect_ai/_view/www/vite.config.js +7 -0
- inspect_ai/_view/www/yarn.lock +116 -0
- inspect_ai/approval/_human/__init__.py +0 -0
- inspect_ai/approval/_human/manager.py +1 -1
- inspect_ai/approval/_policy.py +12 -6
- inspect_ai/log/_log.py +1 -1
- inspect_ai/log/_samples.py +16 -0
- inspect_ai/log/_transcript.py +4 -1
- inspect_ai/model/_call_tools.py +59 -0
- inspect_ai/model/_conversation.py +16 -7
- inspect_ai/model/_generate_config.py +12 -12
- inspect_ai/model/_model.py +117 -18
- inspect_ai/model/_model_output.py +22 -2
- inspect_ai/model/_openai.py +383 -0
- inspect_ai/model/_providers/anthropic.py +152 -55
- inspect_ai/model/_providers/azureai.py +21 -21
- inspect_ai/model/_providers/bedrock.py +37 -40
- inspect_ai/model/_providers/goodfire.py +248 -0
- inspect_ai/model/_providers/google.py +46 -54
- inspect_ai/model/_providers/groq.py +7 -3
- inspect_ai/model/_providers/hf.py +6 -0
- inspect_ai/model/_providers/mistral.py +13 -12
- inspect_ai/model/_providers/openai.py +51 -218
- inspect_ai/model/_providers/openai_o1.py +11 -12
- inspect_ai/model/_providers/providers.py +23 -1
- inspect_ai/model/_providers/together.py +12 -12
- inspect_ai/model/_providers/util/__init__.py +2 -3
- inspect_ai/model/_providers/util/hf_handler.py +1 -1
- inspect_ai/model/_providers/util/llama31.py +1 -1
- inspect_ai/model/_providers/util/util.py +0 -76
- inspect_ai/model/_providers/vertex.py +1 -4
- inspect_ai/scorer/_metric.py +3 -0
- inspect_ai/scorer/_reducer/reducer.py +1 -1
- inspect_ai/scorer/_scorer.py +4 -3
- inspect_ai/solver/__init__.py +4 -5
- inspect_ai/solver/_basic_agent.py +1 -1
- inspect_ai/solver/_bridge/__init__.py +3 -0
- inspect_ai/solver/_bridge/bridge.py +100 -0
- inspect_ai/solver/_bridge/patch.py +170 -0
- inspect_ai/solver/_prompt.py +35 -5
- inspect_ai/solver/_solver.py +6 -0
- inspect_ai/solver/_task_state.py +80 -38
- inspect_ai/tool/__init__.py +2 -0
- inspect_ai/tool/_tool.py +12 -1
- inspect_ai/tool/_tool_call.py +10 -0
- inspect_ai/tool/_tool_def.py +16 -5
- inspect_ai/tool/_tool_with.py +21 -4
- inspect_ai/tool/beta/__init__.py +5 -0
- inspect_ai/tool/beta/_computer/__init__.py +3 -0
- inspect_ai/tool/beta/_computer/_common.py +133 -0
- inspect_ai/tool/beta/_computer/_computer.py +155 -0
- inspect_ai/tool/beta/_computer/_computer_split.py +198 -0
- inspect_ai/tool/beta/_computer/_resources/Dockerfile +100 -0
- inspect_ai/tool/beta/_computer/_resources/README.md +30 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/entrypoint.sh +18 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/novnc_startup.sh +20 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh +48 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/xfce_startup.sh +13 -0
- inspect_ai/tool/beta/_computer/_resources/entrypoint/xvfb_startup.sh +48 -0
- inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Firefox Web Browser.desktop +10 -0
- inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Visual Studio Code.desktop +10 -0
- inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/XPaint.desktop +10 -0
- inspect_ai/tool/beta/_computer/_resources/tool/__init__.py +0 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_logger.py +22 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_run.py +42 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_tool_result.py +33 -0
- inspect_ai/tool/beta/_computer/_resources/tool/_x11_client.py +262 -0
- inspect_ai/tool/beta/_computer/_resources/tool/computer_tool.py +85 -0
- inspect_ai/tool/beta/_computer/_resources/tool/requirements.txt +0 -0
- inspect_ai/util/__init__.py +2 -0
- inspect_ai/util/_display.py +5 -0
- inspect_ai/util/_limit.py +26 -0
- inspect_ai/util/_sandbox/docker/docker.py +64 -1
- inspect_ai/util/_sandbox/docker/internal.py +3 -1
- inspect_ai/util/_sandbox/docker/prereqs.py +1 -1
- inspect_ai/util/_sandbox/environment.py +14 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/METADATA +3 -2
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/RECORD +159 -126
- inspect_ai/_view/www/src/api/Types.mjs +0 -117
- inspect_ai/_view/www/src/api/api-http.mjs +0 -300
- inspect_ai/_view/www/src/api/api-shared.mjs +0 -10
- inspect_ai/_view/www/src/api/index.mjs +0 -49
- inspect_ai/_view/www/src/api/jsonrpc.mjs +0 -208
- inspect_ai/_view/www/src/samples/transcript/TranscriptState.mjs +0 -70
- inspect_ai/_view/www/src/utils/vscode.mjs +0 -16
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/LICENSE +0 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/WHEEL +0 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/entry_points.txt +0 -0
- {inspect_ai-0.3.58.dist-info → inspect_ai-0.3.60.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,4 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
import { asyncJsonParse } from "../utils/Json.mjs";
|
4
|
-
// @ts-ignore
|
1
|
+
import { asyncJsonParse } from "../utils/json-worker";
|
5
2
|
import JSON5 from "json5";
|
6
3
|
|
7
4
|
import {
|
@@ -11,8 +8,9 @@ import {
|
|
11
8
|
kMethodEvalLogSize,
|
12
9
|
kMethodEvalLogBytes,
|
13
10
|
kMethodEvalLogHeaders,
|
14
|
-
} from "./jsonrpc
|
15
|
-
import { getVscodeApi } from "../utils/vscode
|
11
|
+
} from "./jsonrpc";
|
12
|
+
import { getVscodeApi } from "../utils/vscode";
|
13
|
+
import { Capabilities, LogContents, LogViewAPI } from "./Types";
|
16
14
|
|
17
15
|
const vscodeClient = webViewJsonRpcClient(getVscodeApi());
|
18
16
|
|
@@ -38,8 +36,12 @@ async function eval_logs() {
|
|
38
36
|
}
|
39
37
|
}
|
40
38
|
|
41
|
-
async function eval_log(
|
42
|
-
|
39
|
+
async function eval_log(
|
40
|
+
log_file: string,
|
41
|
+
headerOnly?: number,
|
42
|
+
capabilities?: Capabilities,
|
43
|
+
): Promise<LogContents> {
|
44
|
+
const response = await vscodeClient(kMethodEvalLog, [log_file, headerOnly]);
|
43
45
|
if (response) {
|
44
46
|
let json;
|
45
47
|
if (capabilities?.webWorkers) {
|
@@ -52,19 +54,19 @@ async function eval_log(file, headerOnly, capabilities) {
|
|
52
54
|
raw: response,
|
53
55
|
};
|
54
56
|
} else {
|
55
|
-
|
57
|
+
throw new Error(`Unable to load eval log ${log_file}.`);
|
56
58
|
}
|
57
59
|
}
|
58
60
|
|
59
|
-
async function eval_log_size(
|
60
|
-
return await vscodeClient(kMethodEvalLogSize, [
|
61
|
+
async function eval_log_size(log_file: string) {
|
62
|
+
return await vscodeClient(kMethodEvalLogSize, [log_file]);
|
61
63
|
}
|
62
64
|
|
63
|
-
async function eval_log_bytes(
|
64
|
-
return await vscodeClient(kMethodEvalLogBytes, [
|
65
|
+
async function eval_log_bytes(log_file: string, start: number, end: number) {
|
66
|
+
return await vscodeClient(kMethodEvalLogBytes, [log_file, start, end]);
|
65
67
|
}
|
66
68
|
|
67
|
-
async function eval_log_headers(files) {
|
69
|
+
async function eval_log_headers(files: string[]) {
|
68
70
|
const response = await vscodeClient(kMethodEvalLogHeaders, [files]);
|
69
71
|
if (response) {
|
70
72
|
return JSON5.parse(response);
|
@@ -77,17 +79,16 @@ async function download_file() {
|
|
77
79
|
throw Error("Downloading files is not supported in VS Code");
|
78
80
|
}
|
79
81
|
|
80
|
-
async function open_log_file(
|
82
|
+
async function open_log_file(log_file: string, log_dir: string) {
|
81
83
|
const msg = {
|
82
84
|
type: "displayLogFile",
|
83
|
-
url:
|
85
|
+
url: log_file,
|
84
86
|
log_dir: log_dir,
|
85
87
|
};
|
86
|
-
getVscodeApi()
|
88
|
+
getVscodeApi()?.postMessage(msg);
|
87
89
|
}
|
88
90
|
|
89
|
-
|
90
|
-
export default {
|
91
|
+
const api: LogViewAPI = {
|
91
92
|
client_events,
|
92
93
|
eval_logs,
|
93
94
|
eval_log,
|
@@ -97,3 +98,5 @@ export default {
|
|
97
98
|
download_file,
|
98
99
|
open_log_file,
|
99
100
|
};
|
101
|
+
|
102
|
+
export default api;
|
@@ -1,71 +1,81 @@
|
|
1
|
-
|
2
|
-
import {
|
1
|
+
import { openRemoteLogFile, RemoteLogFile } from "../log/remoteLogFile";
|
2
|
+
import { EvalLog, EvalSample } from "../types/log";
|
3
3
|
import { FileSizeLimitError } from "../utils/remoteZipFile.mjs";
|
4
|
+
import { encodePathParts } from "./api-shared";
|
5
|
+
import {
|
6
|
+
ClientAPI,
|
7
|
+
EvalSummary,
|
8
|
+
LogContents,
|
9
|
+
LogViewAPI,
|
10
|
+
LogFiles,
|
11
|
+
} from "./Types";
|
4
12
|
|
5
|
-
const isEvalFile = (file) => {
|
13
|
+
const isEvalFile = (file: string) => {
|
6
14
|
return file.endsWith(".eval");
|
7
15
|
};
|
8
16
|
|
9
17
|
/**
|
10
18
|
* Represents an error thrown when a file exceeds the maximum allowed size.
|
11
|
-
*
|
12
|
-
* @class
|
13
|
-
* @extends {Error}
|
14
19
|
*/
|
15
20
|
export class SampleSizeLimitedExceededError extends Error {
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
*/
|
23
|
-
constructor(id, epoch, maxBytes) {
|
21
|
+
readonly id: string | number;
|
22
|
+
readonly epoch: number;
|
23
|
+
readonly maxBytes: number;
|
24
|
+
readonly displayStack: boolean;
|
25
|
+
|
26
|
+
constructor(id: string | number, epoch: number, maxBytes: number) {
|
24
27
|
super(
|
25
28
|
`Sample ${id} in epoch ${epoch} exceeds the maximum supported size (${maxBytes / 1024 / 1024}MB) and cannot be loaded.`,
|
26
29
|
);
|
30
|
+
|
27
31
|
this.name = "SampleSizeLimitedExceededError";
|
28
32
|
this.id = id;
|
29
33
|
this.epoch = epoch;
|
30
34
|
this.maxBytes = maxBytes;
|
31
35
|
this.displayStack = false;
|
36
|
+
|
37
|
+
Object.setPrototypeOf(this, SampleSizeLimitedExceededError.prototype);
|
32
38
|
}
|
33
39
|
}
|
40
|
+
interface LoadedLogFile {
|
41
|
+
file?: string;
|
42
|
+
remoteLog?: RemoteLogFile;
|
43
|
+
}
|
34
44
|
|
35
45
|
/**
|
36
46
|
* This provides an API implementation that will serve a single
|
37
47
|
* file using an http parameter, designed to be deployed
|
38
48
|
* to a webserver without inspect or the ability to enumerate log
|
39
49
|
* files
|
40
|
-
*
|
41
|
-
* @param { import("./Types.mjs").LogViewAPI } api - The api to use when loading logs
|
42
|
-
* @returns { import("./Types.mjs").ClientAPI } A Client API for the viewer
|
43
50
|
*/
|
44
|
-
export const clientApi = (api) => {
|
45
|
-
let current_log = undefined;
|
46
|
-
let current_path = undefined;
|
51
|
+
export const clientApi = (api: LogViewAPI, log_file?: string): ClientAPI => {
|
52
|
+
let current_log: LogContents | undefined = undefined;
|
53
|
+
let current_path: string | undefined = undefined;
|
47
54
|
|
48
|
-
const loadedEvalFile = {
|
55
|
+
const loadedEvalFile: LoadedLogFile = {
|
49
56
|
file: undefined,
|
50
57
|
remoteLog: undefined,
|
51
58
|
};
|
52
59
|
|
53
|
-
const remoteEvalFile = async (log_file, cached = false) => {
|
60
|
+
const remoteEvalFile = async (log_file: string, cached: boolean = false) => {
|
54
61
|
if (!cached || loadedEvalFile.file !== log_file) {
|
55
62
|
loadedEvalFile.file = log_file;
|
56
|
-
loadedEvalFile.remoteLog = await openRemoteLogFile(
|
63
|
+
loadedEvalFile.remoteLog = await openRemoteLogFile(
|
64
|
+
api,
|
65
|
+
encodePathParts(log_file),
|
66
|
+
5,
|
67
|
+
);
|
57
68
|
}
|
58
69
|
return loadedEvalFile.remoteLog;
|
59
70
|
};
|
60
71
|
|
61
72
|
/**
|
62
73
|
* Gets a log
|
63
|
-
*
|
64
|
-
* @param { string } log_file - The api to use when loading logs
|
65
|
-
* @param { boolean } cached - allow this request to use a cached log file
|
66
|
-
* @returns { Promise<import("./Types.mjs").LogContents> } A Log Viewer API
|
67
74
|
*/
|
68
|
-
const get_log = async (
|
75
|
+
const get_log = async (
|
76
|
+
log_file: string,
|
77
|
+
cached = false,
|
78
|
+
): Promise<LogContents> => {
|
69
79
|
// If the requested log is different or no cached log exists, start fetching
|
70
80
|
if (!cached || log_file !== current_path || !current_log) {
|
71
81
|
// If there's already a pending fetch, return the same promise
|
@@ -91,22 +101,23 @@ export const clientApi = (api) => {
|
|
91
101
|
}
|
92
102
|
return current_log;
|
93
103
|
};
|
94
|
-
let pending_log_promise = null;
|
104
|
+
let pending_log_promise: Promise<LogContents> | null = null;
|
95
105
|
|
96
106
|
/**
|
97
107
|
* Gets a log summary
|
98
|
-
*
|
99
|
-
* @param { string } log_file - The api to use when loading logs
|
100
|
-
* @returns { Promise<import("./Types.mjs").EvalSummary> } A Log Viewer API
|
101
108
|
*/
|
102
|
-
const get_log_summary = async (log_file) => {
|
109
|
+
const get_log_summary = async (log_file: string): Promise<EvalSummary> => {
|
103
110
|
if (isEvalFile(log_file)) {
|
104
111
|
const remoteLogFile = await remoteEvalFile(log_file);
|
105
|
-
|
112
|
+
if (remoteLogFile) {
|
113
|
+
return await remoteLogFile.readLogSummary();
|
114
|
+
} else {
|
115
|
+
throw new Error("Unable to read remote eval file");
|
116
|
+
}
|
106
117
|
} else {
|
107
118
|
const logContents = await get_log(log_file);
|
108
119
|
/**
|
109
|
-
* @type {import("./Types.
|
120
|
+
* @type {import("./Types.js").SampleSummary[]}
|
110
121
|
*/
|
111
122
|
const sampleSummaries = logContents.parsed.samples
|
112
123
|
? logContents.parsed.samples?.map((sample) => {
|
@@ -138,18 +149,21 @@ export const clientApi = (api) => {
|
|
138
149
|
|
139
150
|
/**
|
140
151
|
* Gets a sample
|
141
|
-
*
|
142
|
-
* @param { string } log_file - The api to use when loading logs
|
143
|
-
* @param { string | number } id - The api to use when loading logs
|
144
|
-
* @param { number } epoch - The api to use when loading logs
|
145
|
-
* @returns { Promise<import("../types/log").EvalSample | undefined> } The sample
|
146
152
|
*/
|
147
|
-
const get_log_sample = async (
|
153
|
+
const get_log_sample = async (
|
154
|
+
log_file: string,
|
155
|
+
id: string | number,
|
156
|
+
epoch: number,
|
157
|
+
): Promise<EvalSample | undefined> => {
|
148
158
|
if (isEvalFile(log_file)) {
|
149
159
|
const remoteLogFile = await remoteEvalFile(log_file, true);
|
150
160
|
try {
|
151
|
-
|
152
|
-
|
161
|
+
if (remoteLogFile) {
|
162
|
+
const sample = await remoteLogFile.readSample(String(id), epoch);
|
163
|
+
return sample;
|
164
|
+
} else {
|
165
|
+
throw new Error(`Unable to read remove eval file ${log_file}`);
|
166
|
+
}
|
153
167
|
} catch (error) {
|
154
168
|
if (error instanceof FileSizeLimitError) {
|
155
169
|
throw new SampleSizeLimitedExceededError(id, epoch, error.maxBytes);
|
@@ -168,21 +182,22 @@ export const clientApi = (api) => {
|
|
168
182
|
return undefined;
|
169
183
|
};
|
170
184
|
|
171
|
-
const get_eval_log_header = async (log_file) => {
|
185
|
+
const get_eval_log_header = async (log_file: string) => {
|
172
186
|
// Don't re-use the eval log file since we know these are all different log files
|
173
|
-
const remoteLogFile = await openRemoteLogFile(
|
187
|
+
const remoteLogFile = await openRemoteLogFile(
|
188
|
+
api,
|
189
|
+
encodePathParts(log_file),
|
190
|
+
5,
|
191
|
+
);
|
174
192
|
return remoteLogFile.readHeader();
|
175
193
|
};
|
176
194
|
|
177
195
|
/**
|
178
196
|
* Gets log headers
|
179
|
-
*
|
180
|
-
* @param { string[] } log_files - The api to use when loading logs
|
181
|
-
* @returns { Promise<import("../types/log").EvalLog[]> } The sample
|
182
197
|
*/
|
183
|
-
const get_log_headers = async (log_files) => {
|
184
|
-
const eval_files = {};
|
185
|
-
const json_files = {};
|
198
|
+
const get_log_headers = async (log_files: string[]): Promise<EvalLog[]> => {
|
199
|
+
const eval_files: Record<string, number> = {};
|
200
|
+
const json_files: Record<string, number> = {};
|
186
201
|
let index = 0;
|
187
202
|
|
188
203
|
// Separate files into eval_files and json_files
|
@@ -226,12 +241,34 @@ export const clientApi = (api) => {
|
|
226
241
|
return orderedHeaders.map(({ header }) => header);
|
227
242
|
};
|
228
243
|
|
244
|
+
const get_log_paths = async (): Promise<LogFiles> => {
|
245
|
+
const logFiles = await api.eval_logs();
|
246
|
+
if (logFiles) {
|
247
|
+
return logFiles!;
|
248
|
+
} else if (log_file) {
|
249
|
+
// Is there an explicitly passed log file?
|
250
|
+
const summary = await get_log_summary(log_file);
|
251
|
+
if (summary) {
|
252
|
+
return {
|
253
|
+
files: [
|
254
|
+
{
|
255
|
+
name: log_file,
|
256
|
+
task: summary.eval.task,
|
257
|
+
task_id: summary.eval.task_id,
|
258
|
+
},
|
259
|
+
],
|
260
|
+
};
|
261
|
+
}
|
262
|
+
}
|
263
|
+
throw new Error("Unable to determine log paths.");
|
264
|
+
};
|
265
|
+
|
229
266
|
return {
|
230
267
|
client_events: () => {
|
231
268
|
return api.client_events();
|
232
269
|
},
|
233
270
|
get_log_paths: () => {
|
234
|
-
return
|
271
|
+
return get_log_paths();
|
235
272
|
},
|
236
273
|
get_log_headers: (log_files) => {
|
237
274
|
return get_log_headers(log_files);
|
@@ -241,7 +278,10 @@ export const clientApi = (api) => {
|
|
241
278
|
open_log_file: (log_file, log_dir) => {
|
242
279
|
return api.open_log_file(log_file, log_dir);
|
243
280
|
},
|
244
|
-
download_file: (
|
281
|
+
download_file: (
|
282
|
+
download_file: string,
|
283
|
+
file_contents: string | Blob | ArrayBuffer | ArrayBufferView,
|
284
|
+
) => {
|
245
285
|
return api.download_file(download_file, file_contents);
|
246
286
|
},
|
247
287
|
};
|
@@ -0,0 +1,51 @@
|
|
1
|
+
import browserApi from "./api-browser";
|
2
|
+
import vscodeApi from "./api-vscode";
|
3
|
+
import simpleHttpApi from "./api-http";
|
4
|
+
import { dirname } from "../utils/Path.mjs";
|
5
|
+
import { getVscodeApi } from "../utils/vscode";
|
6
|
+
import { clientApi } from "./client-api";
|
7
|
+
import { ClientAPI } from "./Types";
|
8
|
+
|
9
|
+
//
|
10
|
+
/**
|
11
|
+
* Resolves the client API
|
12
|
+
*/
|
13
|
+
const resolveApi = (): ClientAPI => {
|
14
|
+
if (getVscodeApi()) {
|
15
|
+
// This is VSCode
|
16
|
+
return clientApi(vscodeApi);
|
17
|
+
} else {
|
18
|
+
// See if there is an log_file, log_dir embedded in the
|
19
|
+
// document or passed via URL (could be hosted)
|
20
|
+
const scriptEl = document.getElementById("log_dir_context");
|
21
|
+
if (scriptEl) {
|
22
|
+
// Read the contents
|
23
|
+
const context = scriptEl.textContent;
|
24
|
+
if (context !== null) {
|
25
|
+
const data = JSON.parse(context);
|
26
|
+
if (data.log_dir || data.log_file) {
|
27
|
+
const log_dir = data.log_dir || dirname(data.log_file);
|
28
|
+
const api = simpleHttpApi(log_dir, data.log_file);
|
29
|
+
return clientApi(api, data.log_file);
|
30
|
+
}
|
31
|
+
}
|
32
|
+
}
|
33
|
+
|
34
|
+
// See if there is url params passing info (could be hosted)
|
35
|
+
const urlParams = new URLSearchParams(window.location.search);
|
36
|
+
const log_file = urlParams.get("log_file");
|
37
|
+
const log_dir = urlParams.get("log_dir");
|
38
|
+
if (log_file !== null || log_dir !== null) {
|
39
|
+
const resolved_log_dir = log_dir === null ? undefined : log_dir;
|
40
|
+
const resolved_log_file = log_file === null ? undefined : log_file;
|
41
|
+
const api = simpleHttpApi(resolved_log_dir, resolved_log_file);
|
42
|
+
return clientApi(api, resolved_log_file);
|
43
|
+
}
|
44
|
+
|
45
|
+
// No signal information so use the standard
|
46
|
+
// browser API (inspect view)
|
47
|
+
return clientApi(browserApi);
|
48
|
+
}
|
49
|
+
};
|
50
|
+
|
51
|
+
export default resolveApi();
|
@@ -0,0 +1,225 @@
|
|
1
|
+
// Type definitions
|
2
|
+
interface JsonRpcMessage {
|
3
|
+
jsonrpc: string;
|
4
|
+
id: number;
|
5
|
+
}
|
6
|
+
|
7
|
+
interface JsonRpcRequest extends JsonRpcMessage {
|
8
|
+
method: string;
|
9
|
+
params?: any;
|
10
|
+
}
|
11
|
+
|
12
|
+
interface JsonRpcResponse extends JsonRpcMessage {
|
13
|
+
result?: any;
|
14
|
+
error?: JsonRpcError;
|
15
|
+
}
|
16
|
+
|
17
|
+
interface JsonRpcError {
|
18
|
+
code: number;
|
19
|
+
message: string;
|
20
|
+
data?: {
|
21
|
+
description?: string;
|
22
|
+
[key: string]: any;
|
23
|
+
};
|
24
|
+
}
|
25
|
+
|
26
|
+
interface RequestHandlers {
|
27
|
+
resolve: (value: any) => void;
|
28
|
+
reject: (error: JsonRpcError) => void;
|
29
|
+
}
|
30
|
+
|
31
|
+
interface PostMessageTarget {
|
32
|
+
postMessage: (data: any) => void;
|
33
|
+
onMessage: (handler: (data: any) => void) => () => void;
|
34
|
+
}
|
35
|
+
|
36
|
+
// Constants
|
37
|
+
export const kMethodEvalLogs = "eval_logs";
|
38
|
+
export const kMethodEvalLog = "eval_log";
|
39
|
+
export const kMethodEvalLogSize = "eval_log_size";
|
40
|
+
export const kMethodEvalLogBytes = "eval_log_bytes";
|
41
|
+
export const kMethodEvalLogHeaders = "eval_log_headers";
|
42
|
+
|
43
|
+
export const kJsonRpcParseError = -32700;
|
44
|
+
export const kJsonRpcInvalidRequest = -32600;
|
45
|
+
export const kJsonRpcMethodNotFound = -32601;
|
46
|
+
export const kJsonRpcInvalidParams = -32602;
|
47
|
+
export const kJsonRpcInternalError = -32603;
|
48
|
+
export const kJsonRpcVersion = "2.0";
|
49
|
+
|
50
|
+
export function webViewJsonRpcClient(
|
51
|
+
vscode: any,
|
52
|
+
): (method: string, params?: any) => Promise<any> {
|
53
|
+
const target: PostMessageTarget = {
|
54
|
+
postMessage: (data: any) => {
|
55
|
+
vscode.postMessage(data);
|
56
|
+
},
|
57
|
+
onMessage: (handler: (data: any) => void) => {
|
58
|
+
const onMessage = (ev: MessageEvent) => {
|
59
|
+
handler(ev.data);
|
60
|
+
};
|
61
|
+
window.addEventListener("message", onMessage);
|
62
|
+
return () => {
|
63
|
+
window.removeEventListener("message", onMessage);
|
64
|
+
};
|
65
|
+
},
|
66
|
+
};
|
67
|
+
return jsonRpcPostMessageRequestTransport(target).request;
|
68
|
+
}
|
69
|
+
|
70
|
+
export function jsonRpcError(
|
71
|
+
message: string,
|
72
|
+
data?: any,
|
73
|
+
code?: number,
|
74
|
+
): JsonRpcError {
|
75
|
+
if (typeof data === "string") {
|
76
|
+
data = {
|
77
|
+
description: data,
|
78
|
+
};
|
79
|
+
}
|
80
|
+
return {
|
81
|
+
code: code || -3200,
|
82
|
+
message,
|
83
|
+
data,
|
84
|
+
};
|
85
|
+
}
|
86
|
+
|
87
|
+
export function asJsonRpcError(error: unknown): JsonRpcError {
|
88
|
+
if (typeof error === "object" && error !== null) {
|
89
|
+
const err = error as { message?: string; data?: any; code?: number };
|
90
|
+
if (typeof err.message === "string") {
|
91
|
+
return jsonRpcError(err.message, err.data, err.code);
|
92
|
+
}
|
93
|
+
}
|
94
|
+
return jsonRpcError(String(error));
|
95
|
+
}
|
96
|
+
|
97
|
+
export function jsonRpcPostMessageRequestTransport(target: PostMessageTarget) {
|
98
|
+
const requests = new Map<number, RequestHandlers>();
|
99
|
+
const disconnect = target.onMessage((ev: any) => {
|
100
|
+
const response = asJsonRpcResponse(ev);
|
101
|
+
if (response) {
|
102
|
+
const request = requests.get(response.id);
|
103
|
+
if (request) {
|
104
|
+
requests.delete(response.id);
|
105
|
+
if (response.error) {
|
106
|
+
request.reject(response.error);
|
107
|
+
} else {
|
108
|
+
request.resolve(response.result);
|
109
|
+
}
|
110
|
+
}
|
111
|
+
}
|
112
|
+
});
|
113
|
+
|
114
|
+
return {
|
115
|
+
request: (method: string, params?: any): Promise<any> => {
|
116
|
+
return new Promise((resolve, reject) => {
|
117
|
+
const requestId = Math.floor(Math.random() * 1e6);
|
118
|
+
requests.set(requestId, { resolve, reject });
|
119
|
+
const request: JsonRpcRequest = {
|
120
|
+
jsonrpc: kJsonRpcVersion,
|
121
|
+
id: requestId,
|
122
|
+
method,
|
123
|
+
params,
|
124
|
+
};
|
125
|
+
target.postMessage(request);
|
126
|
+
});
|
127
|
+
},
|
128
|
+
disconnect,
|
129
|
+
};
|
130
|
+
}
|
131
|
+
|
132
|
+
export function jsonRpcPostMessageServer(
|
133
|
+
target: PostMessageTarget,
|
134
|
+
methods:
|
135
|
+
| { [key: string]: (params: any) => Promise<any> }
|
136
|
+
| ((name: string) => ((params: any) => Promise<any>) | undefined),
|
137
|
+
): () => void {
|
138
|
+
const lookupMethod =
|
139
|
+
typeof methods === "function" ? methods : (name: string) => methods[name];
|
140
|
+
|
141
|
+
return target.onMessage((data: any) => {
|
142
|
+
const request = asJsonRpcRequest(data);
|
143
|
+
if (request) {
|
144
|
+
const method = lookupMethod(request.method);
|
145
|
+
if (!method) {
|
146
|
+
target.postMessage(methodNotFoundResponse(request));
|
147
|
+
return;
|
148
|
+
}
|
149
|
+
|
150
|
+
method(request.params || [])
|
151
|
+
.then((value) => {
|
152
|
+
target.postMessage(jsonRpcResponse(request, value));
|
153
|
+
})
|
154
|
+
.catch((error) => {
|
155
|
+
target.postMessage({
|
156
|
+
jsonrpc: request.jsonrpc,
|
157
|
+
id: request.id,
|
158
|
+
error: asJsonRpcError(error),
|
159
|
+
});
|
160
|
+
});
|
161
|
+
}
|
162
|
+
});
|
163
|
+
}
|
164
|
+
|
165
|
+
function isJsonRpcMessage(message: any): message is JsonRpcMessage {
|
166
|
+
return message.jsonrpc !== undefined && message.id !== undefined;
|
167
|
+
}
|
168
|
+
|
169
|
+
function isJsonRpcRequest(message: JsonRpcMessage): message is JsonRpcRequest {
|
170
|
+
return (message as JsonRpcRequest).method !== undefined;
|
171
|
+
}
|
172
|
+
|
173
|
+
function asJsonRpcMessage(data: any): JsonRpcMessage | null {
|
174
|
+
if (isJsonRpcMessage(data) && data.jsonrpc === kJsonRpcVersion) {
|
175
|
+
return data;
|
176
|
+
}
|
177
|
+
return null;
|
178
|
+
}
|
179
|
+
|
180
|
+
function asJsonRpcRequest(data: any): JsonRpcRequest | null {
|
181
|
+
const message = asJsonRpcMessage(data);
|
182
|
+
if (message && isJsonRpcRequest(message)) {
|
183
|
+
return message;
|
184
|
+
}
|
185
|
+
return null;
|
186
|
+
}
|
187
|
+
|
188
|
+
function asJsonRpcResponse(data: any): JsonRpcResponse | null {
|
189
|
+
const message = asJsonRpcMessage(data);
|
190
|
+
if (message) {
|
191
|
+
return message as JsonRpcResponse;
|
192
|
+
}
|
193
|
+
return null;
|
194
|
+
}
|
195
|
+
|
196
|
+
function jsonRpcResponse(
|
197
|
+
request: JsonRpcRequest,
|
198
|
+
result: any,
|
199
|
+
): JsonRpcResponse {
|
200
|
+
return {
|
201
|
+
jsonrpc: request.jsonrpc,
|
202
|
+
id: request.id,
|
203
|
+
result,
|
204
|
+
};
|
205
|
+
}
|
206
|
+
|
207
|
+
function jsonRpcErrorResponse(
|
208
|
+
request: JsonRpcRequest,
|
209
|
+
code: number,
|
210
|
+
message: string,
|
211
|
+
): JsonRpcResponse {
|
212
|
+
return {
|
213
|
+
jsonrpc: request.jsonrpc,
|
214
|
+
id: request.id,
|
215
|
+
error: jsonRpcError(message, undefined, code),
|
216
|
+
};
|
217
|
+
}
|
218
|
+
|
219
|
+
function methodNotFoundResponse(request: JsonRpcRequest): JsonRpcResponse {
|
220
|
+
return jsonRpcErrorResponse(
|
221
|
+
request,
|
222
|
+
kJsonRpcMethodNotFound,
|
223
|
+
`Method '${request.method}' not found.`,
|
224
|
+
);
|
225
|
+
}
|