trajectories-sh 1.21.0 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-35B24E3O.js +154 -0
- package/dist/{chunk-PF2EMA6Q.js → chunk-IUSBK45Q.js} +16 -1
- package/dist/cli.js +8 -8
- package/dist/{comments-RDCVHCZ7.js → comments-PK7DDP36.js} +2 -1
- package/dist/{download-4L7DZO3X.js → download-M6UVD5S5.js} +2 -1
- package/dist/{upload-4QD3JFJE.js → upload-CIW2CHKX.js} +29 -117
- package/package.json +1 -1
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// src/screenshot-chunks.ts
|
|
2
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "fs";
|
|
3
|
+
import { dirname, extname, join } from "path";
|
|
4
|
+
var CHUNK_DIR = "_screenshot_chunks";
|
|
5
|
+
var MANIFEST_NAME = "manifest.json";
|
|
6
|
+
var MANIFEST_VERSION = 1;
|
|
7
|
+
var CHUNK_TARGET_BYTES = 4 * 1024 * 1024;
|
|
8
|
+
var IMAGE_MIME_BY_EXT = {
|
|
9
|
+
".webp": "image/webp",
|
|
10
|
+
".png": "image/png",
|
|
11
|
+
".jpg": "image/jpeg",
|
|
12
|
+
".jpeg": "image/jpeg",
|
|
13
|
+
".gif": "image/gif"
|
|
14
|
+
};
|
|
15
|
+
function mimeFor(path) {
|
|
16
|
+
return IMAGE_MIME_BY_EXT[extname(path).toLowerCase()] ?? null;
|
|
17
|
+
}
|
|
18
|
+
function referencedImagePaths(trajectory) {
|
|
19
|
+
const out = [];
|
|
20
|
+
const seen = /* @__PURE__ */ new Set();
|
|
21
|
+
const collect = (parts) => {
|
|
22
|
+
if (!Array.isArray(parts)) return;
|
|
23
|
+
for (const part of parts) {
|
|
24
|
+
if (!part || typeof part !== "object" || part.type !== "image") continue;
|
|
25
|
+
const path = part.source?.path;
|
|
26
|
+
if (typeof path === "string" && path && !seen.has(path)) {
|
|
27
|
+
seen.add(path);
|
|
28
|
+
out.push(path);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
for (const step of Array.isArray(trajectory?.steps) ? trajectory.steps : []) {
|
|
33
|
+
if (!step || typeof step !== "object") continue;
|
|
34
|
+
collect(step.message);
|
|
35
|
+
for (const r of Array.isArray(step.observation?.results) ? step.observation.results : []) {
|
|
36
|
+
if (r && typeof r === "object") collect(r.content);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
function buildScreenshotChunks(jobDir, outDir, opts = {}) {
|
|
42
|
+
const targetBytes = opts.targetBytes ?? CHUNK_TARGET_BYTES;
|
|
43
|
+
const files = [];
|
|
44
|
+
const packedRelPaths = /* @__PURE__ */ new Set();
|
|
45
|
+
let packedBytes = 0;
|
|
46
|
+
let imagesPacked = 0;
|
|
47
|
+
let trialsPacked = 0;
|
|
48
|
+
for (const entry of readdirSync(jobDir, { withFileTypes: true })) {
|
|
49
|
+
if (!entry.isDirectory() || entry.name.startsWith(".")) continue;
|
|
50
|
+
const trial = entry.name;
|
|
51
|
+
const trajPath = join(jobDir, trial, "agent", "trajectory.json");
|
|
52
|
+
if (!existsSync(trajPath)) continue;
|
|
53
|
+
let paths;
|
|
54
|
+
try {
|
|
55
|
+
paths = referencedImagePaths(JSON.parse(readFileSync(trajPath, "utf-8")));
|
|
56
|
+
} catch {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const images = paths.map((p) => ({ p, abs: join(jobDir, trial, "agent", p), mime: mimeFor(p) })).filter((x) => x.mime && existsSync(x.abs));
|
|
60
|
+
if (images.length === 0) continue;
|
|
61
|
+
const relDir = `${trial}/${CHUNK_DIR}/agent`;
|
|
62
|
+
const absDir = join(outDir, relDir);
|
|
63
|
+
mkdirSync(absDir, { recursive: true });
|
|
64
|
+
const manifest = { version: MANIFEST_VERSION, chunks: [] };
|
|
65
|
+
let current = null;
|
|
66
|
+
const seal = () => {
|
|
67
|
+
if (!current) return;
|
|
68
|
+
const abs = join(absDir, current.name);
|
|
69
|
+
writeFileSync(abs, Buffer.concat(current.parts));
|
|
70
|
+
manifest.chunks.push({ name: current.name, size: current.size, entries: current.entries });
|
|
71
|
+
files.push({ absPath: abs, relPath: `${relDir}/${current.name}`, size: current.size });
|
|
72
|
+
current = null;
|
|
73
|
+
};
|
|
74
|
+
for (const { p, abs, mime } of images) {
|
|
75
|
+
const data = readFileSync(abs);
|
|
76
|
+
if (current && current.size + data.length > targetBytes) seal();
|
|
77
|
+
if (!current) {
|
|
78
|
+
current = { name: `chunk-${String(manifest.chunks.length).padStart(3, "0")}.bin`, parts: [], entries: [], size: 0 };
|
|
79
|
+
}
|
|
80
|
+
current.entries.push([p, current.size, data.length, mime]);
|
|
81
|
+
current.parts.push(data);
|
|
82
|
+
current.size += data.length;
|
|
83
|
+
packedRelPaths.add(`${trial}/agent/${p}`);
|
|
84
|
+
packedBytes += data.length;
|
|
85
|
+
imagesPacked++;
|
|
86
|
+
}
|
|
87
|
+
seal();
|
|
88
|
+
const manifestAbs = join(absDir, MANIFEST_NAME);
|
|
89
|
+
const manifestBytes = Buffer.from(JSON.stringify(manifest));
|
|
90
|
+
writeFileSync(manifestAbs, manifestBytes);
|
|
91
|
+
files.push({ absPath: manifestAbs, relPath: `${relDir}/${MANIFEST_NAME}`, size: manifestBytes.length });
|
|
92
|
+
trialsPacked++;
|
|
93
|
+
}
|
|
94
|
+
return { files, packedRelPaths, packedBytes, imagesPacked, trialsPacked };
|
|
95
|
+
}
|
|
96
|
+
function unpackScreenshotChunks(jobDir) {
|
|
97
|
+
let imagesWritten = 0;
|
|
98
|
+
let trialsUnpacked = 0;
|
|
99
|
+
if (!existsSync(jobDir)) return { imagesWritten, trialsUnpacked };
|
|
100
|
+
for (const entry of readdirSync(jobDir, { withFileTypes: true })) {
|
|
101
|
+
if (!entry.isDirectory()) continue;
|
|
102
|
+
const chunkRoot = join(jobDir, entry.name, CHUNK_DIR);
|
|
103
|
+
if (!existsSync(chunkRoot)) continue;
|
|
104
|
+
const agentDir = join(chunkRoot, "agent");
|
|
105
|
+
const manifestPath = join(agentDir, MANIFEST_NAME);
|
|
106
|
+
let clean = true;
|
|
107
|
+
if (existsSync(manifestPath)) {
|
|
108
|
+
let manifest = null;
|
|
109
|
+
try {
|
|
110
|
+
manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
|
|
111
|
+
} catch {
|
|
112
|
+
manifest = null;
|
|
113
|
+
}
|
|
114
|
+
if (manifest?.version === MANIFEST_VERSION && Array.isArray(manifest.chunks)) {
|
|
115
|
+
let wrote = false;
|
|
116
|
+
for (const chunk of manifest.chunks) {
|
|
117
|
+
const chunkPath = join(agentDir, chunk.name);
|
|
118
|
+
if (!existsSync(chunkPath)) {
|
|
119
|
+
console.warn(` \u26A0 ${entry.name}: chunk ${chunk.name} missing; its screenshots were not restored`);
|
|
120
|
+
clean = false;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const data = readFileSync(chunkPath);
|
|
124
|
+
for (const [p, offset, length] of chunk.entries) {
|
|
125
|
+
if (p.includes("..") || p.startsWith("/")) continue;
|
|
126
|
+
const dest = join(jobDir, entry.name, "agent", p);
|
|
127
|
+
if (existsSync(dest)) continue;
|
|
128
|
+
if (offset + length > data.length) {
|
|
129
|
+
console.warn(` \u26A0 ${entry.name}: ${p} extends past the end of ${chunk.name}; skipped`);
|
|
130
|
+
clean = false;
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
134
|
+
writeFileSync(dest, data.subarray(offset, offset + length));
|
|
135
|
+
imagesWritten++;
|
|
136
|
+
wrote = true;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (wrote) trialsUnpacked++;
|
|
140
|
+
} else {
|
|
141
|
+
console.warn(` \u26A0 ${entry.name}: unreadable screenshot chunk manifest; screenshots were not restored`);
|
|
142
|
+
clean = false;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (clean) rmSync(chunkRoot, { recursive: true, force: true });
|
|
146
|
+
}
|
|
147
|
+
return { imagesWritten, trialsUnpacked };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export {
|
|
151
|
+
CHUNK_DIR,
|
|
152
|
+
buildScreenshotChunks,
|
|
153
|
+
unpackScreenshotChunks
|
|
154
|
+
};
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CHUNK_DIR,
|
|
3
|
+
unpackScreenshotChunks
|
|
4
|
+
} from "./chunk-35B24E3O.js";
|
|
1
5
|
import {
|
|
2
6
|
getApiUrl,
|
|
3
7
|
getAuthHeader
|
|
@@ -57,8 +61,13 @@ async function downloadJob(input, outputDir) {
|
|
|
57
61
|
}
|
|
58
62
|
const { slug, files } = await listResp.json();
|
|
59
63
|
const totalBytes = files.reduce((s, f) => s + f.size, 0);
|
|
64
|
+
const packedTrials = new Set(
|
|
65
|
+
files.filter((f) => f.path.includes(`/${CHUNK_DIR}/`)).map((f) => f.path.split("/")[0])
|
|
66
|
+
);
|
|
60
67
|
console.log(` Job: ${slug} (${jobId})`);
|
|
61
|
-
console.log(
|
|
68
|
+
console.log(
|
|
69
|
+
` ${files.length} files, ${(totalBytes / 1024 / 1024).toFixed(1)} MB` + (packedTrials.size > 0 ? ` (screenshots of ${packedTrials.size} trial${packedTrials.size === 1 ? "" : "s"} packed)` : "")
|
|
70
|
+
);
|
|
62
71
|
mkdirSync(outputDir, { recursive: true });
|
|
63
72
|
const { ProgressBar } = await import("./progress-H7WNKHDR.js");
|
|
64
73
|
const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2193" });
|
|
@@ -123,6 +132,12 @@ async function downloadJob(input, outputDir) {
|
|
|
123
132
|
await Promise.all(pending);
|
|
124
133
|
}
|
|
125
134
|
bar.done();
|
|
135
|
+
if (packedTrials.size > 0) {
|
|
136
|
+
const unpacked = unpackScreenshotChunks(outputDir);
|
|
137
|
+
if (unpacked.imagesWritten > 0) {
|
|
138
|
+
console.log(` \u2713 Unpacked ${unpacked.imagesWritten} screenshots across ${unpacked.trialsUnpacked} trial${unpacked.trialsUnpacked === 1 ? "" : "s"}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
126
141
|
return { files: downloaded, totalSize };
|
|
127
142
|
}
|
|
128
143
|
|
package/dist/cli.js
CHANGED
|
@@ -98,7 +98,7 @@ async function uploadTrajectoryAction(pathArg, opts) {
|
|
|
98
98
|
);
|
|
99
99
|
}
|
|
100
100
|
if (isZip) {
|
|
101
|
-
const { uploadZip } = await import("./upload-
|
|
101
|
+
const { uploadZip } = await import("./upload-CIW2CHKX.js");
|
|
102
102
|
const slug2 = opts.slug ?? pathArg.replace(/\.zip$/i, "").replace(/\/+$/, "").split("/").pop();
|
|
103
103
|
console.log(chalk.bold(`
|
|
104
104
|
Uploading ZIP ${pathArg} \u2192 ${chalk.cyan(slug2)}
|
|
@@ -127,7 +127,7 @@ async function uploadTrajectoryAction(pathArg, opts) {
|
|
|
127
127
|
Uploading trajectory ${pathArg} \u2192 ${chalk.cyan(slug)}
|
|
128
128
|
`));
|
|
129
129
|
try {
|
|
130
|
-
const { uploadJob } = await import("./upload-
|
|
130
|
+
const { uploadJob } = await import("./upload-CIW2CHKX.js");
|
|
131
131
|
const result = await uploadJob(pathArg, {
|
|
132
132
|
slug,
|
|
133
133
|
name: opts.name
|
|
@@ -200,7 +200,7 @@ async function downloadTrajectoryAction(url, opts) {
|
|
|
200
200
|
Downloading trajectory...
|
|
201
201
|
`));
|
|
202
202
|
try {
|
|
203
|
-
const { downloadJob, parseJobId } = await import("./download-
|
|
203
|
+
const { downloadJob, parseJobId } = await import("./download-M6UVD5S5.js");
|
|
204
204
|
const outputDir = opts.output ?? parseJobId(url).replace(/^org:/, "").replace(/:/g, "-");
|
|
205
205
|
const result = await downloadJob(url, outputDir);
|
|
206
206
|
console.log(chalk.green.bold(` \u2713 Downloaded ${result.files} files`));
|
|
@@ -216,7 +216,7 @@ async function downloadTrajectoryAction(url, opts) {
|
|
|
216
216
|
}
|
|
217
217
|
async function deleteTrajectoryAction(url, opts) {
|
|
218
218
|
const chalk = (await import("chalk")).default;
|
|
219
|
-
const { parseJobId } = await import("./download-
|
|
219
|
+
const { parseJobId } = await import("./download-M6UVD5S5.js");
|
|
220
220
|
const { getApiUrl, getAuthHeader } = await import("./config-4XJRECIW.js");
|
|
221
221
|
let jobId = parseJobId(url);
|
|
222
222
|
if (jobId.startsWith("org:")) {
|
|
@@ -284,7 +284,7 @@ function printCommentError(error, json) {
|
|
|
284
284
|
}
|
|
285
285
|
async function addCommentAction(targetInput, opts) {
|
|
286
286
|
try {
|
|
287
|
-
const { addComment, formatComment, parseCommentTarget, readCommentBody } = await import("./comments-
|
|
287
|
+
const { addComment, formatComment, parseCommentTarget, readCommentBody } = await import("./comments-PK7DDP36.js");
|
|
288
288
|
const target = parseCommentTarget(targetInput, opts);
|
|
289
289
|
const body = await readCommentBody(opts);
|
|
290
290
|
const comment2 = await addComment(target, body);
|
|
@@ -300,7 +300,7 @@ ${formatComment(comment2)}
|
|
|
300
300
|
}
|
|
301
301
|
async function listCommentsAction(targetInput, opts) {
|
|
302
302
|
try {
|
|
303
|
-
const { formatComment, listComments, parseCommentTarget } = await import("./comments-
|
|
303
|
+
const { formatComment, listComments, parseCommentTarget } = await import("./comments-PK7DDP36.js");
|
|
304
304
|
const target = parseCommentTarget(targetInput, opts);
|
|
305
305
|
const comments = await listComments(target);
|
|
306
306
|
if (opts.json) {
|
|
@@ -318,7 +318,7 @@ ${comments.map(formatComment).join("\n\n")}
|
|
|
318
318
|
}
|
|
319
319
|
async function updateCommentAction(commentId, opts) {
|
|
320
320
|
try {
|
|
321
|
-
const { formatComment, readCommentBody, updateComment } = await import("./comments-
|
|
321
|
+
const { formatComment, readCommentBody, updateComment } = await import("./comments-PK7DDP36.js");
|
|
322
322
|
const body = await readCommentBody(opts);
|
|
323
323
|
const comment2 = await updateComment(commentId, body);
|
|
324
324
|
if (opts.json) console.log(JSON.stringify(comment2));
|
|
@@ -349,7 +349,7 @@ async function deleteCommentAction(commentId, opts) {
|
|
|
349
349
|
return;
|
|
350
350
|
}
|
|
351
351
|
}
|
|
352
|
-
const { deleteComment } = await import("./comments-
|
|
352
|
+
const { deleteComment } = await import("./comments-PK7DDP36.js");
|
|
353
353
|
await deleteComment(commentId);
|
|
354
354
|
const result = { status: "deleted", comment_id: commentId };
|
|
355
355
|
if (opts.json) console.log(JSON.stringify(result));
|
|
@@ -1,110 +1,19 @@
|
|
|
1
1
|
import {
|
|
2
2
|
preflightQuota
|
|
3
3
|
} from "./chunk-2ELJH3IV.js";
|
|
4
|
+
import {
|
|
5
|
+
buildScreenshotChunks
|
|
6
|
+
} from "./chunk-35B24E3O.js";
|
|
4
7
|
import {
|
|
5
8
|
getApiUrl,
|
|
6
9
|
getAuthHeader
|
|
7
10
|
} from "./chunk-Q2ZEN43L.js";
|
|
8
11
|
|
|
9
12
|
// src/upload.ts
|
|
10
|
-
import { readdirSync
|
|
11
|
-
import { join
|
|
13
|
+
import { readdirSync, statSync, readFileSync, writeFileSync, mkdirSync, copyFileSync, unlinkSync } from "fs";
|
|
14
|
+
import { join, relative, dirname } from "path";
|
|
12
15
|
import { tmpdir } from "os";
|
|
13
16
|
import { randomBytes, createHash } from "crypto";
|
|
14
|
-
|
|
15
|
-
// src/screenshot-chunks.ts
|
|
16
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "fs";
|
|
17
|
-
import { extname, join } from "path";
|
|
18
|
-
var CHUNK_DIR = "_screenshot_chunks";
|
|
19
|
-
var MANIFEST_NAME = "manifest.json";
|
|
20
|
-
var MANIFEST_VERSION = 1;
|
|
21
|
-
var CHUNK_TARGET_BYTES = 4 * 1024 * 1024;
|
|
22
|
-
var IMAGE_MIME_BY_EXT = {
|
|
23
|
-
".webp": "image/webp",
|
|
24
|
-
".png": "image/png",
|
|
25
|
-
".jpg": "image/jpeg",
|
|
26
|
-
".jpeg": "image/jpeg",
|
|
27
|
-
".gif": "image/gif"
|
|
28
|
-
};
|
|
29
|
-
function mimeFor(path) {
|
|
30
|
-
return IMAGE_MIME_BY_EXT[extname(path).toLowerCase()] ?? null;
|
|
31
|
-
}
|
|
32
|
-
function referencedImagePaths(trajectory) {
|
|
33
|
-
const out = [];
|
|
34
|
-
const seen = /* @__PURE__ */ new Set();
|
|
35
|
-
const collect = (parts) => {
|
|
36
|
-
if (!Array.isArray(parts)) return;
|
|
37
|
-
for (const part of parts) {
|
|
38
|
-
if (!part || typeof part !== "object" || part.type !== "image") continue;
|
|
39
|
-
const path = part.source?.path;
|
|
40
|
-
if (typeof path === "string" && path && !seen.has(path)) {
|
|
41
|
-
seen.add(path);
|
|
42
|
-
out.push(path);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
};
|
|
46
|
-
for (const step of Array.isArray(trajectory?.steps) ? trajectory.steps : []) {
|
|
47
|
-
if (!step || typeof step !== "object") continue;
|
|
48
|
-
collect(step.message);
|
|
49
|
-
for (const r of Array.isArray(step.observation?.results) ? step.observation.results : []) {
|
|
50
|
-
if (r && typeof r === "object") collect(r.content);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
return out;
|
|
54
|
-
}
|
|
55
|
-
function buildScreenshotChunks(jobDir, outDir, opts = {}) {
|
|
56
|
-
const targetBytes = opts.targetBytes ?? CHUNK_TARGET_BYTES;
|
|
57
|
-
const files = [];
|
|
58
|
-
let imagesPacked = 0;
|
|
59
|
-
let trialsPacked = 0;
|
|
60
|
-
for (const entry of readdirSync(jobDir, { withFileTypes: true })) {
|
|
61
|
-
if (!entry.isDirectory() || entry.name.startsWith(".")) continue;
|
|
62
|
-
const trial = entry.name;
|
|
63
|
-
const trajPath = join(jobDir, trial, "agent", "trajectory.json");
|
|
64
|
-
if (!existsSync(trajPath)) continue;
|
|
65
|
-
let paths;
|
|
66
|
-
try {
|
|
67
|
-
paths = referencedImagePaths(JSON.parse(readFileSync(trajPath, "utf-8")));
|
|
68
|
-
} catch {
|
|
69
|
-
continue;
|
|
70
|
-
}
|
|
71
|
-
const images = paths.map((p) => ({ p, abs: join(jobDir, trial, "agent", p), mime: mimeFor(p) })).filter((x) => x.mime && existsSync(x.abs));
|
|
72
|
-
if (images.length === 0) continue;
|
|
73
|
-
const relDir = `${trial}/${CHUNK_DIR}/agent`;
|
|
74
|
-
const absDir = join(outDir, relDir);
|
|
75
|
-
mkdirSync(absDir, { recursive: true });
|
|
76
|
-
const manifest = { version: MANIFEST_VERSION, chunks: [] };
|
|
77
|
-
let current = null;
|
|
78
|
-
const seal = () => {
|
|
79
|
-
if (!current) return;
|
|
80
|
-
const abs = join(absDir, current.name);
|
|
81
|
-
writeFileSync(abs, Buffer.concat(current.parts));
|
|
82
|
-
manifest.chunks.push({ name: current.name, size: current.size, entries: current.entries });
|
|
83
|
-
files.push({ absPath: abs, relPath: `${relDir}/${current.name}`, size: current.size });
|
|
84
|
-
current = null;
|
|
85
|
-
};
|
|
86
|
-
for (const { p, abs, mime } of images) {
|
|
87
|
-
const data = readFileSync(abs);
|
|
88
|
-
if (current && current.size + data.length > targetBytes) seal();
|
|
89
|
-
if (!current) {
|
|
90
|
-
current = { name: `chunk-${String(manifest.chunks.length).padStart(3, "0")}.bin`, parts: [], entries: [], size: 0 };
|
|
91
|
-
}
|
|
92
|
-
current.entries.push([p, current.size, data.length, mime]);
|
|
93
|
-
current.parts.push(data);
|
|
94
|
-
current.size += data.length;
|
|
95
|
-
imagesPacked++;
|
|
96
|
-
}
|
|
97
|
-
seal();
|
|
98
|
-
const manifestAbs = join(absDir, MANIFEST_NAME);
|
|
99
|
-
const manifestBytes = Buffer.from(JSON.stringify(manifest));
|
|
100
|
-
writeFileSync(manifestAbs, manifestBytes);
|
|
101
|
-
files.push({ absPath: manifestAbs, relPath: `${relDir}/${MANIFEST_NAME}`, size: manifestBytes.length });
|
|
102
|
-
trialsPacked++;
|
|
103
|
-
}
|
|
104
|
-
return { files, imagesPacked, trialsPacked };
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
// src/upload.ts
|
|
108
17
|
var SKIP = /* @__PURE__ */ new Set(["__pycache__", ".DS_Store", ".git", "node_modules"]);
|
|
109
18
|
var MAX_FILE_SIZE = 50 * 1024 * 1024;
|
|
110
19
|
var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "24");
|
|
@@ -115,9 +24,9 @@ function collectFiles(dir) {
|
|
|
115
24
|
const files = [];
|
|
116
25
|
const skipped = [];
|
|
117
26
|
function walk(current) {
|
|
118
|
-
for (const entry of
|
|
27
|
+
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
|
119
28
|
if (SKIP.has(entry.name)) continue;
|
|
120
|
-
const full =
|
|
29
|
+
const full = join(current, entry.name);
|
|
121
30
|
if (entry.isDirectory()) {
|
|
122
31
|
walk(full);
|
|
123
32
|
} else if (entry.isFile()) {
|
|
@@ -266,13 +175,13 @@ async function convertToWebP(dir) {
|
|
|
266
175
|
console.log(" \u26A0 sharp not available, skipping WebP conversion");
|
|
267
176
|
return { convertedDir: dir, pngsConverted: 0, savedBytes: 0 };
|
|
268
177
|
}
|
|
269
|
-
const tmpDir =
|
|
178
|
+
const tmpDir = join(tmpdir(), `trajectories-webp-${randomBytes(4).toString("hex")}`);
|
|
270
179
|
function copyDir(src, dest) {
|
|
271
|
-
|
|
272
|
-
for (const entry of
|
|
180
|
+
mkdirSync(dest, { recursive: true });
|
|
181
|
+
for (const entry of readdirSync(src, { withFileTypes: true })) {
|
|
273
182
|
if (SKIP.has(entry.name)) continue;
|
|
274
|
-
const srcPath =
|
|
275
|
-
const destPath =
|
|
183
|
+
const srcPath = join(src, entry.name);
|
|
184
|
+
const destPath = join(dest, entry.name);
|
|
276
185
|
if (entry.isDirectory()) {
|
|
277
186
|
copyDir(srcPath, destPath);
|
|
278
187
|
} else {
|
|
@@ -300,7 +209,7 @@ async function convertToWebP(dir) {
|
|
|
300
209
|
let processed = 0;
|
|
301
210
|
const total = pngs.length;
|
|
302
211
|
async function convertOne(f) {
|
|
303
|
-
const srcPath =
|
|
212
|
+
const srcPath = join(tmpDir, f.relPath);
|
|
304
213
|
const webpPath = srcPath.replace(/\.png$/i, ".webp");
|
|
305
214
|
try {
|
|
306
215
|
await pngToWebp(srcPath, webpPath);
|
|
@@ -332,10 +241,10 @@ async function convertToWebP(dir) {
|
|
|
332
241
|
const trajFiles = collectFiles(tmpDir).files.filter((f) => f.relPath.endsWith("trajectory.json"));
|
|
333
242
|
for (const tf of trajFiles) {
|
|
334
243
|
try {
|
|
335
|
-
let content =
|
|
244
|
+
let content = readFileSync(tf.absPath, "utf-8");
|
|
336
245
|
const original = content;
|
|
337
246
|
content = content.replace(/screenshots\/([^"]+)\.png/g, (match, name) => {
|
|
338
|
-
const webpPath =
|
|
247
|
+
const webpPath = join(dirname(tf.absPath), "screenshots", `${name}.webp`);
|
|
339
248
|
try {
|
|
340
249
|
statSync(webpPath);
|
|
341
250
|
return `screenshots/${name}.webp`;
|
|
@@ -344,7 +253,7 @@ async function convertToWebP(dir) {
|
|
|
344
253
|
}
|
|
345
254
|
});
|
|
346
255
|
if (content !== original) {
|
|
347
|
-
|
|
256
|
+
writeFileSync(tf.absPath, content);
|
|
348
257
|
}
|
|
349
258
|
} catch {
|
|
350
259
|
}
|
|
@@ -390,7 +299,7 @@ function validateAtifFormat(dir) {
|
|
|
390
299
|
errors.push("Missing result.json at the job root");
|
|
391
300
|
} else {
|
|
392
301
|
try {
|
|
393
|
-
const jobResult = JSON.parse(
|
|
302
|
+
const jobResult = JSON.parse(readFileSync(join(dir, "result.json"), "utf-8"));
|
|
394
303
|
const jobRequired = [
|
|
395
304
|
["n_total_trials", "n_total_trials (number of trials)"],
|
|
396
305
|
["started_at", "started_at (ISO timestamp)"],
|
|
@@ -426,9 +335,9 @@ function validateAtifFormat(dir) {
|
|
|
426
335
|
return errors;
|
|
427
336
|
}
|
|
428
337
|
for (const trial of trialDirs) {
|
|
429
|
-
const trialResultPath =
|
|
338
|
+
const trialResultPath = join(dir, trial, "result.json");
|
|
430
339
|
try {
|
|
431
|
-
const data = JSON.parse(
|
|
340
|
+
const data = JSON.parse(readFileSync(trialResultPath, "utf-8"));
|
|
432
341
|
const required = [
|
|
433
342
|
["task_name", "task_name (string)"],
|
|
434
343
|
["started_at", "started_at (ISO timestamp)"],
|
|
@@ -458,7 +367,7 @@ function validateAtifFormat(dir) {
|
|
|
458
367
|
errors.push(`Trial "${trial}": missing agent/trajectory.json`);
|
|
459
368
|
} else {
|
|
460
369
|
try {
|
|
461
|
-
const trajData = JSON.parse(
|
|
370
|
+
const trajData = JSON.parse(readFileSync(join(dir, trajectoryRelPath), "utf-8"));
|
|
462
371
|
const steps = trajData.steps ?? [];
|
|
463
372
|
const missingImages = [];
|
|
464
373
|
for (const step of steps) {
|
|
@@ -539,16 +448,19 @@ ${samples}${more}
|
|
|
539
448
|
const t0 = Date.now();
|
|
540
449
|
const contentHash = await computeContentHash(files);
|
|
541
450
|
console.log(` Hash: ${contentHash.slice(0, 16)}... (${((Date.now() - t0) / 1e3).toFixed(1)}s)`);
|
|
542
|
-
const chunkDir =
|
|
451
|
+
const chunkDir = join(tmpdir(), `trajectories-chunks-${randomBytes(4).toString("hex")}`);
|
|
543
452
|
const chunkBuild = buildScreenshotChunks(convertedDir, chunkDir);
|
|
544
|
-
const uploadFiles = [
|
|
545
|
-
|
|
453
|
+
const uploadFiles = [
|
|
454
|
+
...files.filter((f) => !chunkBuild.packedRelPaths.has(f.relPath.split("\\").join("/"))),
|
|
455
|
+
...chunkBuild.files
|
|
456
|
+
];
|
|
457
|
+
const uploadBytes = uploadFiles.reduce((s, f) => s + f.size, 0);
|
|
546
458
|
if (chunkBuild.imagesPacked > 0) {
|
|
547
459
|
console.log(
|
|
548
|
-
` \u2713 Packed ${chunkBuild.imagesPacked} screenshots into ${chunkBuild.files.length - chunkBuild.trialsPacked} chunks (${(
|
|
460
|
+
` \u2713 Packed ${chunkBuild.imagesPacked} screenshots into ${chunkBuild.files.length - chunkBuild.trialsPacked} chunks (${(chunkBuild.packedBytes / 1024 / 1024).toFixed(1)} MB) across ${chunkBuild.trialsPacked} trial${chunkBuild.trialsPacked === 1 ? "" : "s"} \u2014 ${uploadFiles.length} uploads`
|
|
549
461
|
);
|
|
550
462
|
}
|
|
551
|
-
const initResp = await apiPost("/api/cli/push/init", { slug, name, visibility, estimated_bytes:
|
|
463
|
+
const initResp = await apiPost("/api/cli/push/init", { slug, name, visibility, estimated_bytes: uploadBytes, content_hash: contentHash });
|
|
552
464
|
if (!initResp.ok) {
|
|
553
465
|
throw new Error(`Init failed: ${await initResp.text()}`);
|
|
554
466
|
}
|
|
@@ -709,7 +621,7 @@ async function fetchSignedUrlsForBatch(jobId, paths) {
|
|
|
709
621
|
return { urlMap, errors };
|
|
710
622
|
}
|
|
711
623
|
async function uploadOneFileWithError(file, signedUrl) {
|
|
712
|
-
const data =
|
|
624
|
+
const data = readFileSync(file.absPath);
|
|
713
625
|
let lastError = "unknown error";
|
|
714
626
|
const MAX_ATTEMPTS = 8;
|
|
715
627
|
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|