trajectories-sh 1.10.3 → 1.12.0

package/dist/cli.js

@@ -81,7 +81,7 @@ async function uploadTrajectoryAction(directory, opts) {
   Uploading trajectory ${directory} \u2192 ${chalk.cyan(slug)}
 `));
   try {
-    const { uploadJob } = await import("./upload-UXM6LEBL.js");
+    const { uploadJob } = await import("./upload-CDNIKYW3.js");
     const result = await uploadJob(directory, {
       slug,
       name: opts.name,
@@ -92,8 +92,11 @@ async function uploadTrajectoryAction(directory, opts) {
     if (result.meanReward != null) {
       console.log(`    Pass rate: ${(result.meanReward * 100).toFixed(0)}%`);
     }
-    console.log(`    Viewer: ${result.viewerUrl}
-`);
+    console.log(`    Viewer:        ${result.viewerUrl}`);
+    if (result.existingDuplicateUrl) {
+      console.log(`    Earlier copy:  ${result.existingDuplicateUrl}`);
+    }
+    console.log("");
   } catch (e) {
     console.error(chalk.red(`
   \u2717 ${e.message}

package/dist/{upload-UXM6LEBL.js → upload-CDNIKYW3.js}

@@ -10,8 +10,10 @@ import { tmpdir } from "os";
 import { randomBytes, createHash } from "crypto";
 var SKIP = /* @__PURE__ */ new Set(["__pycache__", ".DS_Store", ".git", "node_modules"]);
 var MAX_FILE_SIZE = 50 * 1024 * 1024;
-var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "15");
+var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "32");
 var BATCH_SIZE = 100;
+var WEBP_CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_WEBP_CONCURRENCY ?? "4");
+var HASH_CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_HASH_CONCURRENCY ?? "16");
 function collectFiles(dir) {
   const files = [];
   function walk(current) {
@@ -85,26 +87,30 @@ async function convertToWebP(dir) {
   sharp.cache({ memory: 0, files: 0, items: 0 });
   let converted = 0;
   let savedBytes = 0;
-  try {
-    for (let i = 0; i < pngs.length; i++) {
-      const f = pngs[i];
-      const srcPath = join(tmpDir, f.relPath);
-      const webpPath = srcPath.replace(/\.png$/i, ".webp");
-      try {
-        await pngToWebp(srcPath, webpPath);
-        const origSize = statSync(srcPath).size;
-        const newSize = statSync(webpPath).size;
-        if (newSize < origSize) {
-          unlinkSync(srcPath);
-          savedBytes += origSize - newSize;
-          converted++;
-        } else {
-          unlinkSync(webpPath);
-        }
-      } catch {
+  let processed = 0;
+  const total = pngs.length;
+  async function convertOne(f) {
+    const srcPath = join(tmpDir, f.relPath);
+    const webpPath = srcPath.replace(/\.png$/i, ".webp");
+    try {
+      await pngToWebp(srcPath, webpPath);
+      const origSize = statSync(srcPath).size;
+      const newSize = statSync(webpPath).size;
+      if (newSize < origSize) {
+        unlinkSync(srcPath);
+        savedBytes += origSize - newSize;
+        converted++;
+      } else {
+        unlinkSync(webpPath);
       }
-      process.stdout.write(`  Converting: ${i + 1}/${pngs.length} PNGs\r`);
+    } catch {
+    } finally {
+      processed++;
+      process.stdout.write(`  Converting: ${processed}/${total} PNGs\r`);
     }
+  }
+  try {
+    await runWithConcurrency(pngs, WEBP_CONCURRENCY, convertOne);
     console.log("");
   } finally {
     try {
@@ -135,12 +141,34 @@ async function convertToWebP(dir) {
   }
   return { convertedDir: tmpDir, pngsConverted: converted, savedBytes };
 }
-function computeContentHash(files) {
-  const hash = createHash("sha256");
+async function runWithConcurrency(items, n, worker) {
+  if (items.length === 0) return;
+  let next = 0;
+  const runners = [];
+  for (let k = 0; k < Math.min(n, items.length); k++) {
+    runners.push(
+      (async () => {
+        while (true) {
+          const i = next++;
+          if (i >= items.length) return;
+          await worker(items[i], i);
+        }
+      })()
+    );
+  }
+  await Promise.all(runners);
+}
+async function computeContentHash(files) {
+  const { readFile } = await import("fs/promises");
   const sorted = [...files].sort((a, b) => a.relPath.localeCompare(b.relPath));
-  for (const f of sorted) {
-    hash.update(f.relPath);
-    hash.update(readFileSync(f.absPath));
+  const contents = new Array(sorted.length);
+  await runWithConcurrency(sorted, HASH_CONCURRENCY, async (f, i) => {
+    contents[i] = await readFile(f.absPath);
+  });
+  const hash = createHash("sha256");
+  for (let i = 0; i < sorted.length; i++) {
+    hash.update(sorted[i].relPath);
+    hash.update(contents[i]);
   }
   return hash.digest("hex");
 }
@@ -280,9 +308,11 @@ async function uploadJob(dir, opts) {
   }
   const files = collectFiles(convertedDir);
   const totalSize = files.reduce((s, f) => s + f.size, 0);
-  const contentHash = computeContentHash(files);
   console.log(`  ${files.length} files, ${(totalSize / 1024 / 1024).toFixed(1)} MB`);
-  console.log(`  Hash: ${contentHash.slice(0, 16)}...`);
+  process.stdout.write(`  Hashing...\r`);
+  const t0 = Date.now();
+  const contentHash = await computeContentHash(files);
+  console.log(`  Hash: ${contentHash.slice(0, 16)}... (${((Date.now() - t0) / 1e3).toFixed(1)}s)`);
   try {
     const auth = await getAuthHeader();
     const usageResp = await fetch(`${getApiUrl()}/api/cli/usage`, {
@@ -315,14 +345,18 @@ async function uploadJob(dir, opts) {
   }
   const initResp = await apiPost("/api/cli/push/init", { slug, name, visibility, estimated_bytes: totalSize, content_hash: contentHash });
   if (!initResp.ok) {
-    if (initResp.status === 409) {
-      const err = await initResp.json().catch(() => ({ detail: "Duplicate trajectory" }));
-      throw new Error(err.detail);
-    }
     throw new Error(`Init failed: ${await initResp.text()}`);
   }
-  const { job_id: jobId } = await initResp.json();
+  const initBody = await initResp.json();
+  const jobId = initBody.job_id;
   console.log(`  Job ID: ${jobId}`);
+  const apiUrlEarly = getApiUrl();
+  const siteUrlEarly = apiUrlEarly.includes("localhost") ? "http://localhost:3000" : apiUrlEarly.replace("api.", "").replace(/\/$/, "");
+  const existingDuplicateUrl = initBody.existing_duplicate ? `${siteUrlEarly}/t/${initBody.existing_duplicate.job_id}` : void 0;
+  if (existingDuplicateUrl) {
+    console.log(`  \u24D8 This content was already uploaded as '${initBody.existing_duplicate.slug}' \u2014 uploading anyway.`);
+    console.log(`    Existing: ${existingDuplicateUrl}`);
+  }
   await uploadDirect(jobId, files);
   console.log("  Finalizing...");
   const finalResp = await apiPost(`/api/cli/push/${jobId}/finalize`, {});
@@ -351,7 +385,8 @@ async function uploadJob(dir, opts) {
     slug: result.slug,
     nTrials: result.n_trials,
     meanReward: result.mean_reward,
-    viewerUrl: `${siteUrl}/t/${jobId}`
+    viewerUrl: `${siteUrl}/t/${jobId}`,
+    existingDuplicateUrl
   };
 }
 async function uploadDirect(jobId, files) {
@@ -360,9 +395,16 @@ async function uploadDirect(jobId, files) {
   const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2191" });
   const failedFiles = [];
   const failureReasons = [];
-  for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {
-    const batch = files.slice(batchStart, batchStart + BATCH_SIZE);
-    const urlMap = await fetchSignedUrlsForBatch(jobId, batch.map((f) => f.relPath));
+  const batches = [];
+  for (let i = 0; i < files.length; i += BATCH_SIZE) {
+    batches.push(files.slice(i, i + BATCH_SIZE));
+  }
+  let nextUrlsPromise = batches.length > 0 ? fetchSignedUrlsForBatch(jobId, batches[0].map((f) => f.relPath)) : null;
+  const pending = /* @__PURE__ */ new Set();
+  for (let bi = 0; bi < batches.length; bi++) {
+    const batch = batches[bi];
+    const urlMap = await nextUrlsPromise;
+    nextUrlsPromise = bi + 1 < batches.length ? fetchSignedUrlsForBatch(jobId, batches[bi + 1].map((f) => f.relPath)) : null;
     for (const file of batch) {
       if (!urlMap.has(file.relPath)) {
         failedFiles.push(file.relPath);
@@ -371,12 +413,9 @@ async function uploadDirect(jobId, files) {
           reason: "no signed URL from server after retries"
         });
         bar.tick(0);
+        continue;
       }
-    }
-    const pending = /* @__PURE__ */ new Set();
-    for (const file of batch) {
       const signedUrl = urlMap.get(file.relPath);
-      if (!signedUrl) continue;
       const task = (async () => {
         const result = await uploadOneFileWithError(file, signedUrl);
         if (result.ok) {
@@ -394,8 +433,8 @@ async function uploadDirect(jobId, files) {
         await Promise.race(pending);
       }
     }
-    await Promise.all(pending);
   }
+  await Promise.all(pending);
   bar.done();
   if (failedFiles.length > 0) {
     const byReason = /* @__PURE__ */ new Map();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trajectories-sh",
-  "version": "1.10.3",
+  "version": "1.12.0",
   "description": "CLI for uploading trajectory jobs to trajectories.sh",
   "type": "module",
   "bin": {