trajectories-sh 1.10.3 → 1.11.0

package/dist/cli.js

@@ -81,7 +81,7 @@ async function uploadTrajectoryAction(directory, opts) {
   Uploading trajectory ${directory} \u2192 ${chalk.cyan(slug)}
 `));
   try {
-    const { uploadJob } = await import("./upload-UXM6LEBL.js");
+    const { uploadJob } = await import("./upload-YKSKGJET.js");
     const result = await uploadJob(directory, {
       slug,
       name: opts.name,

package/dist/{upload-UXM6LEBL.js → upload-YKSKGJET.js}

@@ -10,8 +10,10 @@ import { tmpdir } from "os";
 import { randomBytes, createHash } from "crypto";
 var SKIP = /* @__PURE__ */ new Set(["__pycache__", ".DS_Store", ".git", "node_modules"]);
 var MAX_FILE_SIZE = 50 * 1024 * 1024;
-var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "15");
+var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "32");
 var BATCH_SIZE = 100;
+var WEBP_CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_WEBP_CONCURRENCY ?? "4");
+var HASH_CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_HASH_CONCURRENCY ?? "16");
 function collectFiles(dir) {
   const files = [];
   function walk(current) {
@@ -85,26 +87,30 @@ async function convertToWebP(dir) {
   sharp.cache({ memory: 0, files: 0, items: 0 });
   let converted = 0;
   let savedBytes = 0;
-  try {
-    for (let i = 0; i < pngs.length; i++) {
-      const f = pngs[i];
-      const srcPath = join(tmpDir, f.relPath);
-      const webpPath = srcPath.replace(/\.png$/i, ".webp");
-      try {
-        await pngToWebp(srcPath, webpPath);
-        const origSize = statSync(srcPath).size;
-        const newSize = statSync(webpPath).size;
-        if (newSize < origSize) {
-          unlinkSync(srcPath);
-          savedBytes += origSize - newSize;
-          converted++;
-        } else {
-          unlinkSync(webpPath);
-        }
-      } catch {
+  let processed = 0;
+  const total = pngs.length;
+  async function convertOne(f) {
+    const srcPath = join(tmpDir, f.relPath);
+    const webpPath = srcPath.replace(/\.png$/i, ".webp");
+    try {
+      await pngToWebp(srcPath, webpPath);
+      const origSize = statSync(srcPath).size;
+      const newSize = statSync(webpPath).size;
+      if (newSize < origSize) {
+        unlinkSync(srcPath);
+        savedBytes += origSize - newSize;
+        converted++;
+      } else {
+        unlinkSync(webpPath);
       }
-      process.stdout.write(`  Converting: ${i + 1}/${pngs.length} PNGs\r`);
+    } catch {
+    } finally {
+      processed++;
+      process.stdout.write(`  Converting: ${processed}/${total} PNGs\r`);
     }
+  }
+  try {
+    await runWithConcurrency(pngs, WEBP_CONCURRENCY, convertOne);
     console.log("");
   } finally {
     try {
@@ -135,12 +141,34 @@ async function convertToWebP(dir) {
   }
   return { convertedDir: tmpDir, pngsConverted: converted, savedBytes };
 }
-function computeContentHash(files) {
-  const hash = createHash("sha256");
+async function runWithConcurrency(items, n, worker) {
+  if (items.length === 0) return;
+  let next = 0;
+  const runners = [];
+  for (let k = 0; k < Math.min(n, items.length); k++) {
+    runners.push(
+      (async () => {
+        while (true) {
+          const i = next++;
+          if (i >= items.length) return;
+          await worker(items[i], i);
+        }
+      })()
+    );
+  }
+  await Promise.all(runners);
+}
+async function computeContentHash(files) {
+  const { readFile } = await import("fs/promises");
   const sorted = [...files].sort((a, b) => a.relPath.localeCompare(b.relPath));
-  for (const f of sorted) {
-    hash.update(f.relPath);
-    hash.update(readFileSync(f.absPath));
+  const contents = new Array(sorted.length);
+  await runWithConcurrency(sorted, HASH_CONCURRENCY, async (f, i) => {
+    contents[i] = await readFile(f.absPath);
+  });
+  const hash = createHash("sha256");
+  for (let i = 0; i < sorted.length; i++) {
+    hash.update(sorted[i].relPath);
+    hash.update(contents[i]);
   }
   return hash.digest("hex");
 }
@@ -280,9 +308,11 @@ async function uploadJob(dir, opts) {
   }
   const files = collectFiles(convertedDir);
   const totalSize = files.reduce((s, f) => s + f.size, 0);
-  const contentHash = computeContentHash(files);
   console.log(`  ${files.length} files, ${(totalSize / 1024 / 1024).toFixed(1)} MB`);
-  console.log(`  Hash: ${contentHash.slice(0, 16)}...`);
+  process.stdout.write(`  Hashing...\r`);
+  const t0 = Date.now();
+  const contentHash = await computeContentHash(files);
+  console.log(`  Hash: ${contentHash.slice(0, 16)}... (${((Date.now() - t0) / 1e3).toFixed(1)}s)`);
   try {
     const auth = await getAuthHeader();
     const usageResp = await fetch(`${getApiUrl()}/api/cli/usage`, {
@@ -360,9 +390,16 @@ async function uploadDirect(jobId, files) {
   const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2191" });
   const failedFiles = [];
   const failureReasons = [];
-  for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {
-    const batch = files.slice(batchStart, batchStart + BATCH_SIZE);
-    const urlMap = await fetchSignedUrlsForBatch(jobId, batch.map((f) => f.relPath));
+  const batches = [];
+  for (let i = 0; i < files.length; i += BATCH_SIZE) {
+    batches.push(files.slice(i, i + BATCH_SIZE));
+  }
+  let nextUrlsPromise = batches.length > 0 ? fetchSignedUrlsForBatch(jobId, batches[0].map((f) => f.relPath)) : null;
+  const pending = /* @__PURE__ */ new Set();
+  for (let bi = 0; bi < batches.length; bi++) {
+    const batch = batches[bi];
+    const urlMap = await nextUrlsPromise;
+    nextUrlsPromise = bi + 1 < batches.length ? fetchSignedUrlsForBatch(jobId, batches[bi + 1].map((f) => f.relPath)) : null;
     for (const file of batch) {
       if (!urlMap.has(file.relPath)) {
         failedFiles.push(file.relPath);
@@ -371,12 +408,9 @@ async function uploadDirect(jobId, files) {
           reason: "no signed URL from server after retries"
         });
         bar.tick(0);
+        continue;
       }
-    }
-    const pending = /* @__PURE__ */ new Set();
-    for (const file of batch) {
       const signedUrl = urlMap.get(file.relPath);
-      if (!signedUrl) continue;
       const task = (async () => {
         const result = await uploadOneFileWithError(file, signedUrl);
         if (result.ok) {
@@ -394,8 +428,8 @@ async function uploadDirect(jobId, files) {
         await Promise.race(pending);
       }
     }
-    await Promise.all(pending);
   }
+  await Promise.all(pending);
   bar.done();
   if (failedFiles.length > 0) {
     const byReason = /* @__PURE__ */ new Map();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trajectories-sh",
-  "version": "1.10.3",
+  "version": "1.11.0",
   "description": "CLI for uploading trajectory jobs to trajectories.sh",
   "type": "module",
   "bin": {