trajectories-sh 1.10.2 → 1.11.0

package/dist/cli.js

@@ -81,7 +81,7 @@ async function uploadTrajectoryAction(directory, opts) {
   Uploading trajectory ${directory} \u2192 ${chalk.cyan(slug)}
 `));
   try {
-    const { uploadJob } = await import("./upload-K2GBG3J3.js");
+    const { uploadJob } = await import("./upload-YKSKGJET.js");
     const result = await uploadJob(directory, {
       slug,
       name: opts.name,

package/dist/{upload-K2GBG3J3.js → upload-YKSKGJET.js}

@@ -10,7 +10,10 @@ import { tmpdir } from "os";
 import { randomBytes, createHash } from "crypto";
 var SKIP = /* @__PURE__ */ new Set(["__pycache__", ".DS_Store", ".git", "node_modules"]);
 var MAX_FILE_SIZE = 50 * 1024 * 1024;
-var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "8");
+var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "32");
+var BATCH_SIZE = 100;
+var WEBP_CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_WEBP_CONCURRENCY ?? "4");
+var HASH_CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_HASH_CONCURRENCY ?? "16");
 function collectFiles(dir) {
   const files = [];
   function walk(current) {
@@ -84,26 +87,30 @@ async function convertToWebP(dir) {
   sharp.cache({ memory: 0, files: 0, items: 0 });
   let converted = 0;
   let savedBytes = 0;
-  try {
-    for (let i = 0; i < pngs.length; i++) {
-      const f = pngs[i];
-      const srcPath = join(tmpDir, f.relPath);
-      const webpPath = srcPath.replace(/\.png$/i, ".webp");
-      try {
-        await pngToWebp(srcPath, webpPath);
-        const origSize = statSync(srcPath).size;
-        const newSize = statSync(webpPath).size;
-        if (newSize < origSize) {
-          unlinkSync(srcPath);
-          savedBytes += origSize - newSize;
-          converted++;
-        } else {
-          unlinkSync(webpPath);
-        }
-      } catch {
+  let processed = 0;
+  const total = pngs.length;
+  async function convertOne(f) {
+    const srcPath = join(tmpDir, f.relPath);
+    const webpPath = srcPath.replace(/\.png$/i, ".webp");
+    try {
+      await pngToWebp(srcPath, webpPath);
+      const origSize = statSync(srcPath).size;
+      const newSize = statSync(webpPath).size;
+      if (newSize < origSize) {
+        unlinkSync(srcPath);
+        savedBytes += origSize - newSize;
+        converted++;
+      } else {
+        unlinkSync(webpPath);
       }
-      process.stdout.write(`  Converting: ${i + 1}/${pngs.length} PNGs\r`);
+    } catch {
+    } finally {
+      processed++;
+      process.stdout.write(`  Converting: ${processed}/${total} PNGs\r`);
     }
+  }
+  try {
+    await runWithConcurrency(pngs, WEBP_CONCURRENCY, convertOne);
     console.log("");
   } finally {
     try {
@@ -134,12 +141,34 @@ async function convertToWebP(dir) {
   }
   return { convertedDir: tmpDir, pngsConverted: converted, savedBytes };
 }
-function computeContentHash(files) {
-  const hash = createHash("sha256");
+async function runWithConcurrency(items, n, worker) {
+  if (items.length === 0) return;
+  let next = 0;
+  const runners = [];
+  for (let k = 0; k < Math.min(n, items.length); k++) {
+    runners.push(
+      (async () => {
+        while (true) {
+          const i = next++;
+          if (i >= items.length) return;
+          await worker(items[i], i);
+        }
+      })()
+    );
+  }
+  await Promise.all(runners);
+}
+async function computeContentHash(files) {
+  const { readFile } = await import("fs/promises");
   const sorted = [...files].sort((a, b) => a.relPath.localeCompare(b.relPath));
-  for (const f of sorted) {
-    hash.update(f.relPath);
-    hash.update(readFileSync(f.absPath));
+  const contents = new Array(sorted.length);
+  await runWithConcurrency(sorted, HASH_CONCURRENCY, async (f, i) => {
+    contents[i] = await readFile(f.absPath);
+  });
+  const hash = createHash("sha256");
+  for (let i = 0; i < sorted.length; i++) {
+    hash.update(sorted[i].relPath);
+    hash.update(contents[i]);
   }
   return hash.digest("hex");
 }
@@ -279,9 +308,11 @@ async function uploadJob(dir, opts) {
   }
   const files = collectFiles(convertedDir);
   const totalSize = files.reduce((s, f) => s + f.size, 0);
-  const contentHash = computeContentHash(files);
   console.log(`  ${files.length} files, ${(totalSize / 1024 / 1024).toFixed(1)} MB`);
-  console.log(`  Hash: ${contentHash.slice(0, 16)}...`);
+  process.stdout.write(`  Hashing...\r`);
+  const t0 = Date.now();
+  const contentHash = await computeContentHash(files);
+  console.log(`  Hash: ${contentHash.slice(0, 16)}... (${((Date.now() - t0) / 1e3).toFixed(1)}s)`);
   try {
     const auth = await getAuthHeader();
     const usageResp = await fetch(`${getApiUrl()}/api/cli/usage`, {
@@ -322,7 +353,7 @@ async function uploadJob(dir, opts) {
   }
   const { job_id: jobId } = await initResp.json();
   console.log(`  Job ID: ${jobId}`);
-  await uploadArchive(jobId, convertedDir);
+  await uploadDirect(jobId, files);
   console.log("  Finalizing...");
   const finalResp = await apiPost(`/api/cli/push/${jobId}/finalize`, {});
   if (!finalResp.ok) {
@@ -353,135 +384,120 @@ async function uploadJob(dir, opts) {
     viewerUrl: `${siteUrl}/t/${jobId}`
   };
 }
-async function uploadArchive(jobId, dir) {
-  const tar = await import("tar");
-  const tmpArchive = join(tmpdir(), `trajectories-archive-${randomBytes(4).toString("hex")}.tar.gz`);
-  const entries = collectFiles(dir).map((f) => f.relPath);
-  await tar.create(
-    {
-      gzip: true,
-      file: tmpArchive,
-      cwd: dir,
-      portable: true
-    },
-    entries
-  );
-  const archiveSize = statSync(tmpArchive).size;
-  console.log(`  Archive: ${(archiveSize / 1024 / 1024).toFixed(1)} MB (gzipped, ${entries.length} entries)`);
-  try {
-    const maxAttempts = 3;
-    let lastError = null;
-    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-      if (attempt > 1) {
-        console.log(`  Retrying archive upload (attempt ${attempt}/${maxAttempts})\u2026`);
+async function uploadDirect(jobId, files) {
+  const { ProgressBar } = await import("./progress-H7WNKHDR.js");
+  const totalBytes = files.reduce((s, f) => s + f.size, 0);
+  const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2191" });
+  const failedFiles = [];
+  const failureReasons = [];
+  const batches = [];
+  for (let i = 0; i < files.length; i += BATCH_SIZE) {
+    batches.push(files.slice(i, i + BATCH_SIZE));
+  }
+  let nextUrlsPromise = batches.length > 0 ? fetchSignedUrlsForBatch(jobId, batches[0].map((f) => f.relPath)) : null;
+  const pending = /* @__PURE__ */ new Set();
+  for (let bi = 0; bi < batches.length; bi++) {
+    const batch = batches[bi];
+    const urlMap = await nextUrlsPromise;
+    nextUrlsPromise = bi + 1 < batches.length ? fetchSignedUrlsForBatch(jobId, batches[bi + 1].map((f) => f.relPath)) : null;
+    for (const file of batch) {
+      if (!urlMap.has(file.relPath)) {
+        failedFiles.push(file.relPath);
+        failureReasons.push({
+          path: file.relPath,
+          reason: "no signed URL from server after retries"
+        });
+        bar.tick(0);
+        continue;
       }
-      try {
-        await uploadArchiveOnce(jobId, tmpArchive, archiveSize, entries.length);
-        return;
-      } catch (e) {
-        lastError = e;
-        const msg = e?.message ?? String(e);
-        if (!/failed to store|extraction failed|extraction timed out/i.test(msg)) {
-          throw e;
-        }
-        if (attempt < maxAttempts) {
-          console.log(`  \u26A0 ${msg}`);
-          await sleep(3e3 * attempt);
+      const signedUrl = urlMap.get(file.relPath);
+      const task = (async () => {
+        const result = await uploadOneFileWithError(file, signedUrl);
+        if (result.ok) {
+          bar.tick(file.size);
+        } else {
+          failedFiles.push(file.relPath);
+          failureReasons.push({ path: file.relPath, reason: result.error });
+          bar.tick(0);
         }
+      })().finally(() => {
+        pending.delete(task);
+      });
+      pending.add(task);
+      if (pending.size >= CONCURRENCY) {
+        await Promise.race(pending);
       }
     }
-    throw lastError instanceof Error ? lastError : new Error(`Archive upload failed after ${maxAttempts} attempts`);
-  } finally {
-    try {
-      unlinkSync(tmpArchive);
-    } catch {
+  }
+  await Promise.all(pending);
+  bar.done();
+  if (failedFiles.length > 0) {
+    const byReason = /* @__PURE__ */ new Map();
+    for (const { reason } of failureReasons) {
+      byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
     }
+    const reasonLines = [...byReason.entries()].sort((a, b) => b[1] - a[1]).map(([reason, count]) => `  ${count}\xD7 ${reason}`).join("\n");
+    const samplePath = failedFiles[0];
+    throw new Error(
+      `Upload incomplete: ${failedFiles.length}/${files.length} file(s) failed. First failed file: ${samplePath}.
+Failure breakdown:
+${reasonLines}
+Retry the upload or check your network connection.`
+    );
   }
 }
-async function uploadArchiveOnce(jobId, archivePath, archiveSize, entryCount) {
-  const { ProgressBar } = await import("./progress-H7WNKHDR.js");
-  const urlResp = await apiPost(`/api/cli/push/${jobId}/upload-url`, {});
-  if (!urlResp.ok) {
-    throw new Error(`Failed to get archive upload URL: ${await urlResp.text()}`);
-  }
-  const { signed_url: signedUrl } = await urlResp.json();
-  if (!signedUrl) {
-    throw new Error("Server returned no signed URL for archive upload");
+async function fetchSignedUrlsForBatch(jobId, paths) {
+  const urlMap = /* @__PURE__ */ new Map();
+  let pending = paths.slice();
+  const attempts = 4;
+  for (let attempt = 0; attempt < attempts && pending.length > 0; attempt++) {
+    if (attempt > 0) {
+      const wait = 1500 * attempt;
+      await sleep(wait);
+    }
+    const resp = await apiPost(`/api/cli/push/${jobId}/upload-urls`, {
+      paths: pending
+    });
+    if (!resp.ok) {
+      throw new Error(`Failed to get upload URLs: ${await resp.text()}`);
+    }
+    const { urls } = await resp.json();
+    const stillPending = [];
+    for (const u of urls) {
+      if (u.signed_url) {
+        urlMap.set(u.path, u.signed_url);
+      } else {
+        stillPending.push(u.path);
+      }
+    }
+    pending = stillPending;
   }
-  const archiveData = readFileSync(archivePath);
-  let putError = null;
+  return urlMap;
+}
+async function uploadOneFileWithError(file, signedUrl) {
+  const data = readFileSync(file.absPath);
+  let lastError = "unknown error";
   for (let attempt = 0; attempt < 3; attempt++) {
-    const bar = new ProgressBar({ total: 1, totalBytes: archiveSize, label: "\u2191 archive" });
-    bar.setBytes(archiveSize);
     try {
       const resp = await fetch(signedUrl, {
         method: "PUT",
-        headers: { "Content-Type": "application/gzip" },
-        body: archiveData,
-        signal: AbortSignal.timeout(30 * 60 * 1e3)
+        headers: { "Content-Type": "application/octet-stream" },
+        body: data,
+        signal: AbortSignal.timeout(12e4)
       });
-      bar.setCount(1);
-      bar.done();
-      if (resp.ok) {
-        putError = null;
-        break;
-      }
-      putError = `HTTP ${resp.status} ${resp.statusText}`;
+      if (resp.ok) return { ok: true };
+      lastError = `HTTP ${resp.status} ${resp.statusText}`;
       try {
         const body = await resp.text();
-        if (body) putError += ` \u2014 ${body.slice(0, 200).replace(/\s+/g, " ")}`;
+        if (body) lastError += ` \u2014 ${body.slice(0, 150).replace(/\s+/g, " ")}`;
       } catch {
       }
     } catch (e) {
-      bar.done();
-      putError = `fetch error: ${e?.name ?? "Error"} ${e?.message ?? e}`;
-    }
-    if (attempt < 2) await sleep(2e3 * (attempt + 1));
-  }
-  if (putError) {
-    throw new Error(`Archive upload failed: ${putError}`);
-  }
-  const extractResp = await apiPost(`/api/cli/push/${jobId}/extract`, {});
-  if (!extractResp.ok) {
-    throw new Error(`Failed to start extraction: ${await extractResp.text()}`);
-  }
-  const extractBar = new ProgressBar({ total: entryCount, totalBytes: 0, label: "\u2699 extract" });
-  const pollStart = Date.now();
-  const pollTimeoutMs = 30 * 60 * 1e3;
-  let lastStatus = null;
-  while (Date.now() - pollStart < pollTimeoutMs) {
-    await sleep(1500);
-    const statusResp = await apiGet(`/api/cli/push/${jobId}/extract-status`);
-    if (!statusResp.ok) {
-      throw new Error(`Failed to poll extract status: ${await statusResp.text()}`);
-    }
-    lastStatus = await statusResp.json();
-    const uploaded = Number(lastStatus?.uploaded ?? 0);
-    const errors = Number(lastStatus?.errors ?? 0);
-    extractBar.setCount(uploaded + errors);
-    if (lastStatus?.status === "done") break;
-    if (lastStatus?.status === "error") {
-      throw new Error(`Archive extraction failed: ${lastStatus?.detail ?? "unknown"}`);
+      lastError = `fetch error: ${e?.name ?? "Error"} ${e?.message ?? e}`;
     }
+    if (attempt < 2) await sleep(1e3 * (attempt + 1));
   }
-  extractBar.done();
-  if (!lastStatus || lastStatus.status !== "done") {
-    throw new Error("Archive extraction timed out");
-  }
-  if (lastStatus.errors && lastStatus.errors > 0) {
-    throw new Error(
-      `Archive extraction completed but ${lastStatus.errors} file(s) failed to store. Retry the upload or check server logs.`
-    );
-  }
-}
-async function apiGet(path, timeout = 6e4) {
-  const auth = await getAuthHeader();
-  if (!auth) throw new Error("Not authenticated. Run: trajectories auth login");
-  return fetch(`${getApiUrl()}${path}`, {
-    method: "GET",
-    headers: { Authorization: auth },
-    signal: AbortSignal.timeout(timeout)
-  });
+  return { ok: false, error: lastError };
 }
 function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trajectories-sh",
-  "version": "1.10.2",
+  "version": "1.11.0",
   "description": "CLI for uploading trajectory jobs to trajectories.sh",
   "type": "module",
   "bin": {