trajectories-sh 1.10.1 → 1.10.3

package/dist/cli.js

@@ -81,7 +81,7 @@ async function uploadTrajectoryAction(directory, opts) {
   Uploading trajectory ${directory} \u2192 ${chalk.cyan(slug)}
 `));
   try {
-    const { uploadJob } = await import("./upload-IBWVHRL2.js");
+    const { uploadJob } = await import("./upload-UXM6LEBL.js");
     const result = await uploadJob(directory, {
       slug,
       name: opts.name,

package/dist/{upload-IBWVHRL2.js → upload-UXM6LEBL.js}

@@ -10,7 +10,8 @@ import { tmpdir } from "os";
 import { randomBytes, createHash } from "crypto";
 var SKIP = /* @__PURE__ */ new Set(["__pycache__", ".DS_Store", ".git", "node_modules"]);
 var MAX_FILE_SIZE = 50 * 1024 * 1024;
-var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "8");
+var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "15");
+var BATCH_SIZE = 100;
 function collectFiles(dir) {
   const files = [];
   function walk(current) {
@@ -322,7 +323,7 @@ async function uploadJob(dir, opts) {
   }
   const { job_id: jobId } = await initResp.json();
   console.log(`  Job ID: ${jobId}`);
-  await uploadArchive(jobId, convertedDir);
+  await uploadDirect(jobId, files);
   console.log("  Finalizing...");
   const finalResp = await apiPost(`/api/cli/push/${jobId}/finalize`, {});
   if (!finalResp.ok) {
@@ -353,149 +354,116 @@ async function uploadJob(dir, opts) {
     viewerUrl: `${siteUrl}/t/${jobId}`
   };
 }
-async function uploadArchive(jobId, dir) {
-  const tar = await import("tar");
-  const tmpArchive = join(tmpdir(), `trajectories-archive-${randomBytes(4).toString("hex")}.tar.gz`);
-  const entries = collectFiles(dir).map((f) => f.relPath);
-  await tar.create(
-    {
-      gzip: true,
-      file: tmpArchive,
-      cwd: dir,
-      portable: true
-    },
-    entries
-  );
-  const archiveSize = statSync(tmpArchive).size;
-  console.log(`  Archive: ${(archiveSize / 1024 / 1024).toFixed(1)} MB (gzipped, ${entries.length} entries)`);
-  try {
-    const maxAttempts = 3;
-    let lastError = null;
-    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-      if (attempt > 1) {
-        console.log(`  Retrying archive upload (attempt ${attempt}/${maxAttempts})\u2026`);
+async function uploadDirect(jobId, files) {
+  const { ProgressBar } = await import("./progress-H7WNKHDR.js");
+  const totalBytes = files.reduce((s, f) => s + f.size, 0);
+  const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2191" });
+  const failedFiles = [];
+  const failureReasons = [];
+  for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {
+    const batch = files.slice(batchStart, batchStart + BATCH_SIZE);
+    const urlMap = await fetchSignedUrlsForBatch(jobId, batch.map((f) => f.relPath));
+    for (const file of batch) {
+      if (!urlMap.has(file.relPath)) {
+        failedFiles.push(file.relPath);
+        failureReasons.push({
+          path: file.relPath,
+          reason: "no signed URL from server after retries"
+        });
+        bar.tick(0);
       }
-      try {
-        await uploadArchiveOnce(jobId, tmpArchive, archiveSize, entries.length);
-        return;
-      } catch (e) {
-        lastError = e;
-        const msg = e?.message ?? String(e);
-        if (!/failed to store|extraction failed|extraction timed out/i.test(msg)) {
-          throw e;
-        }
-        if (attempt < maxAttempts) {
-          console.log(`  \u26A0 ${msg}`);
-          await sleep(3e3 * attempt);
+    }
+    const pending = /* @__PURE__ */ new Set();
+    for (const file of batch) {
+      const signedUrl = urlMap.get(file.relPath);
+      if (!signedUrl) continue;
+      const task = (async () => {
+        const result = await uploadOneFileWithError(file, signedUrl);
+        if (result.ok) {
+          bar.tick(file.size);
+        } else {
+          failedFiles.push(file.relPath);
+          failureReasons.push({ path: file.relPath, reason: result.error });
+          bar.tick(0);
         }
+      })().finally(() => {
+        pending.delete(task);
+      });
+      pending.add(task);
+      if (pending.size >= CONCURRENCY) {
+        await Promise.race(pending);
       }
     }
-    throw lastError instanceof Error ? lastError : new Error(`Archive upload failed after ${maxAttempts} attempts`);
-  } finally {
-    try {
-      unlinkSync(tmpArchive);
-    } catch {
+    await Promise.all(pending);
+  }
+  bar.done();
+  if (failedFiles.length > 0) {
+    const byReason = /* @__PURE__ */ new Map();
+    for (const { reason } of failureReasons) {
+      byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
     }
+    const reasonLines = [...byReason.entries()].sort((a, b) => b[1] - a[1]).map(([reason, count]) => `  ${count}\xD7 ${reason}`).join("\n");
+    const samplePath = failedFiles[0];
+    throw new Error(
+      `Upload incomplete: ${failedFiles.length}/${files.length} file(s) failed. First failed file: ${samplePath}.
+Failure breakdown:
+${reasonLines}
+Retry the upload or check your network connection.`
+    );
   }
 }
-async function uploadArchiveOnce(jobId, archivePath, archiveSize, entryCount) {
-  const { createReadStream } = await import("fs");
-  const { ProgressBar } = await import("./progress-H7WNKHDR.js");
-  const urlResp = await apiPost(`/api/cli/push/${jobId}/upload-url`, {});
-  if (!urlResp.ok) {
-    throw new Error(`Failed to get archive upload URL: ${await urlResp.text()}`);
-  }
-  const { signed_url: signedUrl } = await urlResp.json();
-  if (!signedUrl) {
-    throw new Error("Server returned no signed URL for archive upload");
+async function fetchSignedUrlsForBatch(jobId, paths) {
+  const urlMap = /* @__PURE__ */ new Map();
+  let pending = paths.slice();
+  const attempts = 4;
+  for (let attempt = 0; attempt < attempts && pending.length > 0; attempt++) {
+    if (attempt > 0) {
+      const wait = 1500 * attempt;
+      await sleep(wait);
+    }
+    const resp = await apiPost(`/api/cli/push/${jobId}/upload-urls`, {
+      paths: pending
+    });
+    if (!resp.ok) {
+      throw new Error(`Failed to get upload URLs: ${await resp.text()}`);
+    }
+    const { urls } = await resp.json();
+    const stillPending = [];
+    for (const u of urls) {
+      if (u.signed_url) {
+        urlMap.set(u.path, u.signed_url);
+      } else {
+        stillPending.push(u.path);
+      }
+    }
+    pending = stillPending;
   }
-  let putError = null;
+  return urlMap;
+}
+async function uploadOneFileWithError(file, signedUrl) {
+  const data = readFileSync(file.absPath);
+  let lastError = "unknown error";
   for (let attempt = 0; attempt < 3; attempt++) {
-    const bar = new ProgressBar({ total: 1, totalBytes: archiveSize, label: "\u2191 archive" });
-    const stream = createReadStream(archivePath);
-    let bytesSent = 0;
-    stream.on("data", (chunk) => {
-      bytesSent += chunk.length;
-      bar.setBytes(bytesSent);
-    });
     try {
       const resp = await fetch(signedUrl, {
         method: "PUT",
-        headers: {
-          "Content-Type": "application/gzip",
-          "Content-Length": String(archiveSize)
-        },
-        // undici accepts Node streams here
-        body: stream,
-        duplex: "half",
-        signal: AbortSignal.timeout(30 * 60 * 1e3)
+        headers: { "Content-Type": "application/octet-stream" },
+        body: data,
+        signal: AbortSignal.timeout(12e4)
       });
-      bar.setCount(1);
-      bar.done();
-      if (resp.ok) {
-        putError = null;
-        break;
-      }
-      putError = `HTTP ${resp.status} ${resp.statusText}`;
+      if (resp.ok) return { ok: true };
+      lastError = `HTTP ${resp.status} ${resp.statusText}`;
       try {
         const body = await resp.text();
-        if (body) putError += ` \u2014 ${body.slice(0, 200).replace(/\s+/g, " ")}`;
+        if (body) lastError += ` \u2014 ${body.slice(0, 150).replace(/\s+/g, " ")}`;
       } catch {
       }
     } catch (e) {
-      bar.done();
-      putError = `fetch error: ${e?.name ?? "Error"} ${e?.message ?? e}`;
-    }
-    try {
-      stream.destroy?.();
-    } catch {
-    }
-    if (attempt < 2) await sleep(2e3 * (attempt + 1));
-  }
-  if (putError) {
-    throw new Error(`Archive upload failed: ${putError}`);
-  }
-  const extractResp = await apiPost(`/api/cli/push/${jobId}/extract`, {});
-  if (!extractResp.ok) {
-    throw new Error(`Failed to start extraction: ${await extractResp.text()}`);
-  }
-  const extractBar = new ProgressBar({ total: entryCount, totalBytes: 0, label: "\u2699 extract" });
-  const pollStart = Date.now();
-  const pollTimeoutMs = 30 * 60 * 1e3;
-  let lastStatus = null;
-  while (Date.now() - pollStart < pollTimeoutMs) {
-    await sleep(1500);
-    const statusResp = await apiGet(`/api/cli/push/${jobId}/extract-status`);
-    if (!statusResp.ok) {
-      throw new Error(`Failed to poll extract status: ${await statusResp.text()}`);
-    }
-    lastStatus = await statusResp.json();
-    const uploaded = Number(lastStatus?.uploaded ?? 0);
-    const errors = Number(lastStatus?.errors ?? 0);
-    extractBar.setCount(uploaded + errors);
-    if (lastStatus?.status === "done") break;
-    if (lastStatus?.status === "error") {
-      throw new Error(`Archive extraction failed: ${lastStatus?.detail ?? "unknown"}`);
+      lastError = `fetch error: ${e?.name ?? "Error"} ${e?.message ?? e}`;
     }
+    if (attempt < 2) await sleep(1e3 * (attempt + 1));
   }
-  extractBar.done();
-  if (!lastStatus || lastStatus.status !== "done") {
-    throw new Error("Archive extraction timed out");
-  }
-  if (lastStatus.errors && lastStatus.errors > 0) {
-    throw new Error(
-      `Archive extraction completed but ${lastStatus.errors} file(s) failed to store. Retry the upload or check server logs.`
-    );
-  }
-}
-async function apiGet(path, timeout = 6e4) {
-  const auth = await getAuthHeader();
-  if (!auth) throw new Error("Not authenticated. Run: trajectories auth login");
-  return fetch(`${getApiUrl()}${path}`, {
-    method: "GET",
-    headers: { Authorization: auth },
-    signal: AbortSignal.timeout(timeout)
-  });
+  return { ok: false, error: lastError };
 }
 function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trajectories-sh",
-  "version": "1.10.1",
+  "version": "1.10.3",
   "description": "CLI for uploading trajectory jobs to trajectories.sh",
   "type": "module",
   "bin": {