trajectories-sh 1.9.9 → 1.10.0

package/dist/cli.js

@@ -81,7 +81,7 @@ async function uploadTrajectoryAction(directory, opts) {
   Uploading trajectory ${directory} \u2192 ${chalk.cyan(slug)}
 `));
   try {
-    const { uploadJob } = await import("./upload-3VW7BFIP.js");
+    const { uploadJob } = await import("./upload-EHVKRW64.js");
     const result = await uploadJob(directory, {
       slug,
       name: opts.name,
@@ -145,7 +145,7 @@ async function downloadTrajectoryAction(url, opts) {
   Downloading trajectory...
 `));
   try {
-    const { downloadJob, parseJobId } = await import("./download-FJ7U5MTQ.js");
+    const { downloadJob, parseJobId } = await import("./download-O3JONHXT.js");
     const outputDir = opts.output ?? parseJobId(url).replace(/^org:/, "").replace(/:/g, "-");
     const result = await downloadJob(url, outputDir);
     console.log(chalk.green.bold(`  \u2713 Downloaded ${result.files} files`));
@@ -161,7 +161,7 @@ async function downloadTrajectoryAction(url, opts) {
 }
 async function deleteTrajectoryAction(url, opts) {
   const chalk = (await import("chalk")).default;
-  const { parseJobId } = await import("./download-FJ7U5MTQ.js");
+  const { parseJobId } = await import("./download-O3JONHXT.js");
   const { getApiUrl, getAuthHeader } = await import("./config-NOIAMV2U.js");
   let jobId = parseJobId(url);
   if (jobId.startsWith("org:")) {

package/dist/{download-FJ7U5MTQ.js → download-O3JONHXT.js}

@@ -56,7 +56,7 @@ async function downloadJob(input, outputDir) {
   console.log(`  Job: ${slug} (${jobId})`);
   console.log(`  ${files.length} files, ${(totalBytes / 1024 / 1024).toFixed(1)} MB`);
   mkdirSync(outputDir, { recursive: true });
-  const { ProgressBar } = await import("./progress-2P5L7X24.js");
+  const { ProgressBar } = await import("./progress-H7WNKHDR.js");
   const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2193" });
   let downloaded = 0;
   let totalSize = 0;

package/dist/{progress-2P5L7X24.js → progress-H7WNKHDR.js}

@@ -17,6 +17,16 @@ var ProgressBar = class {
     this.currentBytes += bytes;
     this.render();
   }
+  /** Set absolute byte count (does not change `current`). Useful for streaming a single large file. */
+  setBytes(bytes) {
+    this.currentBytes = bytes;
+    this.render();
+  }
+  /** Set absolute file count (does not change `currentBytes`). */
+  setCount(count) {
+    this.current = count;
+    this.render();
+  }
   render() {
     const pct = this.total > 0 ? Math.round(this.current / this.total * 100) : 0;
     const elapsed = (Date.now() - this.startTime) / 1e3;

package/dist/{upload-3VW7BFIP.js → upload-EHVKRW64.js}

@@ -11,7 +11,6 @@ import { randomBytes, createHash } from "crypto";
 var SKIP = /* @__PURE__ */ new Set(["__pycache__", ".DS_Store", ".git", "node_modules"]);
 var MAX_FILE_SIZE = 50 * 1024 * 1024;
 var CONCURRENCY = Number(process.env.TRAJECTORIES_CLI_CONCURRENCY ?? "8");
-var BATCH_SIZE = 100;
 function collectFiles(dir) {
   const files = [];
   function walk(current) {
@@ -323,7 +322,7 @@ async function uploadJob(dir, opts) {
   }
   const { job_id: jobId } = await initResp.json();
   console.log(`  Job ID: ${jobId}`);
-  await uploadDirect(jobId, files);
+  await uploadArchive(jobId, convertedDir);
   console.log("  Finalizing...");
   const finalResp = await apiPost(`/api/cli/push/${jobId}/finalize`, {});
   if (!finalResp.ok) {
@@ -354,96 +353,125 @@ async function uploadJob(dir, opts) {
     viewerUrl: `${siteUrl}/t/${jobId}`
   };
 }
-async function uploadDirect(jobId, files) {
-  const { ProgressBar } = await import("./progress-2P5L7X24.js");
-  const totalBytes = files.reduce((s, f) => s + f.size, 0);
-  const bar = new ProgressBar({ total: files.length, totalBytes, label: "\u2191" });
-  const failedFiles = [];
-  const failureReasons = [];
-  for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {
-    const batch = files.slice(batchStart, batchStart + BATCH_SIZE);
-    const urlResp = await apiPost(`/api/cli/push/${jobId}/upload-urls`, {
-      paths: batch.map((f) => f.relPath)
-    });
+async function uploadArchive(jobId, dir) {
+  const tar = await import("tar");
+  const { createReadStream } = await import("fs");
+  const { ProgressBar } = await import("./progress-H7WNKHDR.js");
+  const tmpArchive = join(tmpdir(), `trajectories-archive-${randomBytes(4).toString("hex")}.tar.gz`);
+  const entries = collectFiles(dir).map((f) => f.relPath);
+  await tar.create(
+    {
+      gzip: true,
+      file: tmpArchive,
+      cwd: dir,
+      portable: true
+    },
+    entries
+  );
+  const archiveSize = statSync(tmpArchive).size;
+  console.log(`  Archive: ${(archiveSize / 1024 / 1024).toFixed(1)} MB (gzipped, ${entries.length} entries)`);
+  try {
+    const urlResp = await apiPost(`/api/cli/push/${jobId}/upload-url`, {});
     if (!urlResp.ok) {
-      throw new Error(`Failed to get upload URLs: ${await urlResp.text()}`);
+      throw new Error(`Failed to get archive upload URL: ${await urlResp.text()}`);
+    }
+    const { signed_url: signedUrl } = await urlResp.json();
+    if (!signedUrl) {
+      throw new Error("Server returned no signed URL for archive upload");
     }
-    const { urls } = await urlResp.json();
-    const urlMap = /* @__PURE__ */ new Map();
-    for (const u of urls) {
-      if (u.signed_url) urlMap.set(u.path, u.signed_url);
-      else {
-        failedFiles.push(u.path);
-        failureReasons.push({
-          path: u.path,
-          reason: `no signed URL from server: ${u.error ?? "unknown"}`
+    let putError = null;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      const bar = new ProgressBar({ total: 1, totalBytes: archiveSize, label: "\u2191 archive" });
+      const stream = createReadStream(tmpArchive);
+      let bytesSent = 0;
+      stream.on("data", (chunk) => {
+        bytesSent += chunk.length;
+        bar.setBytes(bytesSent);
+      });
+      try {
+        const resp = await fetch(signedUrl, {
+          method: "PUT",
+          headers: {
+            "Content-Type": "application/gzip",
+            "Content-Length": String(archiveSize)
+          },
+          // undici accepts Node streams here
+          body: stream,
+          duplex: "half",
+          signal: AbortSignal.timeout(30 * 60 * 1e3)
         });
-        bar.tick(0);
+        bar.setCount(1);
+        bar.done();
+        if (resp.ok) {
+          putError = null;
+          break;
+        }
+        putError = `HTTP ${resp.status} ${resp.statusText}`;
+        try {
+          const body = await resp.text();
+          if (body) putError += ` \u2014 ${body.slice(0, 200).replace(/\s+/g, " ")}`;
+        } catch {
+        }
+      } catch (e) {
+        bar.done();
+        putError = `fetch error: ${e?.name ?? "Error"} ${e?.message ?? e}`;
       }
+      try {
+        stream.destroy?.();
+      } catch {
+      }
+      if (attempt < 2) await sleep(2e3 * (attempt + 1));
     }
-    const pending = /* @__PURE__ */ new Set();
-    for (const file of batch) {
-      const signedUrl = urlMap.get(file.relPath);
-      if (!signedUrl) continue;
-      const task = (async () => {
-        const result = await uploadOneFileWithError(file, signedUrl);
-        if (result.ok) {
-          bar.tick(file.size);
-        } else {
-          failedFiles.push(file.relPath);
-          failureReasons.push({ path: file.relPath, reason: result.error });
-          bar.tick(0);
-        }
-      })().finally(() => {
-        pending.delete(task);
-      });
-      pending.add(task);
-      if (pending.size >= CONCURRENCY) {
-        await Promise.race(pending);
+    if (putError) {
+      throw new Error(`Archive upload failed: ${putError}`);
+    }
+    const extractResp = await apiPost(`/api/cli/push/${jobId}/extract`, {});
+    if (!extractResp.ok) {
+      throw new Error(`Failed to start extraction: ${await extractResp.text()}`);
+    }
+    const extractBar = new ProgressBar({ total: entries.length, totalBytes: 0, label: "\u2699 extract" });
+    const pollStart = Date.now();
+    const pollTimeoutMs = 30 * 60 * 1e3;
+    let lastStatus = null;
+    while (Date.now() - pollStart < pollTimeoutMs) {
+      await sleep(1500);
+      const statusResp = await apiGet(`/api/cli/push/${jobId}/extract-status`);
+      if (!statusResp.ok) {
+        throw new Error(`Failed to poll extract status: ${await statusResp.text()}`);
+      }
+      lastStatus = await statusResp.json();
+      const uploaded = Number(lastStatus?.uploaded ?? 0);
+      const errors = Number(lastStatus?.errors ?? 0);
+      extractBar.setCount(uploaded + errors);
+      if (lastStatus?.status === "done") break;
+      if (lastStatus?.status === "error") {
+        throw new Error(`Archive extraction failed: ${lastStatus?.detail ?? "unknown"}`);
       }
     }
-    await Promise.all(pending);
-  }
-  bar.done();
-  if (failedFiles.length > 0) {
-    const byReason = /* @__PURE__ */ new Map();
-    for (const { reason } of failureReasons) {
-      byReason.set(reason, (byReason.get(reason) ?? 0) + 1);
+    extractBar.done();
+    if (!lastStatus || lastStatus.status !== "done") {
+      throw new Error("Archive extraction timed out");
     }
-    const reasonLines = [...byReason.entries()].sort((a, b) => b[1] - a[1]).map(([reason, count]) => `  ${count}\xD7 ${reason}`).join("\n");
-    const samplePath = failedFiles[0];
-    throw new Error(
-      `Upload incomplete: ${failedFiles.length}/${files.length} file(s) failed. First failed file: ${samplePath}.
-Failure breakdown:
-${reasonLines}
-Retry the upload or check your network connection.`
-    );
-  }
-}
-async function uploadOneFileWithError(file, signedUrl) {
-  const data = readFileSync(file.absPath);
-  let lastError = "unknown error";
-  for (let attempt = 0; attempt < 3; attempt++) {
+    if (lastStatus.errors && lastStatus.errors > 0) {
+      throw new Error(
+        `Archive extraction completed but ${lastStatus.errors} file(s) failed to store. Retry the upload or check server logs.`
+      );
+    }
+  } finally {
     try {
-      const resp = await fetch(signedUrl, {
-        method: "PUT",
-        headers: { "Content-Type": "application/octet-stream" },
-        body: data,
-        signal: AbortSignal.timeout(12e4)
-      });
-      if (resp.ok) return { ok: true };
-      lastError = `HTTP ${resp.status} ${resp.statusText}`;
-      try {
-        const body = await resp.text();
-        if (body) lastError += ` \u2014 ${body.slice(0, 150).replace(/\s+/g, " ")}`;
-      } catch {
-      }
-    } catch (e) {
-      lastError = `fetch error: ${e?.name ?? "Error"} ${e?.message ?? e}`;
+      unlinkSync(tmpArchive);
+    } catch {
     }
-    if (attempt < 2) await sleep(1e3 * (attempt + 1));
   }
-  return { ok: false, error: lastError };
+}
+async function apiGet(path, timeout = 6e4) {
+  const auth = await getAuthHeader();
+  if (!auth) throw new Error("Not authenticated. Run: trajectories auth login");
+  return fetch(`${getApiUrl()}${path}`, {
+    method: "GET",
+    headers: { Authorization: auth },
+    signal: AbortSignal.timeout(timeout)
+  });
 }
 function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trajectories-sh",
-  "version": "1.9.9",
+  "version": "1.10.0",
   "description": "CLI for uploading trajectory jobs to trajectories.sh",
   "type": "module",
   "bin": {