trainfabric 0.1.36 → 0.1.38

package/dist/index.cjs

@@ -8480,18 +8480,28 @@ var RunHandle = class {
     const startedAt = Date.now();
     const pollMs = options?.pollMs ?? 1e3;
     const timeoutMs = options?.timeoutMs;
+    let transientFailures = 0;
     for (; ; ) {
-      const detail = await this.client.runs.get(this.snapshot.id);
-      this.snapshot = detail.run;
-      if (["completed", "failed", "canceled", "terminated"].includes(detail.run.status)) {
-        this.close();
-        return detail;
+      try {
+        const detail = await this.client.runs.get(this.snapshot.id);
+        transientFailures = 0;
+        this.snapshot = detail.run;
+        if (["completed", "failed", "canceled", "terminated"].includes(detail.run.status)) {
+          this.close();
+          return detail;
+        }
+      } catch (error) {
+        if (!this.isRetryablePollingError(error)) {
+          this.close();
+          throw error;
+        }
+        transientFailures += 1;
       }
       if (timeoutMs !== void 0 && Date.now() - startedAt > timeoutMs) {
         this.close();
         throw new Error(`Timed out waiting for run ${this.snapshot.id}.`);
       }
-      await new Promise((resolve) => setTimeout(resolve, pollMs));
+      await new Promise((resolve) => setTimeout(resolve, Math.min(pollMs * Math.max(transientFailures, 1), 1e4)));
     }
   }
   close() {
@@ -8505,6 +8515,12 @@ var RunHandle = class {
     this.streaming = true;
     void this.streamLoop();
   }
+  isRetryablePollingError(error) {
+    if (!(error instanceof Error)) {
+      return false;
+    }
+    return /status (502|503|504|429)\b|fetch failed|network|ECONNRESET|ETIMEDOUT|ENOTFOUND/i.test(error.message);
+  }
   async streamLoop() {
     while (this.shouldStream) {
       this.abortController = new AbortController();
@@ -8607,7 +8623,8 @@ var RunsClient = class extends ResourceClient {
   }
   async waitForProfile(profileSessionId, options) {
     const pollMs = options?.pollMs ?? 250;
-    const timeoutMs = options?.timeoutMs ?? 2e4;
+    const envTimeoutMs = Number(process.env.TRAINFABRIC_PROFILE_TIMEOUT_MS);
+    const timeoutMs = options?.timeoutMs ?? (Number.isFinite(envTimeoutMs) && envTimeoutMs > 0 ? envTimeoutMs : 9e4);
     const startedAt = Date.now();
     for (; ; ) {
       const detail = await this.profileStatus(profileSessionId);
@@ -9262,7 +9279,7 @@ function buildComputeSpec(options) {
 
 // src/index.ts
 var DEFAULT_TRAINFABRIC_API_URL2 = "https://api.trainfabric.com";
-var CLI_VERSION = "0.1.36";
+var CLI_VERSION = "0.1.38";
 var CONFIG_DIR = import_node_path3.default.join(import_node_os2.default.homedir(), ".trainfabric");
 var CONFIG_PATH = import_node_path3.default.join(CONFIG_DIR, "config.json");
 var FALLBACK_SECRET_PATH = import_node_path3.default.join(CONFIG_DIR, "session.enc");
@@ -9562,8 +9579,11 @@ function printQuoteSummary(bundle) {
     console.log(`  estimated infra cost: ${formatUsd(quote.expectedInfraCostUsd)}`);
     console.log(`  infra GPU-hour equivalent: ${formatUsdPerGpuHour(quote.expectedInfraCostUsd, gpuCount, quote.estimatedRuntimeSeconds)}`);
     console.log(`  estimated margin: ${formatUsd(quote.expectedMarginUsd)} (${formatPercent(quote.expectedMarginRatio)})`);
-    console.log(`  estimated runtime: ${Math.round(quote.estimatedRuntimeSeconds / 60)} min`);
-    console.log(`  reserved GPU time: ${formatGpuHours(gpuCount, quote.estimatedRuntimeSeconds)}`);
+    console.log(`  profiled runtime estimate: ${Math.round(quote.estimatedRuntimeSeconds / 60)} min`);
+    if (quote.reservationWindowSeconds && quote.reservationWindowSeconds !== quote.estimatedRuntimeSeconds) {
+      console.log(`  reservation window: ${Math.round(quote.reservationWindowSeconds / 60)} min`);
+    }
+    console.log(`  reserved GPU time: ${formatGpuHours(gpuCount, quote.reservationWindowSeconds ?? quote.estimatedRuntimeSeconds)}`);
     console.log(`  data path: ${quote.dataPath ?? "auto"}`);
     console.log(`  GPUs: ${gpuCount}`);
     console.log(`  pool: ${quote.selectedPoolId}`);
@@ -10002,7 +10022,7 @@ program2.command("runtime:build").option("--project <projectId>").requiredOption
     )
   );
 });
-program2.command("runs:quote").option("--project <projectId>").requiredOption("--dataset <datasetId>").requiredOption("--model <baseModel>").option("--eval <evalDatasetId>").option("--gpus <gpuCount>", "gpu count", "1").option("--nodes <nodeCount>", "node count", "1").option("--accelerator <acceleratorClass>", "optional hard accelerator constraint (for example: a10g, a100, h100)").option("--min-memory <gigabytes>", "minimum GPU memory in GB").option("--precision <precision>", "fp16_bf16, fp8, or fp32").option("--interconnect <interconnect>", "pcie or nvlink").option("--mode <mode>", "efficient, balanced, or power").option("--repo <path>").option("--git <url>").option("--branch <branch>").option("--summary", "print a human-readable price summary instead of raw JSON").description("Preview Efficient, Balanced, and Power launch options").action(async (options) => {
+program2.command("runs:quote").option("--project <projectId>").requiredOption("--dataset <datasetId>").requiredOption("--model <baseModel>").option("--eval <evalDatasetId>").option("--gpus <gpuCount>", "gpu count", "1").option("--nodes <nodeCount>", "node count", "1").option("--accelerator <acceleratorClass>", "optional hard accelerator constraint (for example: a10g, a100, h100)").option("--min-memory <gigabytes>", "minimum GPU memory in GB").option("--precision <precision>", "fp16_bf16, fp8, or fp32").option("--interconnect <interconnect>", "pcie or nvlink").option("--mode <mode>", "efficient, balanced, or power").option("--repo <path>").option("--git <url>").option("--branch <branch>").option("--json", "print raw JSON output; this is the default unless --summary is used").option("--summary", "print a human-readable price summary instead of raw JSON").description("Preview Efficient, Balanced, and Power launch options").action(async (options) => {
   const config = loadConfig();
   const bundle = filterQuoteBundleForRequestedMode(
     await createClient(config).runs.quote(buildRunInput(options, config)),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trainfabric",
-  "version": "0.1.36",
+  "version": "0.1.38",
   "description": "Trainfabric CLI for launching GPU training jobs on the hosted Trainfabric backend.",
   "type": "module",
   "main": "dist/index.cjs",