pixelkiln 0.20.1 → 0.20.2

package/dist/cli.js

@@ -919,7 +919,7 @@ var ComfyUIProvider = class _ComfyUIProvider {
   rateLimit() {
     return { spacingMs: 0, maxInFlight: 1 };
   }
-  async submit(spec, styleImages) {
+  async submit(spec, styleImages, context) {
     this.validate(spec, styleImages);
     const options = resolvedOptions(spec);
     const uploadedImages = /* @__PURE__ */ new Map();
@@ -961,22 +961,35 @@ var ComfyUIProvider = class _ComfyUIProvider {
     if (spec.generator === "frames") {
       const frames = options.frames;
       const values = spec.providerInputs[frames.vary];
-      const promptIds = [];
-      for (let index = 0; index < values.length; index++) {
+      let promptIds = [];
+      if (context?.previousJobId) {
+        const previous = decodeFrameSetJob(context.previousJobId);
+        if (previous.outputNodeId !== options.outputNodeId || previous.expectedCount !== values.length) {
+          throw new Error("Saved ComfyUI frame checkpoint does not match the current frame set");
+        }
+        promptIds = [...previous.promptIds];
+      }
+      for (let index = promptIds.length; index < values.length; index++) {
         promptIds.push(await this.client.submit(await buildWorkflow(index)));
+        const frameJob2 = {
+          expectedCount: values.length,
+          promptIds: [...promptIds],
+          outputNodeId: options.outputNodeId
+        };
+        await context?.checkpoint({
+          jobId: encodeFrameSetJob(frameJob2),
+          metadata: comfyFrameSubmissionMetadata(spec, frameJob2),
+          complete: promptIds.length === values.length
+        });
       }
+      const frameJob = {
+        expectedCount: values.length,
+        promptIds,
+        outputNodeId: options.outputNodeId
+      };
       return {
-        jobId: encodeFrameSetJob({ promptIds, outputNodeId: options.outputNodeId }),
-        metadata: {
-          inputs,
-          frameSet: {
-            count: values.length,
-            vary: frames.vary,
-            seedStep: frames.seedStep,
-            fps: spec.quality?.fps ?? 12,
-            promptIds
-          }
-        }
+        jobId: encodeFrameSetJob(frameJob),
+        metadata: comfyFrameSubmissionMetadata(spec, frameJob)
       };
     }
     const workflow = await buildWorkflow();
@@ -1033,6 +1046,9 @@ var ComfyUIProvider = class _ComfyUIProvider {
   async poll(jobId, generator, context) {
     if (generator === "frames") {
       const frameJob = decodeFrameSetJob(jobId);
+      if (frameJob.promptIds.length < frameJob.expectedCount) {
+        return { status: "processing" };
+      }
       const images2 = [];
       for (let index = 0; index < frameJob.promptIds.length; index++) {
         const promptId2 = frameJob.promptIds[index];
@@ -1305,6 +1321,19 @@ function comfyFrameMetadata(spec, job, images, fps) {
     }
   };
 }
+function comfyFrameSubmissionMetadata(spec, job) {
+  const options = resolvedOptions(spec);
+  return {
+    inputs: providerInputProvenance(spec.providerInputs),
+    frameSet: {
+      count: job.expectedCount,
+      vary: options.frames.vary,
+      seedStep: options.frames.seedStep,
+      fps: spec.quality?.fps ?? 12,
+      promptIds: job.promptIds
+    }
+  };
+}
 function isImageBinding(workflow, binding) {
   const node = workflow[binding.nodeId];
   return binding.input === "image" && (node?.class_type === "LoadImage" || node?.class_type === "LoadImageMask");
@@ -1360,10 +1389,18 @@ function decodeFrameSetJob(jobId) {
   } catch {
     throw new Error(`Invalid ComfyUI frame-set job id "${jobId}"`);
   }
-  if (!isObject(value) || typeof value.outputNodeId !== "string" || !value.outputNodeId || !Array.isArray(value.promptIds) || value.promptIds.length < 2 || value.promptIds.length > 64 || !value.promptIds.every((id) => typeof id === "string" && id.length > 0)) {
+  if (!isObject(value) || !Array.isArray(value.promptIds)) {
     throw new Error(`Invalid ComfyUI frame-set job id "${jobId}"`);
   }
-  return { outputNodeId: value.outputNodeId, promptIds: value.promptIds };
+  const expectedCount = value.expectedCount === void 0 ? value.promptIds.length : value.expectedCount;
+  if (typeof value.outputNodeId !== "string" || !value.outputNodeId || typeof expectedCount !== "number" || !Number.isSafeInteger(expectedCount) || expectedCount < 2 || expectedCount > 64 || value.promptIds.length < 1 || value.promptIds.length > expectedCount || !value.promptIds.every((id) => typeof id === "string" && id.length > 0)) {
+    throw new Error(`Invalid ComfyUI frame-set job id "${jobId}"`);
+  }
+  return {
+    expectedCount,
+    outputNodeId: value.outputNodeId,
+    promptIds: value.promptIds
+  };
 }
 function decodeJob(jobId) {
   const split = jobId.lastIndexOf("#");
@@ -2146,6 +2183,8 @@ var LockEntrySchema = z2.object({
   ).optional(),
   /** Set at submit time, before the request is awaited, so a crash is recoverable. */
   jobId: z2.string().nullable().default(null),
+  /** False only when a provider checkpoint says more requests remain. */
+  submissionComplete: z2.boolean().optional(),
   /** For `1dir`: the multi-candidate parent object awaiting selection. */
   reviewObjectId: z2.string().nullable().default(null),
   /** The chosen candidate's own object id, once selected. */
@@ -5916,6 +5955,7 @@ function resumeActions(specs, lock) {
     const key = lockKey(spec.styleId, spec.assetId);
     const entry = lock.entries[key];
     if (!entry || entry.specHash !== spec.specHash) continue;
+    if (entry.submissionComplete === false) continue;
     const command = resumeCommandForStatus(entry.status);
     if (!command) continue;
     if (command === "poll" && !entry.jobId) continue;
@@ -5966,10 +6006,13 @@ async function buildPlan(specs, lock, opts = {}) {
       reason = `${entry.error ?? "download failed"}; run pixelkiln fetch${force} (no generation cost)`;
     } else if (entry.status === "failed") {
       state = "failed";
-      reason = entry.error ?? "previous attempt failed";
+      reason = entry.jobId && entry.submissionComplete === false ? `${entry.error ?? "submission interrupted"}; saved checkpoint will resume` : entry.error ?? "previous attempt failed";
     } else if (entry.status === "selected") {
       state = "recoverable";
       reason = "provider output is selected; run pixelkiln fetch (no generation cost)";
+    } else if ((entry.status === "pending" || entry.status === "processing") && entry.jobId && entry.submissionComplete === false) {
+      state = "failed";
+      reason = "submission stopped after a saved checkpoint; rerun pixelkiln submit to resume";
     } else if (entry.status === "pending" || entry.status === "processing") {
       state = "in-flight";
       reason = entry.jobId ? "awaiting processing; run pixelkiln poll" : "submission state has no job id; run pixelkiln doctor before retrying";
@@ -6113,6 +6156,11 @@ async function submit(provider, loaded, items, lock, lockPath, opts = {}) {
     if (since < spacing) await sleep2(spacing - since);
     await requireRevisionReady(spec, lock);
     const previousEntry = lock.entries[key];
+    const resumesCheckpoint = Boolean(
+      previousEntry?.specHash === spec.specHash && previousEntry.provider === provider.id && previousEntry.generator === spec.generator && previousEntry.submissionComplete === false && previousEntry.jobId
+    );
+    const previousJobId = resumesCheckpoint ? previousEntry.jobId : void 0;
+    const previousMetadata = resumesCheckpoint ? previousEntry.providerMetadata[provider.id] : void 0;
     const supersededOutputs = previousEntry?.outputs.length ? previousEntry.outputs : previousEntry?.supersededOutputs ?? [];
     upsert(lock, key, {
       styleId: spec.styleId,
@@ -6131,7 +6179,8 @@ async function submit(provider, loaded, items, lock, lockPath, opts = {}) {
         ...spec.revision.strength == null ? {} : { strength: spec.revision.strength }
       } : null,
       status: "pending",
-      jobId: null,
+      jobId: previousJobId ?? null,
+      ...previousJobId ? { submissionComplete: false } : { submissionComplete: void 0 },
       objectId: null,
       reviewObjectId: null,
       candidateIndex: null,
@@ -6140,10 +6189,10 @@ async function submit(provider, loaded, items, lock, lockPath, opts = {}) {
       // Keep the old ownership proof while new bytes are pending. Fetch may
       // replace that file only while its hash still matches this record.
       supersededOutputs,
-      providerMetadata: {},
+      providerMetadata: resumesCheckpoint ? previousEntry.providerMetadata : {},
       sourceUrl: null,
       sourceUrls: [],
-      submittedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      submittedAt: resumesCheckpoint ? previousEntry.submittedAt : (/* @__PURE__ */ new Date()).toISOString(),
       cost: estimate.amount,
       costUnit: estimate.unit,
       // Persist routing before the request starts. If the process stops while
@@ -6156,9 +6205,32 @@ async function submit(provider, loaded, items, lock, lockPath, opts = {}) {
     lastSubmitAt = Date.now();
     try {
       const refs = styleImages.get(spec.styleId) ?? [];
-      const { jobId, metadata } = await provider.submit(spec, refs);
+      const { jobId, metadata } = await provider.submit(spec, refs, {
+        ...previousJobId ? { previousJobId } : {},
+        ...previousMetadata ? { previousMetadata } : {},
+        checkpoint: async (checkpoint) => {
+          if (!checkpoint.jobId) throw new Error("Provider checkpoint returned an empty job id");
+          const entry = lock.entries[key];
+          upsert(lock, key, {
+            jobId: checkpoint.jobId,
+            submissionComplete: checkpoint.complete,
+            reviewObjectId: checkpoint.complete && (spec.generator === "frames" || estimate.candidates > 1 && !spec.tileFeature) ? checkpoint.jobId : null,
+            status: checkpoint.complete ? "processing" : "pending",
+            error: null,
+            providerMetadata: checkpoint.metadata ? {
+              ...entry.providerMetadata,
+              [provider.id]: {
+                ...entry.providerMetadata[provider.id],
+                ...checkpoint.metadata
+              }
+            } : entry.providerMetadata
+          });
+          await saveLock(lockPath, lock);
+        }
+      });
       upsert(lock, key, {
         jobId,
+        submissionComplete: true,
         // A multi-candidate generator routes through review; record the parent
         // so `pick` knows where to look.
         reviewObjectId: spec.generator === "frames" || estimate.candidates > 1 && !spec.tileFeature ? jobId : null,
@@ -6172,10 +6244,23 @@ async function submit(provider, loaded, items, lock, lockPath, opts = {}) {
         `  ${key} \u2192 ${jobId}  (${spec.width}x${spec.height}` + (estimate.candidates > 1 ? `, ${estimate.candidates} ${spec.tileFeature ? "outputs" : "candidates"}` : "") + `, ${estimate.amount})`
       );
     } catch (err) {
-      failed++;
       const message5 = err instanceof Error ? err.message : String(err);
-      upsert(lock, key, { status: "failed", error: message5, cost: 0 });
-      log2(`  FAILED ${key}: ${message5}`);
+      const entry = lock.entries[key];
+      if (entry.jobId && entry.submissionComplete === true) {
+        upsert(lock, key, { status: "processing", error: null });
+        inFlight.set(entry.jobId, spec);
+        submitted++;
+        spent += estimate.amount;
+        log2(`  ${key} \u2192 ${entry.jobId}  (recovered from completed checkpoint)`);
+      } else {
+        failed++;
+        upsert(lock, key, {
+          status: "failed",
+          error: message5,
+          cost: entry.jobId ? estimate.amount : 0
+        });
+        log2(`  FAILED ${key}: ${message5}`);
+      }
     }
     await saveLock(lockPath, lock);
   }
@@ -6193,12 +6278,19 @@ async function poll(provider, lock, lockPath, opts = {}) {
   const specByKey = new Map((opts.specs ?? []).map((s) => [lockKey(s.styleId, s.assetId), s]));
   const selectedKeys = opts.specs ? new Set(specByKey.keys()) : null;
   const result = { review: 0, completed: 0, failed: 0, stillRunning: 0 };
+  const incompleteSubmissions = Object.entries(lock.entries).filter(
+    ([key, entry]) => entry.provider === provider.id && (!selectedKeys || selectedKeys.has(key)) && entry.jobId && entry.submissionComplete === false && (entry.status === "pending" || entry.status === "processing")
+  );
+  for (const [key] of incompleteSubmissions) {
+    log2(`  incomplete submission ${key}; rerun \`pixelkiln submit\` to resume`);
+  }
+  result.stillRunning = incompleteSubmissions.length;
   const pending = () => Object.entries(lock.entries).filter(
-    ([key, e]) => e.provider === provider.id && (!selectedKeys || selectedKeys.has(key)) && e.jobId && (e.status === "pending" || e.status === "processing")
+    ([key, e]) => e.provider === provider.id && (!selectedKeys || selectedKeys.has(key)) && e.jobId && e.submissionComplete !== false && (e.status === "pending" || e.status === "processing")
   );
   while (pending().length > 0) {
     if (Date.now() - started > timeout) {
-      result.stillRunning = pending().length;
+      result.stillRunning += pending().length;
       log2(`  timed out with ${result.stillRunning} job(s) still running \u2014 re-run \`poll\` to resume`);
       break;
     }