windmill-client 1.805.0 → 1.806.0

package/dist/client.d.ts

@@ -499,6 +499,32 @@ type Flatten<T> = {
  * overloaded one keeps only its last signature. Neither survives JSON anyway.
  */
 export type JsonifiedFn<T extends (...args: any[]) => Promise<any>> = (...args: Parameters<T>) => Promise<Jsonified<Awaited<ReturnType<T>>>>;
+/** Re-dispatch policy for a failed task.
+ *
+ * Every attempt is a step of its own (`fetch`, `fetch#2`, `fetch#3`), and the
+ * wait between two of them is a durable sleep, so a retrying task holds no
+ * worker while it backs off.
+ *
+ * A workflow sleeps once per round, so tasks backing off in the same fan-out
+ * wait one after another rather than together: the delay before a fan-out
+ * retries is the sum of every backoff pending in it, not the longest one, and
+ * it grows with both the width of the fan-out and `attempts`. Retries with no
+ * `delay` all go out in a single round.
+ */
+export interface TaskRetry {
+    /** Attempts after the first failure: `2` runs the task at most 3 times.
+     *  A whole number from 0 to 100; anything else is rejected where the policy
+     *  is written. */
+    attempts: number;
+    /** Seconds to wait before the first retry. Default 0, retry immediately.
+     *  Sub-second delays are dropped — a durable sleep resolves to the second. */
+    delay?: number;
+    /** Applied to the delay after each attempt: 1 (the default) keeps it
+     *  constant, 2 doubles it. */
+    multiplier?: number;
+    /** Ceiling for the delay in seconds, for a `multiplier` above 1. */
+    max_delay?: number;
+}
 export interface TaskOptions {
     timeout?: number;
     tag?: string;
@@ -507,6 +533,7 @@ export interface TaskOptions {
     concurrency_limit?: number;
     concurrency_key?: string;
     concurrency_time_window_s?: number;
+    retry?: TaskRetry;
 }
 export declare let _workflowCtx: WorkflowCtx | null;
 export declare function setWorkflowCtx(ctx: WorkflowCtx | null): void;
@@ -529,6 +556,11 @@ export declare class WorkflowCtx {
      *  into a `complete` — the parent would then record the caught branch's value as
      *  a successful step. Boxed: the thrown value may be any falsy value. */
     private _pendingStepFailure;
+    /** Failed tasks whose rejection nothing has consumed, by step key. An unawaited
+     *  task is still dispatched and still fails, but nothing drives the rejecting
+     *  thenable it returned. The first `.then()` on that thenable drops the entry,
+     *  so what remains is only what the body never looked at. */
+    private _unobservedTaskFailures;
     /** When set, the task matching this key executes its inner function directly */
     _executingKey: string | null;
     /** Serializes fast-path POSTs across concurrent step() calls within one
@@ -548,6 +580,15 @@ export declare class WorkflowCtx {
      *  fixed by the workflow body, so replays reproduce the same keys. */
     _allocKey(name: string): string;
     _nextStep(name: string, script: string, args?: Record<string, any>, dispatch_type?: string, options?: TaskOptions): PromiseLike<any>;
+    /** Wait out the backoff between two attempts of a retried task, as a durable
+     *  sleep, and return once there is nothing to wait for — no delay configured,
+     *  or the sleep already in the checkpoint.
+     *
+     *  Raises where it stands, the way `_sleep` does, rather than handing back a
+     *  thenable: a task call the body never awaits is still dispatched (the runner
+     *  flushes `pending`), so a backoff that only fired when awaited would drop
+     *  the retry and let the round report the workflow complete. */
+    private _retryBackoff;
     /** Return and clear any pending (unawaited) steps. */
     _flushPending(): Array<{
         name: string;
@@ -587,6 +628,9 @@ export declare class WorkflowCtx {
     _takePendingStepFailure(): {
         error: unknown;
     } | null;
+    /** Report the task failures the body never looked at, and forget them. Which
+     *  rounds may call this is the runner's constraint, stated where it is enforced. */
+    _warnUnobservedTaskFailures(): void;
 }
 export declare function sleep(seconds: number): Promise<void>;
 /**
@@ -604,9 +648,11 @@ export declare function step<T>(name: string, fn: () => T | Promise<T>): Promise
  * @example
  * const extract_data = task(async (url: string) => { ... });
  * const run_external = task("f/external_script", async (x: number) => { ... });
+ * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
  *
  * Inside a `workflow()`, calling a task dispatches it as a step.
- * Outside a workflow, the function body executes directly.
+ * Outside a workflow, the function body executes directly and
+ * {@link TaskOptions} — retry included — does not apply.
  *
  * A task runs as its own job, so its result is always encoded as JSON and
  * decoded back before the caller sees it: a `Date` comes back as a string, a

package/dist/client.mjs

@@ -1008,6 +1008,35 @@ function checkpointableResult(value) {
 function jsonRoundTrip(value) {
 	return JSON.parse(encodeCheckpointPayload({ value: checkpointableResult(value) })).value;
 }
+/** The worker deserializes a sleep into a `u32` of seconds and fails the whole
+*  job on anything wider, so a delay a multiplier has run away with has to be
+*  capped here rather than sent. */
+const MAX_SLEEP_SECONDS = 4294967295;
+/** Every attempt claims its keys before the first one is dispatched, so an
+*  unbounded `attempts` is a workflow that hangs allocating rather than a very
+*  patient one. */
+const MAX_RETRY_ATTEMPTS = 100;
+/** Rejected where the policy is written, so a workflow fails at its first line
+*  rather than mid-run on a replay. */
+function assertUsableRetry(retry) {
+	if (retry === void 0) return;
+	const { attempts } = retry;
+	if (!Number.isInteger(attempts) || attempts < 0 || attempts > MAX_RETRY_ATTEMPTS) throw new Error(`retry.attempts must be a whole number between 0 and ${MAX_RETRY_ATTEMPTS}, got ${attempts}`);
+}
+/** How many retries the policy asks for, defended against a value that reached
+*  `_nextStep` without going through `assertUsableRetry`. */
+function retryAttempts(retry) {
+	const attempts = Math.trunc(retry?.attempts ?? 0) || 0;
+	return Math.min(Math.max(attempts, 0), MAX_RETRY_ATTEMPTS);
+}
+/** Seconds to wait before retry number `attempt` (0 is the first retry). */
+function retryDelaySeconds(retry, attempt) {
+	const base = retry.delay ?? 0;
+	if (!(base > 0)) return 0;
+	const grown = base * Math.pow(retry.multiplier ?? 1, attempt);
+	const seconds = Math.floor(Math.min(retry.max_delay ?? grown, grown, MAX_SLEEP_SECONDS));
+	return seconds > 0 ? seconds : 0;
+}
 /** A step key travels as one path segment when its URLs are minted, so it must be
 *  non-empty and free of `/` and dot segments — otherwise `waitForApproval` would
 *  accept a key `getApprovalUrls` can never address. */
@@ -1039,6 +1068,11 @@ var WorkflowCtx = class {
 	*  into a `complete` — the parent would then record the caught branch's value as
 	*  a successful step. Boxed: the thrown value may be any falsy value. */
 	_pendingStepFailure = null;
+	/** Failed tasks whose rejection nothing has consumed, by step key. An unawaited
+	*  task is still dispatched and still fails, but nothing drives the rejecting
+	*  thenable it returned. The first `.then()` on that thenable drops the entry,
+	*  so what remains is only what the body never looked at. */
+	_unobservedTaskFailures = new Map();
 	/** When set, the task matching this key executes its inner function directly */
 	_executingKey;
 	/** Serializes fast-path POSTs across concurrent step() calls within one
@@ -1072,52 +1106,90 @@ var WorkflowCtx = class {
 	}
 	_nextStep(name, script, args = {}, dispatch_type = "inline", options) {
 		this._rethrowSwallowed();
-		const key = this._allocKey(name || script || "step");
-		if (key in this.completed) {
-			const value = this.completed[key];
-			if (value && typeof value === "object" && value.__wmill_error) {
-				const err = taskErrorFromMarker(value, `Task '${name}' failed`);
-				return { then: (_resolve, reject) => {
-					if (reject) reject(err);
-					else throw err;
-				} };
+		const stepName = name || script || "step";
+		const maxRetries = retryAttempts(options?.retry);
+		const baseKey = this._allocKey(stepName);
+		const attemptKeys = [baseKey];
+		const backoffKeys = [];
+		for (let i = 0; i < maxRetries; i++) {
+			backoffKeys.push(this._allocKey(`${baseKey}#retry${i + 2}`));
+			attemptKeys.push(this._allocKey(`${baseKey}#${i + 2}`));
+		}
+		for (let attempt = 0;; attempt++) {
+			const key = attemptKeys[attempt];
+			if (key in this.completed) {
+				const value = this.completed[key];
+				if (value && typeof value === "object" && value.__wmill_error) {
+					if (attempt < maxRetries) {
+						this._retryBackoff(backoffKeys[attempt], baseKey, options.retry, attempt);
+						continue;
+					}
+					const err = taskErrorFromMarker(value, `Task '${name}' failed`);
+					this._unobservedTaskFailures.set(baseKey, err);
+					return { then: (_resolve, reject) => {
+						this._unobservedTaskFailures.delete(baseKey);
+						if (reject) reject(err);
+						else throw err;
+					} };
+				}
+				return { then: (resolve) => resolve(value) };
 			}
-			return { then: (resolve) => resolve(value) };
+			if (this._executingKey === key) return {
+				then: (resolve) => resolve(null),
+				_execute_directly: true
+			};
+			if (this._executingKey !== null) return { then: () => new Promise(() => {}) };
+			const stepInfo = {
+				name: name || key,
+				script: script || key,
+				args,
+				key,
+				dispatch_type
+			};
+			if (options) {
+				if (options.timeout !== void 0) stepInfo.timeout = options.timeout;
+				if (options.tag !== void 0) stepInfo.tag = options.tag;
+				if (options.cache_ttl !== void 0) stepInfo.cache_ttl = options.cache_ttl;
+				if (options.priority !== void 0) stepInfo.priority = options.priority;
+				if (options.concurrency_limit !== void 0) stepInfo.concurrent_limit = options.concurrency_limit;
+				if (options.concurrency_key !== void 0) stepInfo.concurrency_key = options.concurrency_key;
+				if (options.concurrency_time_window_s !== void 0) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s;
+			}
+			this.pending.push(stepInfo);
+			return { then: () => {
+				if (this._suspended) return new Promise(() => {});
+				this._suspended = true;
+				const steps = [...this.pending];
+				this.pending = [];
+				const names = steps.map((s) => s.name).join(", ");
+				console.log(`\n--- WAC: ${names} ---`);
+				this._raiseSuspend({
+					mode: steps.length > 1 ? "parallel" : "sequential",
+					steps
+				});
+			} };
 		}
-		if (this._executingKey === key) return {
-			then: (resolve) => resolve(null),
-			_execute_directly: true
-		};
-		if (this._executingKey !== null) return { then: () => new Promise(() => {}) };
-		const stepInfo = {
-			name: name || key,
-			script: script || key,
-			args,
+	}
+	/** Wait out the backoff between two attempts of a retried task, as a durable
+	*  sleep, and return once there is nothing to wait for — no delay configured,
+	*  or the sleep already in the checkpoint.
+	*
+	*  Raises where it stands, the way `_sleep` does, rather than handing back a
+	*  thenable: a task call the body never awaits is still dispatched (the runner
+	*  flushes `pending`), so a backoff that only fired when awaited would drop
+	*  the retry and let the round report the workflow complete. */
+	_retryBackoff(key, baseKey, retry, attempt) {
+		const seconds = retryDelaySeconds(retry, attempt);
+		if (seconds < 1) return;
+		if (key in this.completed) return;
+		if (this._executingKey !== null) return;
+		console.log(`\n--- WAC: sleep(${key}, ${seconds}s) before retrying ${baseKey} ---`);
+		this._raiseSuspend({
+			mode: "sleep",
 			key,
-			dispatch_type
-		};
-		if (options) {
-			if (options.timeout !== void 0) stepInfo.timeout = options.timeout;
-			if (options.tag !== void 0) stepInfo.tag = options.tag;
-			if (options.cache_ttl !== void 0) stepInfo.cache_ttl = options.cache_ttl;
-			if (options.priority !== void 0) stepInfo.priority = options.priority;
-			if (options.concurrency_limit !== void 0) stepInfo.concurrent_limit = options.concurrency_limit;
-			if (options.concurrency_key !== void 0) stepInfo.concurrency_key = options.concurrency_key;
-			if (options.concurrency_time_window_s !== void 0) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s;
-		}
-		this.pending.push(stepInfo);
-		return { then: () => {
-			if (this._suspended) return new Promise(() => {});
-			this._suspended = true;
-			const steps = [...this.pending];
-			this.pending = [];
-			const names = steps.map((s) => s.name).join(", ");
-			console.log(`\n--- WAC: ${names} ---`);
-			this._raiseSuspend({
-				mode: steps.length > 1 ? "parallel" : "sequential",
-				steps
-			});
-		} };
+			seconds,
+			steps: []
+		});
 	}
 	/** Return and clear any pending (unawaited) steps. */
 	_flushPending() {
@@ -1284,6 +1356,13 @@ var WorkflowCtx = class {
 		this._pendingStepFailure = null;
 		return f;
 	}
+	/** Report the task failures the body never looked at, and forget them. Which
+	*  rounds may call this is the runner's constraint, stated where it is enforced. */
+	_warnUnobservedTaskFailures() {
+		if (this._executingKey !== null) return;
+		for (const [key, err] of this._unobservedTaskFailures) console.log(`\n--- WAC: task '${key}' failed but was never awaited, so the workflow result does not reflect it: ${err.message} ---`);
+		this._unobservedTaskFailures.clear();
+	}
 };
 async function sleep(seconds) {
 	const ctx = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx");
@@ -1309,9 +1388,11 @@ async function step(name, fn) {
 * @example
 * const extract_data = task(async (url: string) => { ... });
 * const run_external = task("f/external_script", async (x: number) => { ... });
+* const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } });
 *
 * Inside a `workflow()`, calling a task dispatches it as a step.
-* Outside a workflow, the function body executes directly.
+* Outside a workflow, the function body executes directly and
+* {@link TaskOptions} — retry included — does not apply.
 *
 * A task runs as its own job, so its result is always encoded as JSON and
 * decoded back before the caller sees it: a `Date` comes back as a string, a
@@ -1329,6 +1410,7 @@ function task(fnOrPath, maybeFnOrOptions, maybeOptions) {
 		fn = fnOrPath;
 		taskOptions = maybeFnOrOptions;
 	}
+	assertUsableRetry(taskOptions?.retry);
 	const taskName = fn.name || taskPath || "";
 	const wrapper = function(...args) {
 		const ctx = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx");
@@ -1387,6 +1469,7 @@ function task(fnOrPath, maybeFnOrOptions, maybeOptions) {
 * // inside workflow: await extract({ url: "https://..." })
 */
 function taskScript(path, options) {
+	assertUsableRetry(options?.retry);
 	const name = path.split("/").pop() || path;
 	const wrapper = function(...args) {
 		const ctx = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx");
@@ -1412,6 +1495,7 @@ function taskScript(path, options) {
 * // inside workflow: await pipeline({ input: data })
 */
 function taskFlow(path, options) {
+	assertUsableRetry(options?.retry);
 	const name = path.split("/").pop() || path;
 	const wrapper = function(...args) {
 		const ctx = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx");

package/dist/core/OpenAPI.mjs

@@ -29,7 +29,7 @@ const OpenAPI = {
 	PASSWORD: void 0,
 	TOKEN: getEnv("WM_TOKEN"),
 	USERNAME: void 0,
-	VERSION: "1.805.0",
+	VERSION: "1.806.0",
 	WITH_CREDENTIALS: !getEnv("WM_RAW_APP"),
 	interceptors: {
 		request: new Interceptors(),

package/dist/index.d.ts

@@ -4,7 +4,7 @@ export { OpenAPI, type OpenAPIConfig } from './core/OpenAPI';
 export * from './services.gen';
 export * from './types.gen';
 export type { DenoS3LightClientSettings } from "./s3Types";
-export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";
+export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";
 import { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, WorkflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, getWorkspace, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, parseS3Object, commitKafkaOffsets } from "./client";
 import { AdminService, AuditService, FlowService, GranularAclService, GroupService, JobService, ResourceService, VariableService, ScriptService, ScheduleService, SettingsService, UserService, WorkspaceService, TeamsService } from "./services.gen";
 declare const wmill: {