windmill-client 1.733.1 → 1.734.0

package/dist/client.d.ts

@@ -1,6 +1,6 @@
 import { DenoS3LightClientSettings, S3ObjectRecord, type S3Object } from "./s3Types";
 export { type S3Object, type S3ObjectRecord, type S3ObjectURI, } from "./s3Types";
-export { datatable, ducklake, type SqlTemplateFunction, type DatatableSqlTemplateFunction, } from "./sqlUtils";
+export { datatable, ducklake, upsertPartition, appendPartition, type SqlTemplateFunction, type DatatableSqlTemplateFunction, type DucklakeMaterializeOptions, type SqlStatement, } from "./sqlUtils";
 export type Sql = string;
 export type Email = string;
 export type Base64 = string;
@@ -13,6 +13,7 @@ export declare function workerHasInternalServer(): boolean;
  * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable)
  */
 export declare function setClient(token?: string, baseUrl?: string): void;
+export declare const getEnv: (key: string) => string | undefined;
 /**
  * Create a client configuration from env variables
  * @returns client configuration

package/dist/client.mjs

@@ -1,6 +1,6 @@
 import { OpenAPI } from "./core/OpenAPI.mjs";
 import { AppService, HelpersService, JobService, KafkaTriggerService, MetricsService, OidcService, ResourceService, UserService, VariableService } from "./services.gen.mjs";
-import { datatable, ducklake } from "./sqlUtils.mjs";
+import { appendPartition, datatable, ducklake, upsertPartition } from "./sqlUtils.mjs";
 
 //#region src/client.ts
 const SHARED_FOLDER = "/shared";
@@ -1346,4 +1346,4 @@ async function commitKafkaOffsets(triggerPath, topic, partition, offset) {
 }
 
 //#endregion
-export { SHARED_FOLDER, StepSuspend, WorkflowCtx, _workflowCtx, appendToResultStream, base64ToUint8Array, commitKafkaOffsets, databaseUrlFromResource, deleteS3File, denoS3LightClientSettings, getFlowUserState, getIdToken, getInternalState, getPresignedS3PublicUrl, getPresignedS3PublicUrls, getProgress, getResource, getResult, getResultMaybe, getResumeEndpoints, getResumeUrls, getRootJobId, getState, getStatePath, getVariable, getWorkspace, loadS3File, loadS3FileStream, parallel, parseS3Object, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, resolveDefaultResource, runFlow, runFlowAsync, runScript, runScriptAsync, runScriptByHash, runScriptByHashAsync, runScriptByPath, runScriptByPathAsync, setClient, setFlowUserState, setInternalState, setProgress, setResource, setState, setVariable, setWorkflowCtx, signS3Object, signS3Objects, sleep, step, streamResult, task, taskFlow, taskScript, uint8ArrayToBase64, usernameToEmail, waitForApproval, waitJob, workerHasInternalServer, workflow, writeS3File };
+export { SHARED_FOLDER, StepSuspend, WorkflowCtx, _workflowCtx, appendToResultStream, base64ToUint8Array, commitKafkaOffsets, databaseUrlFromResource, deleteS3File, denoS3LightClientSettings, getEnv, getFlowUserState, getIdToken, getInternalState, getPresignedS3PublicUrl, getPresignedS3PublicUrls, getProgress, getResource, getResult, getResultMaybe, getResumeEndpoints, getResumeUrls, getRootJobId, getState, getStatePath, getVariable, getWorkspace, loadS3File, loadS3FileStream, parallel, parseS3Object, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, resolveDefaultResource, runFlow, runFlowAsync, runScript, runScriptAsync, runScriptByHash, runScriptByHashAsync, runScriptByPath, runScriptByPathAsync, setClient, setFlowUserState, setInternalState, setProgress, setResource, setState, setVariable, setWorkflowCtx, signS3Object, signS3Objects, sleep, step, streamResult, task, taskFlow, taskScript, uint8ArrayToBase64, usernameToEmail, waitForApproval, waitJob, workerHasInternalServer, workflow, writeS3File };

package/dist/core/OpenAPI.mjs

@@ -29,7 +29,7 @@ const OpenAPI = {
 	PASSWORD: void 0,
 	TOKEN: getEnv("WM_TOKEN"),
 	USERNAME: void 0,
-	VERSION: "1.733.1",
+	VERSION: "1.734.0",
 	WITH_CREDENTIALS: true,
 	interceptors: {
 		request: new Interceptors(),

package/dist/index.d.ts

@@ -4,8 +4,8 @@ export { OpenAPI, type OpenAPIConfig } from './core/OpenAPI';
 export * from './services.gen';
 export * from './types.gen';
 export type { DenoS3LightClientSettings } from "./s3Types";
-export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, type TaskOptions, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";
-import { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, WorkflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, getWorkspace, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, parseS3Object, commitKafkaOffsets } from "./client";
+export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, type TaskOptions, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";
+import { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, WorkflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, getWorkspace, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, parseS3Object, commitKafkaOffsets } from "./client";
 import { AdminService, AuditService, FlowService, GranularAclService, GroupService, JobService, ResourceService, VariableService, ScriptService, ScheduleService, SettingsService, UserService, WorkspaceService, TeamsService } from "./services.gen";
 declare const wmill: {
     setClient: typeof setClient;
@@ -59,6 +59,8 @@ declare const wmill: {
     streamResult: typeof streamResult;
     datatable: typeof datatable;
     ducklake: typeof ducklake;
+    upsertPartition: typeof upsertPartition;
+    appendPartition: typeof appendPartition;
     SHARED_FOLDER: string;
     getWorkspace: typeof getWorkspace;
     getStatePath: typeof getStatePath;

package/dist/index.js

@@ -126,7 +126,7 @@ const OpenAPI = {
 	PASSWORD: void 0,
 	TOKEN: getEnv$1("WM_TOKEN"),
 	USERNAME: void 0,
-	VERSION: "1.733.1",
+	VERSION: "1.734.0",
 	WITH_CREDENTIALS: true,
 	interceptors: {
 		request: new Interceptors(),
@@ -15005,6 +15005,127 @@ function ducklake(name = "main") {
 	let { name: n, schema } = parseName(name);
 	return buildSqlTemplateFunction(ducklakeProvider(n, schema));
 }
+/** Idempotently materialize `selectSql` into a ducklake table for one
+* partition (or the whole table when `partition` is omitted) — the client-side
+* equivalent of the `// materialize` engine.
+* With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+* replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+* Safe to re-run for the same partition (backfill / failure-recovery).
+*
+* Returns a lazy statement — call `.execute()` to run it:
+* `await wmill.upsertPartition({ table, selectSql, partition }).execute()`. */
+function upsertPartition(opts) {
+	return finishMaterialize(buildUpsertStatement(opts), opts);
+}
+function buildUpsertStatement(opts) {
+	let { name: n, schema } = parseName(opts.ducklake ?? "main");
+	let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+	let pcol = sql.raw(opts.partitionCol ?? "_wm_partition");
+	let t = sql.raw(`dl.${opts.table}`);
+	let body = sql.raw(opts.selectSql);
+	if (opts.partition === void 0) {
+		if (opts.uniqueKey) {
+			let uk = sql.raw(opts.uniqueKey);
+			return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT * FROM (${body}) WHERE false;
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${uk} IN (SELECT ${uk} FROM (${body}));
+INSERT INTO ${t} SELECT * FROM (${body});
+COMMIT;`;
+		}
+		return sql`CREATE OR REPLACE TABLE ${t} AS SELECT * FROM (${body});`;
+	}
+	if (opts.uniqueKey) {
+		let uk = sql.raw(opts.uniqueKey);
+		return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${pcol} = ${opts.partition} AND ${uk} IN (SELECT ${uk} FROM (${body}));
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});
+COMMIT;`;
+	}
+	return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${pcol} = ${opts.partition};
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});
+COMMIT;`;
+}
+/** INSERT-only materialization (no dedup/replace) for append-only tables.
+* Re-running the same partition duplicates rows — use only for immutable
+* event-log sources.
+*
+* Returns a lazy statement — call `.execute()` to run it:
+* `await wmill.appendPartition({ table, selectSql, partition }).execute()`. */
+function appendPartition(opts) {
+	return finishMaterialize(buildAppendStatement(opts), opts);
+}
+function buildAppendStatement(opts) {
+	let { name: n, schema } = parseName(opts.ducklake ?? "main");
+	let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+	let pcol = sql.raw(opts.partitionCol ?? "_wm_partition");
+	let t = sql.raw(`dl.${opts.table}`);
+	let body = sql.raw(opts.selectSql);
+	if (opts.partition === void 0) return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT * FROM (${body}) WHERE false;
+INSERT INTO ${t} SELECT * FROM (${body});`;
+	return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});`;
+}
+function finishMaterialize(stmt, opts) {
+	if (getEnv("WM_PIPELINE") !== "true") return stmt;
+	let { name: n, schema } = parseName(opts.ducklake ?? "main");
+	let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+	let t = sql.raw(`dl.${opts.table}`);
+	let pcol = opts.partitionCol ?? "_wm_partition";
+	let where = opts.partition !== void 0 ? sql.raw(`WHERE ${pcol} = '${String(opts.partition).replace(/'/g, "''")}'`) : sql.raw("");
+	let summary = sql`SELECT (SELECT count(*) FROM ${t} ${where}) AS rows, (SELECT max(snapshot_id) FROM ducklake_snapshots('dl')) AS snapshot_id`;
+	let assetPath = schema ? `${n}/${schema}.${opts.table}` : `${n}/${opts.table}`;
+	let partition = opts.partition ?? "";
+	let run = async () => {
+		try {
+			await stmt.execute();
+		} catch (e) {
+			await recordMaterialization(assetPath, partition, "failed", null, null, String(e));
+			throw e;
+		}
+		let snapshot_id = null;
+		let row_count = null;
+		try {
+			let s = await summary.fetchOne();
+			snapshot_id = s?.snapshot_id ?? null;
+			row_count = s?.rows ?? null;
+		} catch {}
+		await recordMaterialization(assetPath, partition, "materialized", snapshot_id, row_count, null);
+	};
+	return {
+		...stmt,
+		execute: () => run(),
+		fetch: () => run(),
+		fetchOne: () => run(),
+		fetchOneScalar: () => run()
+	};
+}
+async function recordMaterialization(assetPath, partition, status, snapshot_id, row_count, error) {
+	try {
+		await fetch(`${OpenAPI.BASE}/w/${getWorkspace()}/assets/record_materialization`, {
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${OpenAPI.TOKEN}`
+			},
+			body: JSON.stringify({
+				asset_kind: "ducklake",
+				asset_path: assetPath,
+				partition,
+				status,
+				snapshot_id,
+				row_count,
+				job_id: getEnv("WM_JOB_ID") ?? null,
+				error
+			})
+		});
+	} catch {}
+}
 function inferSqlType(value) {
 	if (typeof value === "bigint") return "BIGINT";
 	if (typeof value === "number") {
@@ -16449,6 +16570,8 @@ const wmill = {
 	streamResult,
 	datatable,
 	ducklake,
+	upsertPartition,
+	appendPartition,
 	SHARED_FOLDER,
 	getWorkspace,
 	getStatePath,
@@ -16552,6 +16675,7 @@ Object.defineProperty(exports, '_workflowCtx', {
     return _workflowCtx;
   }
 });
+exports.appendPartition = appendPartition;
 exports.appendToResultStream = appendToResultStream;
 exports.commitKafkaOffsets = commitKafkaOffsets;
 exports.datatable = datatable;
@@ -16597,6 +16721,7 @@ exports.streamResult = streamResult;
 exports.task = task;
 exports.taskFlow = taskFlow;
 exports.taskScript = taskScript;
+exports.upsertPartition = upsertPartition;
 exports.usernameToEmail = usernameToEmail;
 exports.waitForApproval = waitForApproval;
 exports.waitJob = waitJob;

package/dist/index.mjs

@@ -2,7 +2,7 @@ import { ApiError } from "./core/ApiError.mjs";
 import { CancelError, CancelablePromise } from "./core/CancelablePromise.mjs";
 import { OpenAPI } from "./core/OpenAPI.mjs";
 import { AdminService, AgentWorkersService, AppService, AssetService, AuditService, AzureTriggerService, CaptureService, ConcurrencyGroupsService, ConfigService, DocumentationService, DraftService, EmailTriggerService, FavoriteService, FlowConversationsService, FlowService, FolderService, GcpTriggerService, GitSyncService, GranularAclService, GroupService, HealthService, HelpersService, HttpTriggerService, IndexSearchService, InputService, IntegrationService, JobService, KafkaTriggerService, McpOauthService, McpService, MetricsService, MqttTriggerService, NativeTriggerService, NatsTriggerService, NpmProxyService, OauthService, OidcService, OpenapiService, PathAutocompleteService, PostgresTriggerService, RawAppService, ResourceService, ScheduleService, ScriptService, ServiceLogsService, SettingService, SettingsService, SqsTriggerService, TeamsService, TokenService, TriggerService, UserService, VariableService, VolumeService, WebsocketTriggerService, WorkerService, WorkspaceDependenciesService, WorkspaceIntegrationService, WorkspaceService } from "./services.gen.mjs";
-import { datatable, ducklake } from "./sqlUtils.mjs";
+import { appendPartition, datatable, ducklake, upsertPartition } from "./sqlUtils.mjs";
 import { SHARED_FOLDER, StepSuspend, WorkflowCtx, _workflowCtx, appendToResultStream, base64ToUint8Array, commitKafkaOffsets, databaseUrlFromResource, deleteS3File, denoS3LightClientSettings, getFlowUserState, getIdToken, getInternalState, getPresignedS3PublicUrl, getPresignedS3PublicUrls, getProgress, getResource, getResult, getResultMaybe, getResumeEndpoints, getResumeUrls, getRootJobId, getState, getStatePath, getVariable, getWorkspace, loadS3File, loadS3FileStream, parallel, parseS3Object, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, resolveDefaultResource, runFlow, runFlowAsync, runScript, runScriptAsync, runScriptByHash, runScriptByHashAsync, runScriptByPath, runScriptByPathAsync, setClient, setFlowUserState, setInternalState, setProgress, setResource, setState, setVariable, setWorkflowCtx, signS3Object, signS3Objects, sleep, step, streamResult, task, taskFlow, taskScript, uint8ArrayToBase64, usernameToEmail, waitForApproval, waitJob, workflow, writeS3File } from "./client.mjs";
 
 //#region src/index.ts
@@ -58,6 +58,8 @@ const wmill = {
 	streamResult,
 	datatable,
 	ducklake,
+	upsertPartition,
+	appendPartition,
 	SHARED_FOLDER,
 	getWorkspace,
 	getStatePath,
@@ -90,4 +92,4 @@ const wmill = {
 var src_default = wmill;
 
 //#endregion
-export { AdminService, AgentWorkersService, ApiError, AppService, AssetService, AuditService, AzureTriggerService, CancelError, CancelablePromise, CaptureService, ConcurrencyGroupsService, ConfigService, DocumentationService, DraftService, EmailTriggerService, FavoriteService, FlowConversationsService, FlowService, FolderService, GcpTriggerService, GitSyncService, GranularAclService, GroupService, HealthService, HelpersService, HttpTriggerService, IndexSearchService, InputService, IntegrationService, JobService, KafkaTriggerService, McpOauthService, McpService, MetricsService, MqttTriggerService, NativeTriggerService, NatsTriggerService, NpmProxyService, OauthService, OidcService, OpenAPI, OpenapiService, PathAutocompleteService, PostgresTriggerService, RawAppService, ResourceService, ScheduleService, ScriptService, ServiceLogsService, SettingService, SettingsService, SqsTriggerService, StepSuspend, TeamsService, TokenService, TriggerService, UserService, VariableService, VolumeService, WebsocketTriggerService, WorkerService, WorkflowCtx, WorkspaceDependenciesService, WorkspaceIntegrationService, WorkspaceService, _workflowCtx, appendToResultStream, commitKafkaOffsets, datatable, src_default as default, deleteS3File, denoS3LightClientSettings, ducklake, getFlowUserState, getIdToken, getPresignedS3PublicUrl, getPresignedS3PublicUrls, getProgress, getResource, getResumeUrls, getRootJobId, getState, getVariable, loadS3File, loadS3FileStream, parallel, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, runFlow, runFlowAsync, runScript, runScriptAsync, runScriptByHash, runScriptByHashAsync, runScriptByPath, runScriptByPathAsync, setClient, setFlowUserState, setProgress, setResource, setState, setVariable, setWorkflowCtx, signS3Object, signS3Objects, sleep, step, streamResult, task, taskFlow, taskScript, usernameToEmail, waitForApproval, waitJob, workflow, writeS3File };
+export { AdminService, AgentWorkersService, ApiError, AppService, AssetService, AuditService, AzureTriggerService, CancelError, CancelablePromise, CaptureService, ConcurrencyGroupsService, ConfigService, DocumentationService, DraftService, EmailTriggerService, FavoriteService, FlowConversationsService, FlowService, FolderService, GcpTriggerService, GitSyncService, GranularAclService, GroupService, HealthService, HelpersService, HttpTriggerService, IndexSearchService, InputService, IntegrationService, JobService, KafkaTriggerService, McpOauthService, McpService, MetricsService, MqttTriggerService, NativeTriggerService, NatsTriggerService, NpmProxyService, OauthService, OidcService, OpenAPI, OpenapiService, PathAutocompleteService, PostgresTriggerService, RawAppService, ResourceService, ScheduleService, ScriptService, ServiceLogsService, SettingService, SettingsService, SqsTriggerService, StepSuspend, TeamsService, TokenService, TriggerService, UserService, VariableService, VolumeService, WebsocketTriggerService, WorkerService, WorkflowCtx, WorkspaceDependenciesService, WorkspaceIntegrationService, WorkspaceService, _workflowCtx, appendPartition, appendToResultStream, commitKafkaOffsets, datatable, src_default as default, deleteS3File, denoS3LightClientSettings, ducklake, getFlowUserState, getIdToken, getPresignedS3PublicUrl, getPresignedS3PublicUrls, getProgress, getResource, getResumeUrls, getRootJobId, getState, getVariable, loadS3File, loadS3FileStream, parallel, requestInteractiveSlackApproval, requestInteractiveTeamsApproval, runFlow, runFlowAsync, runScript, runScriptAsync, runScriptByHash, runScriptByHashAsync, runScriptByPath, runScriptByPathAsync, setClient, setFlowUserState, setProgress, setResource, setState, setVariable, setWorkflowCtx, signS3Object, signS3Objects, sleep, step, streamResult, task, taskFlow, taskScript, upsertPartition, usernameToEmail, waitForApproval, waitJob, workflow, writeS3File };

package/dist/sqlUtils.d.ts

@@ -86,4 +86,39 @@ export declare function datatable(name?: string): DatatableSqlTemplateFunction;
  * let sql = wmill.ducklake("my_lake:analytics")
  */
 export declare function ducklake(name?: string): SqlTemplateFunction;
+/** Options for the ducklake materialization helpers. `partition` is bound as a
+ * DuckDB arg (never interpolated); `selectSql`, `table`, `schema`, `uniqueKey`
+ * are trusted structural SQL inlined via `raw`. */
+export interface DucklakeMaterializeOptions {
+    /** ducklake name (default "main"), optionally "name:schema". */
+    ducklake?: string;
+    /** target table within the ducklake. */
+    table: string;
+    /** the SELECT producing the rows for this slice. */
+    selectSql: string;
+    /** the partition value (bound). Omit for a whole-table materialization — no
+     * partition column, and replace becomes a `CREATE OR REPLACE TABLE`. */
+    partition?: string;
+    /** dedup key → upsert in slice (delete-by-key + insert); omit → replace (delete partition + insert). */
+    uniqueKey?: string;
+    /** physical partition column (default "_wm_partition"). */
+    partitionCol?: string;
+}
+/** Idempotently materialize `selectSql` into a ducklake table for one
+ * partition (or the whole table when `partition` is omitted) — the client-side
+ * equivalent of the `// materialize` engine.
+ * With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+ * replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+ * Safe to re-run for the same partition (backfill / failure-recovery).
+ *
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.upsertPartition({ table, selectSql, partition }).execute()`. */
+export declare function upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement<any>;
+/** INSERT-only materialization (no dedup/replace) for append-only tables.
+ * Re-running the same partition duplicates rows — use only for immutable
+ * event-log sources.
+ *
+ * Returns a lazy statement — call `.execute()` to run it:
+ * `await wmill.appendPartition({ table, selectSql, partition }).execute()`. */
+export declare function appendPartition(opts: Omit<DucklakeMaterializeOptions, "uniqueKey">): SqlStatement<any>;
 export {};

package/dist/sqlUtils.mjs

@@ -1,5 +1,6 @@
+import { OpenAPI } from "./core/OpenAPI.mjs";
 import { JobService } from "./services.gen.mjs";
-import { getWorkspace, workerHasInternalServer } from "./client.mjs";
+import { getEnv, getWorkspace, workerHasInternalServer } from "./client.mjs";
 
 //#region src/sqlUtils.ts
 /**
@@ -33,7 +34,7 @@ function ducklakeProvider(name, schema) {
 	};
 }
 function buildSqlStatement(provider, content, contentBody, args) {
-	async function fetch({ resultCollection } = {}) {
+	async function fetch$1({ resultCollection } = {}) {
 		let finalContent = content;
 		if (resultCollection) finalContent = `-- result_collection=${resultCollection}\n${finalContent}`;
 		try {
@@ -72,16 +73,16 @@ function buildSqlStatement(provider, content, contentBody, args) {
 	return {
 		content,
 		args,
-		fetch,
-		fetchOne: (params) => fetch({
+		fetch: fetch$1,
+		fetchOne: (params) => fetch$1({
 			...params,
 			resultCollection: "last_statement_first_row"
 		}),
-		fetchOneScalar: (params) => fetch({
+		fetchOneScalar: (params) => fetch$1({
 			...params,
 			resultCollection: "last_statement_first_row_scalar"
 		}),
-		execute: (params) => fetch(params)
+		execute: (params) => fetch$1(params)
 	};
 }
 function serializeArgValue(v) {
@@ -188,6 +189,127 @@ function ducklake(name = "main") {
 	let { name: n, schema } = parseName(name);
 	return buildSqlTemplateFunction(ducklakeProvider(n, schema));
 }
+/** Idempotently materialize `selectSql` into a ducklake table for one
+* partition (or the whole table when `partition` is omitted) — the client-side
+* equivalent of the `// materialize` engine.
+* With `uniqueKey` it upserts the slice (delete-by-key + insert); otherwise it
+* replaces it (whole table → `CREATE OR REPLACE`; partition → delete + insert).
+* Safe to re-run for the same partition (backfill / failure-recovery).
+*
+* Returns a lazy statement — call `.execute()` to run it:
+* `await wmill.upsertPartition({ table, selectSql, partition }).execute()`. */
+function upsertPartition(opts) {
+	return finishMaterialize(buildUpsertStatement(opts), opts);
+}
+function buildUpsertStatement(opts) {
+	let { name: n, schema } = parseName(opts.ducklake ?? "main");
+	let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+	let pcol = sql.raw(opts.partitionCol ?? "_wm_partition");
+	let t = sql.raw(`dl.${opts.table}`);
+	let body = sql.raw(opts.selectSql);
+	if (opts.partition === void 0) {
+		if (opts.uniqueKey) {
+			let uk = sql.raw(opts.uniqueKey);
+			return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT * FROM (${body}) WHERE false;
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${uk} IN (SELECT ${uk} FROM (${body}));
+INSERT INTO ${t} SELECT * FROM (${body});
+COMMIT;`;
+		}
+		return sql`CREATE OR REPLACE TABLE ${t} AS SELECT * FROM (${body});`;
+	}
+	if (opts.uniqueKey) {
+		let uk = sql.raw(opts.uniqueKey);
+		return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${pcol} = ${opts.partition} AND ${uk} IN (SELECT ${uk} FROM (${body}));
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});
+COMMIT;`;
+	}
+	return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+BEGIN TRANSACTION;
+DELETE FROM ${t} WHERE ${pcol} = ${opts.partition};
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});
+COMMIT;`;
+}
+/** INSERT-only materialization (no dedup/replace) for append-only tables.
+* Re-running the same partition duplicates rows — use only for immutable
+* event-log sources.
+*
+* Returns a lazy statement — call `.execute()` to run it:
+* `await wmill.appendPartition({ table, selectSql, partition }).execute()`. */
+function appendPartition(opts) {
+	return finishMaterialize(buildAppendStatement(opts), opts);
+}
+function buildAppendStatement(opts) {
+	let { name: n, schema } = parseName(opts.ducklake ?? "main");
+	let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+	let pcol = sql.raw(opts.partitionCol ?? "_wm_partition");
+	let t = sql.raw(`dl.${opts.table}`);
+	let body = sql.raw(opts.selectSql);
+	if (opts.partition === void 0) return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT * FROM (${body}) WHERE false;
+INSERT INTO ${t} SELECT * FROM (${body});`;
+	return sql`CREATE TABLE IF NOT EXISTS ${t} AS SELECT *, CAST(NULL AS VARCHAR) AS ${pcol} FROM (${body}) WHERE false;
+ALTER TABLE ${t} SET PARTITIONED BY (${pcol});
+INSERT INTO ${t} SELECT *, ${opts.partition} AS ${pcol} FROM (${body});`;
+}
+function finishMaterialize(stmt, opts) {
+	if (getEnv("WM_PIPELINE") !== "true") return stmt;
+	let { name: n, schema } = parseName(opts.ducklake ?? "main");
+	let sql = buildSqlTemplateFunction(ducklakeProvider(n, schema));
+	let t = sql.raw(`dl.${opts.table}`);
+	let pcol = opts.partitionCol ?? "_wm_partition";
+	let where = opts.partition !== void 0 ? sql.raw(`WHERE ${pcol} = '${String(opts.partition).replace(/'/g, "''")}'`) : sql.raw("");
+	let summary = sql`SELECT (SELECT count(*) FROM ${t} ${where}) AS rows, (SELECT max(snapshot_id) FROM ducklake_snapshots('dl')) AS snapshot_id`;
+	let assetPath = schema ? `${n}/${schema}.${opts.table}` : `${n}/${opts.table}`;
+	let partition = opts.partition ?? "";
+	let run = async () => {
+		try {
+			await stmt.execute();
+		} catch (e) {
+			await recordMaterialization(assetPath, partition, "failed", null, null, String(e));
+			throw e;
+		}
+		let snapshot_id = null;
+		let row_count = null;
+		try {
+			let s = await summary.fetchOne();
+			snapshot_id = s?.snapshot_id ?? null;
+			row_count = s?.rows ?? null;
+		} catch {}
+		await recordMaterialization(assetPath, partition, "materialized", snapshot_id, row_count, null);
+	};
+	return {
+		...stmt,
+		execute: () => run(),
+		fetch: () => run(),
+		fetchOne: () => run(),
+		fetchOneScalar: () => run()
+	};
+}
+async function recordMaterialization(assetPath, partition, status, snapshot_id, row_count, error) {
+	try {
+		await fetch(`${OpenAPI.BASE}/w/${getWorkspace()}/assets/record_materialization`, {
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				authorization: `Bearer ${OpenAPI.TOKEN}`
+			},
+			body: JSON.stringify({
+				asset_kind: "ducklake",
+				asset_path: assetPath,
+				partition,
+				status,
+				snapshot_id,
+				row_count,
+				job_id: getEnv("WM_JOB_ID") ?? null,
+				error
+			})
+		});
+	} catch {}
+}
 function inferSqlType(value) {
 	if (typeof value === "bigint") return "BIGINT";
 	if (typeof value === "number") {
@@ -235,4 +357,4 @@ function parseName(name) {
 }
 
 //#endregion
-export { datatable, ducklake };
+export { appendPartition, datatable, ducklake, upsertPartition };

package/dist/types.gen.d.ts

@@ -1538,6 +1538,10 @@ export type Script = {
 export type NewScript = {
     path: string;
     parent_hash?: string;
+    /**
+     * When true, the backend resolves the parent to the current deployed head for this path within the transaction (ignoring parent_hash), instead of failing with a "lineage must be linear" error when the supplied parent_hash is stale.
+     */
+    auto_parent?: boolean;
     summary: string;
     description?: string;
     content: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "windmill-client",
   "description": "Windmill SDK client for browsers and Node.js",
-  "version": "1.733.1",
+  "version": "1.734.0",
   "author": "Ruben Fiszel",
   "license": "Apache 2.0",
   "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme",