windmill-client 1.253.2 → 1.253.7

package/dist/client.d.ts

@@ -1,4 +1,4 @@
-import { DenoS3LightClientSettings } from "./s3Types";
+import { DenoS3LightClientSettings, type S3Object } from "./s3Types";
 export { AdminService, AuditService, FlowService, GranularAclService, GroupService, JobService, ResourceService, VariableService, ScriptService, ScheduleService, SettingsService, UserService, WorkspaceService, } from "./index";
 export type Sql = string;
 export type Email = string;
@@ -68,6 +68,27 @@ export declare function getVariable(path: string): Promise<string>;
 export declare function setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise<void>;
 export declare function databaseUrlFromResource(path: string): Promise<string>;
 export declare function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise<DenoS3LightClientSettings>;
+/**
+ * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.
+ *
+ * ```typescript
+ * let fileContentStream = await wmill.loadS3File(inputFile)
+ * // if the file is a raw text file, it can be decoded and printed directly:
+ * const text = new TextDecoder().decode(fileContentStream)
+ * console.log(text);
+ * ```
+ */
+export declare function loadS3File(s3object: S3Object, s3ResourcePath: string | undefined): Promise<Uint8Array | undefined>;
+/**
+ * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.
+ *
+ * ```typescript
+ * const s3object = await writeS3File(s3Object, "Hello Windmill!")
+ * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8')
+ * console.log(fileContentAsUtf8Str)
+ * ```
+ */
+export declare function writeS3File(s3object: S3Object | undefined, fileContent: string | ReadableStream<Uint8Array>, fileExpiration: Date | undefined, s3ResourcePath: string | undefined): Promise<S3Object>;
 /**
  * Get URLs needed for resuming a flow after this step
  * @param approver approver name

package/dist/client.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uint8ArrayToBase64 = exports.base64ToUint8Array = exports.getIdToken = exports.getResumeEndpoints = exports.getResumeUrls = exports.denoS3LightClientSettings = exports.databaseUrlFromResource = exports.setVariable = exports.getVariable = exports.getState = exports.getInternalState = exports.setState = exports.setInternalState = exports.setResource = exports.getStatePath = exports.resolveDefaultResource = exports.getResource = exports.getWorkspace = exports.setClient = exports.SHARED_FOLDER = exports.WorkspaceService = exports.UserService = exports.SettingsService = exports.ScheduleService = exports.ScriptService = exports.VariableService = exports.ResourceService = exports.JobService = exports.GroupService = exports.GranularAclService = exports.FlowService = exports.AuditService = exports.AdminService = void 0;
+exports.uint8ArrayToBase64 = exports.base64ToUint8Array = exports.getIdToken = exports.getResumeEndpoints = exports.getResumeUrls = exports.writeS3File = exports.loadS3File = exports.denoS3LightClientSettings = exports.databaseUrlFromResource = exports.setVariable = exports.getVariable = exports.getState = exports.getInternalState = exports.setState = exports.setInternalState = exports.setResource = exports.getStatePath = exports.resolveDefaultResource = exports.getResource = exports.getWorkspace = exports.setClient = exports.SHARED_FOLDER = exports.WorkspaceService = exports.UserService = exports.SettingsService = exports.ScheduleService = exports.ScriptService = exports.VariableService = exports.ResourceService = exports.JobService = exports.GroupService = exports.GranularAclService = exports.FlowService = exports.AuditService = exports.AdminService = void 0;
 const index_1 = require("./index");
 const index_2 = require("./index");
 var index_3 = require("./index");
@@ -291,23 +291,153 @@ function denoS3LightClientSettings(s3_resource_path) {
     });
 }
 exports.denoS3LightClientSettings = denoS3LightClientSettings;
-// export async function loadS3File(
-//   s3object: S3Object,
-//   s3ResourcePath: string | undefined
-// ): Promise<Response> {
-//   const settings = await denoS3LightClientSettings(s3ResourcePath);
-//   const s3 = new S3Client(settings);
-//   return await s3.getObject(s3object.s3);
-// }
-// export async function writeS3File(
-//   s3object: S3Object,
-//   fileContent: ReadableStream<Uint8Array> | Uint8Array | string,
-//   s3ResourcePath: string | undefined
-// ): Promise<Response> {
-//   const settings = await denoS3LightClientSettings(s3ResourcePath);
-//   const s3 = new S3Client(settings);
-//   return await s3.putObject(s3object.s3, fileContent);
-// }
+/**
+ * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.
+ *
+ * ```typescript
+ * let fileContentStream = await wmill.loadS3File(inputFile)
+ * // if the file is a raw text file, it can be decoded and printed directly:
+ * const text = new TextDecoder().decode(fileContentStream)
+ * console.log(text);
+ * ```
+ */
+function loadS3File(s3object, s3ResourcePath) {
+    return __awaiter(this, void 0, void 0, function* () {
+        !clientSet && setClient();
+        let part_number = 0;
+        let file_total_size = undefined;
+        let fetch = function (controller) {
+            return __awaiter(this, void 0, void 0, function* () {
+                // console.log("fetching part", part_number)
+                if (part_number === undefined || part_number === null) {
+                    // console.log("finished, closing")
+                    controller.close();
+                    return;
+                }
+                let part_response = yield index_1.HelpersService.multipartFileDownload({
+                    workspace: getWorkspace(),
+                    requestBody: {
+                        file_key: s3object.s3,
+                        part_number: part_number,
+                        file_size: file_total_size,
+                        s3_resource_path: s3ResourcePath,
+                    }
+                });
+                if (part_response.part_content.length > 0) {
+                    // console.log("enqueueing part", part_number, part_response.part_content.length)
+                    let chunk = new Uint8Array(part_response.part_content.length);
+                    part_response.part_content.forEach((value, index) => {
+                        chunk[index] = value;
+                    });
+                    controller.enqueue(chunk);
+                }
+                part_number = part_response.next_part_number;
+                file_total_size = part_response.file_size;
+            });
+        };
+        let fileContentStream = new ReadableStream({
+            pull(controller) {
+                return __awaiter(this, void 0, void 0, function* () {
+                    yield fetch(controller);
+                });
+            }
+        });
+        // For now we read all the stream in here. In the future return the stream and let the users consume it as they wish
+        const reader = fileContentStream.getReader();
+        const chunks = [];
+        while (true) {
+            const { value: chunk, done } = yield reader.read();
+            if (done) {
+                break;
+            }
+            chunks.push(chunk);
+        }
+        let fileContentLength = 0;
+        chunks.forEach(item => {
+            fileContentLength += item.length;
+        });
+        let fileContent = new Uint8Array(fileContentLength);
+        let offset = 0;
+        chunks.forEach(chunk => {
+            fileContent.set(chunk, offset);
+            offset += chunk.length;
+        });
+        return fileContent;
+    });
+}
+exports.loadS3File = loadS3File;
+/**
+ * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource.
+ *
+ * ```typescript
+ * const s3object = await writeS3File(s3Object, "Hello Windmill!")
+ * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8')
+ * console.log(fileContentAsUtf8Str)
+ * ```
+ */
+function writeS3File(s3object, fileContent, fileExpiration, s3ResourcePath) {
+    return __awaiter(this, void 0, void 0, function* () {
+        !clientSet && setClient();
+        let fileContentStream;
+        if (typeof fileContent === 'string') {
+            fileContentStream = new Blob([fileContent], {
+                type: 'text/plain'
+            }).stream();
+        }
+        else {
+            fileContentStream = fileContent;
+        }
+        let path = s3object === null || s3object === void 0 ? void 0 : s3object.s3;
+        let upload_id = undefined;
+        let parts = [];
+        const reader = fileContentStream.getReader();
+        let { value: chunk, done: readerDone } = yield reader.read();
+        if (chunk === undefined || readerDone) {
+            throw Error('Error reading stream, no data read');
+        }
+        while (true) {
+            let { value: chunk_2, done: readerDone } = yield reader.read();
+            if (!readerDone && chunk_2 !== undefined && chunk.length <= 5 * 1024 * 1024) {
+                // AWS enforces part to be bigger than 5MB, so we accumulate bytes until we reach that limit before triggering the request to the BE
+                chunk = new Uint8Array([...chunk, ...chunk_2]);
+                continue;
+            }
+            try {
+                let response = yield index_1.HelpersService.multipartFileUpload({
+                    workspace: getWorkspace(),
+                    requestBody: {
+                        file_key: path,
+                        file_extension: undefined,
+                        part_content: Array.from(chunk),
+                        upload_id: upload_id,
+                        parts: parts,
+                        is_final: readerDone,
+                        cancel_upload: false,
+                        s3_resource_path: s3ResourcePath,
+                        file_expiration: fileExpiration === null || fileExpiration === void 0 ? void 0 : fileExpiration.toString(),
+                    }
+                });
+                path = response.file_key;
+                upload_id = response.upload_id;
+                parts = response.parts;
+                if (response.is_done) {
+                    break;
+                }
+                if (chunk_2 === undefined) {
+                    throw Error('File upload is not finished, yet there is no more data to stream. This is unexpected');
+                }
+                chunk = chunk_2;
+            }
+            catch (e) {
+                throw Error(`Unexpected error uploading data to S3 ${e}`);
+            }
+        }
+        return {
+            s3: path
+        };
+    });
+}
+exports.writeS3File = writeS3File;
 /**
  * Get URLs needed for resuming a flow after this step
  * @param approver approver name

package/dist/core/OpenAPI.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAPI = void 0;
 exports.OpenAPI = {
     BASE: '/api',
-    VERSION: '1.253.2',
+    VERSION: '1.253.7',
     WITH_CREDENTIALS: false,
     CREDENTIALS: 'include',
     TOKEN: undefined,

package/dist/core/request.js

@@ -192,7 +192,6 @@ const sendRequest = (config, options, url, body, formData, headers, onCancel) =>
         method: options.method,
         signal: controller.signal,
     };
-    request.referrerPolicy = "no-referrer";
     if (config.WITH_CREDENTIALS) {
         request.credentials = config.CREDENTIALS;
     }

package/dist/index.d.ts

@@ -175,4 +175,4 @@ export { VariableService } from './services/VariableService';
 export { WorkerService } from './services/WorkerService';
 export { WorkspaceService } from './services/WorkspaceService';
 export type { S3Object, DenoS3LightClientSettings } from "./s3Types";
-export { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings } from "./client";
+export { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3File, writeS3File } from "./client";

package/dist/index.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.UserService = exports.SettingsService = exports.SettingService = exports.ScriptService = exports.ScheduleService = exports.ResourceService = exports.RawAppService = exports.OidcService = exports.OauthService = exports.MetricsService = exports.JobService = exports.IntegrationService = exports.InputService = exports.HelpersService = exports.GroupService = exports.GranularAclService = exports.FolderService = exports.FlowService = exports.FavoriteService = exports.DraftService = exports.ConfigService = exports.ConcurrencyGroupsService = exports.CaptureService = exports.AuditService = exports.AppService = exports.AdminService = exports.WindmillFilePreview = exports.Script = exports.RunnableType = exports.RawScriptForDependencies = exports.RawScript = exports.QueuedJob = exports.Preview = exports.Policy = exports.NewScript = exports.MainArgSignature = exports.ListableApp = exports.LargeFileStorage = exports.Job = exports.Http = exports.GlobalUserInfo = exports.FlowStatusModule = exports.CompletedJob = exports.AuditLog = exports.AppWithLastVersion = exports.ActionKind = exports.OpenAPI = exports.CancelError = exports.CancelablePromise = exports.ApiError = void 0;
-exports.denoS3LightClientSettings = exports.getIdToken = exports.getState = exports.setState = exports.getResumeUrls = exports.setResource = exports.getResource = exports.setVariable = exports.getVariable = exports.setClient = exports.WorkspaceService = exports.WorkerService = exports.VariableService = void 0;
+exports.writeS3File = exports.loadS3File = exports.denoS3LightClientSettings = exports.getIdToken = exports.getState = exports.setState = exports.getResumeUrls = exports.setResource = exports.getResource = exports.setVariable = exports.getVariable = exports.setClient = exports.WorkspaceService = exports.WorkerService = exports.VariableService = void 0;
 /* generated using openapi-typescript-codegen -- do no edit */
 /* istanbul ignore file */
 /* tslint:disable */
@@ -122,3 +122,5 @@ Object.defineProperty(exports, "setState", { enumerable: true, get: function ()
 Object.defineProperty(exports, "getState", { enumerable: true, get: function () { return client_1.getState; } });
 Object.defineProperty(exports, "getIdToken", { enumerable: true, get: function () { return client_1.getIdToken; } });
 Object.defineProperty(exports, "denoS3LightClientSettings", { enumerable: true, get: function () { return client_1.denoS3LightClientSettings; } });
+Object.defineProperty(exports, "loadS3File", { enumerable: true, get: function () { return client_1.loadS3File; } });
+Object.defineProperty(exports, "writeS3File", { enumerable: true, get: function () { return client_1.writeS3File; } });

package/dist/models/LargeFileStorage.d.ts

@@ -1,6 +1,7 @@
 export type LargeFileStorage = {
     type?: LargeFileStorage.type;
     s3_resource_path?: string;
+    public_resource?: boolean;
 };
 export declare namespace LargeFileStorage {
     enum type {

package/dist/services/HelpersService.d.ts

@@ -123,6 +123,7 @@ export declare class HelpersService {
     }): CancelablePromise<{
         next_marker?: string;
         windmill_large_files: Array<WindmillLargeFile>;
+        restricted_access?: boolean;
     }>;
     /**
      * Load metadata of the file
@@ -148,6 +149,20 @@ export declare class HelpersService {
         readBytesFrom?: number;
         readBytesLength?: number;
     }): CancelablePromise<WindmillFilePreview>;
+    /**
+     * Load a preview of a parquet file
+     * @returns any Parquet Preview
+     * @throws ApiError
+     */
+    static loadParquetPreview({ workspace, path, offset, limit, sortCol, sortDesc, search, }: {
+        workspace: string;
+        path: string;
+        offset?: number;
+        limit?: number;
+        sortCol?: string;
+        sortDesc?: boolean;
+        search?: string;
+    }): CancelablePromise<any>;
     /**
      * Generate a unique URL to download the file
      * @returns any Download URL
@@ -197,6 +212,7 @@ export declare class HelpersService {
             is_final: boolean;
             cancel_upload: boolean;
             s3_resource_path?: string;
+            file_expiration?: string;
         };
     }): CancelablePromise<{
         upload_id: string;
@@ -204,4 +220,25 @@ export declare class HelpersService {
         is_done: boolean;
         file_key: string;
     }>;
+    /**
+     * Download file to S3 bucket
+     * @returns any Chunk of the downloaded file
+     * @throws ApiError
+     */
+    static multipartFileDownload({ workspace, requestBody, }: {
+        workspace: string;
+        /**
+         * Query args for a multipart file upload to S3
+         */
+        requestBody: {
+            file_key: string;
+            part_number: number;
+            file_size?: number;
+            s3_resource_path?: string;
+        };
+    }): CancelablePromise<{
+        file_size?: number;
+        part_content: Array<number>;
+        next_part_number?: number;
+    }>;
 }

package/dist/services/HelpersService.js

@@ -156,6 +156,28 @@ class HelpersService {
             },
         });
     }
+    /**
+     * Load a preview of a parquet file
+     * @returns any Parquet Preview
+     * @throws ApiError
+     */
+    static loadParquetPreview({ workspace, path, offset, limit, sortCol, sortDesc, search, }) {
+        return (0, request_1.request)(OpenAPI_1.OpenAPI, {
+            method: 'GET',
+            url: '/w/{workspace}/job_helpers/load_parquet_preview/{path}',
+            path: {
+                'workspace': workspace,
+                'path': path,
+            },
+            query: {
+                'offset': offset,
+                'limit': limit,
+                'sort_col': sortCol,
+                'sort_desc': sortDesc,
+                'search': search,
+            },
+        });
+    }
     /**
      * Generate a unique URL to download the file
      * @returns any Download URL
@@ -224,5 +246,21 @@ class HelpersService {
             mediaType: 'application/json',
         });
     }
+    /**
+     * Download file to S3 bucket
+     * @returns any Chunk of the downloaded file
+     * @throws ApiError
+     */
+    static multipartFileDownload({ workspace, requestBody, }) {
+        return (0, request_1.request)(OpenAPI_1.OpenAPI, {
+            method: 'POST',
+            url: '/w/{workspace}/job_helpers/multipart_download_s3_file',
+            path: {
+                'workspace': workspace,
+            },
+            body: requestBody,
+            mediaType: 'application/json',
+        });
+    }
 }
 exports.HelpersService = HelpersService;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "windmill-client",
   "description": "Windmill SDK client for browsers and Node.js",
-  "version": "1.253.2",
+  "version": "1.253.7",
   "author": "Ruben Fiszel",
   "license": "Apache 2.0",
   "devDependencies": {