windmill-cli 1.586.0 → 1.586.1

package/esm/src/commands/app/raw_apps.js

@@ -0,0 +1,212 @@
+// deno-lint-ignore-file no-explicit-any
+import * as dntShim from "../../../_dnt.shims.js";
+import { requireLogin } from "../../core/auth.js";
+import { resolveWorkspace, validatePath } from "../../core/context.js";
+import { colors, log, SEP, windmillUtils, yamlParseFile, } from "../../../deps.js";
+import * as wmill from "../../../gen/services.gen.js";
+import { isSuperset } from "../../types.js";
+import { replaceInlineScripts, repopulateFields } from "./apps.js";
+import { createBundle, detectFrameworks } from "./bundle.js";
+import { mergeConfigWithConfigFile } from "../../core/conf.js";
+const alreadySynced = [];
+async function collectAppFiles(localPath) {
+    const files = {};
+    async function readDirRecursive(dir, basePath = "/") {
+        for await (const entry of dntShim.Deno.readDir(dir)) {
+            const fullPath = dir + entry.name;
+            const relativePath = basePath + entry.name;
+            if (entry.isDirectory) {
+                // Skip the runnables and node_modules subfolders
+                if (entry.name === "runnables" || entry.name === "node_modules" || entry.name === "dist" || entry.name === ".claude") {
+                    continue;
+                }
+                await readDirRecursive(fullPath + SEP, relativePath + SEP);
+            }
+            else if (entry.isFile) {
+                // Skip raw_app.yaml as it's metadata, not an app file
+                // Skip node_modules and package-lock.json as they are generated
+                if (relativePath === "raw_app.yaml" ||
+                    relativePath === "package-lock.json") {
+                    continue;
+                }
+                const content = await dntShim.Deno.readTextFile(fullPath);
+                files[relativePath] = content;
+            }
+        }
+    }
+    await readDirRecursive(localPath);
+    return files;
+}
+export async function pushRawApp(workspace, remotePath, localPath, message) {
+    if (alreadySynced.includes(localPath)) {
+        return;
+    }
+    alreadySynced.push(localPath);
+    remotePath = remotePath.replaceAll(SEP, "/");
+    let app = undefined;
+    // deleting old app if it exists in raw mode
+    try {
+        app = await wmill.getAppByPath({
+            workspace,
+            path: remotePath,
+        });
+    }
+    catch {
+        //ignore
+    }
+    if (app?.["policy"]?.["execution_mode"] == "anonymous") {
+        app.public = true;
+    }
+    // console.log(app);
+    if (app) {
+        app.policy = undefined;
+    }
+    if (!localPath.endsWith(SEP)) {
+        localPath += SEP;
+    }
+    const path = localPath + "raw_app.yaml";
+    const localApp = (await yamlParseFile(path));
+    replaceInlineScripts(localApp.runnables, localPath + SEP + "runnables/");
+    repopulateFields(localApp.runnables);
+    await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false);
+    const files = await collectAppFiles(localPath);
+    async function createBundleRaw() {
+        log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`));
+        // Detect frameworks to determine entry point
+        const frameworks = detectFrameworks(localPath);
+        const entryFile = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx";
+        const entryPoint = localPath + entryFile;
+        return await createBundle({
+            entryPoint: entryPoint,
+            production: true,
+            minify: true,
+        });
+    }
+    if (app) {
+        if (isSuperset(localApp, app)) {
+            log.info(colors.green(`App ${remotePath} is up to date`));
+            return;
+        }
+        const { js, css } = await createBundleRaw();
+        log.info(colors.bold.yellow(`Updating app ${remotePath}...`));
+        await wmill.updateAppRaw({
+            workspace,
+            path: remotePath,
+            formData: {
+                app: {
+                    value: { runnables: localApp.runnables, files },
+                    path: remotePath,
+                    summary: localApp.summary,
+                    policy: localApp.policy,
+                    deployment_message: message,
+                    custom_path: localApp.custom_path,
+                },
+                js,
+                css,
+            },
+        });
+    }
+    else {
+        const { js, css } = await createBundleRaw();
+        await wmill.createAppRaw({
+            workspace,
+            formData: {
+                app: {
+                    value: { runnables: localApp.runnables, files },
+                    path: remotePath,
+                    summary: localApp.summary,
+                    policy: localApp.policy,
+                    deployment_message: message,
+                    custom_path: localApp.custom_path,
+                },
+                js,
+                css,
+            },
+        });
+        // await wmill.createApp({
+        //   workspace,
+        //   requestBody: {
+        //     path: remotePath,
+        //     deployment_message: message,
+        //     value: { runnables: localApp.runnables, files },
+        //     summary: localApp.summary,
+        //     policy: localApp.policy,
+        //   },
+        // });
+    }
+}
+export async function generatingPolicy(app, path, publicApp) {
+    log.info(colors.gray(`Generating fresh policy for app ${path}...`));
+    try {
+        app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, app.policy);
+        app.policy.execution_mode = publicApp ? "anonymous" : "publisher";
+    }
+    catch (e) {
+        log.error(colors.red(`Error generating policy for app ${path}: ${e}`));
+        throw e;
+    }
+}
+export async function generateLocksCommand(opts, appPath) {
+    const { generateAppLocksInternal } = await import("./app_metadata.js");
+    const { elementsToMap, FSFSElement } = await import("../sync/sync.js");
+    const { ignoreF } = await import("../sync/sync.js");
+    const { Confirm } = await import("../../../deps.js");
+    if (appPath == "") {
+        appPath = undefined;
+    }
+    const workspace = await resolveWorkspace(opts);
+    await requireLogin(opts);
+    opts = await mergeConfigWithConfigFile(opts);
+    if (appPath) {
+        // Generate metadata for a specific app
+        await generateAppLocksInternal(appPath, false, workspace, opts, false, false);
+    }
+    else {
+        // Generate metadata for all apps
+        const ignore = await ignoreF(opts);
+        const elems = await elementsToMap(await FSFSElement(dntShim.Deno.cwd(), [], true), (p, isD) => {
+            return ignore(p, isD) || (!isD && !p.endsWith(SEP + "raw_app.yaml"));
+        }, false, {});
+        const appFolders = Object.keys(elems)
+            .filter((p) => p.endsWith(SEP + "raw_app.yaml"))
+            .map((p) => p.substring(0, p.length - (SEP + "raw_app.yaml").length));
+        let hasAny = false;
+        log.info("Checking metadata for all apps:");
+        for (const appFolder of appFolders) {
+            const candidate = await generateAppLocksInternal(appFolder, true, workspace, opts, false, true);
+            if (candidate) {
+                hasAny = true;
+                log.info(colors.green(`+ ${candidate}`));
+            }
+        }
+        if (hasAny) {
+            if (opts.dryRun) {
+                log.info(colors.gray(`Dry run complete.`));
+                return;
+            }
+            if (!opts.yes &&
+                !(await Confirm.prompt({
+                    message: "Update the metadata of the above apps?",
+                    default: true,
+                }))) {
+                return;
+            }
+        }
+        else {
+            log.info(colors.green.bold("No metadata to update"));
+            return;
+        }
+        for (const appFolder of appFolders) {
+            await generateAppLocksInternal(appFolder, false, workspace, opts, false, true);
+        }
+    }
+}
+async function pushRawAppCommand(opts, filePath, remotePath) {
+    if (!validatePath(remotePath)) {
+        return;
+    }
+    const workspace = await resolveWorkspace(opts);
+    await requireLogin(opts);
+    await pushRawApp(workspace.workspaceId, remotePath, filePath);
+    log.info(colors.bold.underline.green("Raw app pushed"));
+}

package/esm/src/commands/app/wmillTsDev.js

@@ -0,0 +1,89 @@
+//comment this line and last to dev
+export function wmillTsDev(port) {
+    return `
+let reqs: Record<string, any> = {}
+let ws: WebSocket | null = null
+let wsReady: Promise<void>
+let wsReadyResolve: () => void
+
+function initWebSocket() {
+    wsReady = new Promise((resolve) => {
+        wsReadyResolve = resolve
+    })
+
+    ws = new WebSocket('ws://localhost:${port}')
+
+    ws.onopen = () => {
+        console.log('[wmill] WebSocket connected')
+        wsReadyResolve()
+    }
+
+    ws.onmessage = (event) => {
+        const data = JSON.parse(event.data)
+        if (data.type === 'backendRes' || data.type === 'backendAsyncRes') {
+            console.log('Message from WebSocket backend', data)
+            const job = reqs[data.reqId]
+            if (job) {
+                const result = data.result
+                if (data.error) {
+                    job.reject(new Error(result.stack ?? result.message))
+                } else {
+                    job.resolve(result)
+                }
+                delete reqs[data.reqId]
+            } else {
+                console.error('No job found for', data.reqId)
+            }
+        }
+    }
+
+    ws.onerror = (error) => {
+        console.error('[wmill] WebSocket error:', error)
+    }
+
+    ws.onclose = () => {
+        console.log('[wmill] WebSocket closed, reconnecting...')
+        setTimeout(initWebSocket, 1000)
+    }
+}
+
+initWebSocket()
+
+async function doRequest(type: string, o: object) {
+    await wsReady
+    return new Promise((resolve, reject) => {
+        const reqId = Math.random().toString(36)
+        reqs[reqId] = { resolve, reject }
+        ws?.send(JSON.stringify({ ...o, type, reqId }))
+    })
+}
+
+export const backend = new Proxy(
+    {},
+    {
+        get(_, runnable_id: string) {
+            return (v: any) => {
+                return doRequest('backend', { runnable_id, v })
+            }
+        }
+    })
+
+export const backendAsync = new Proxy(
+    {},
+    {
+        get(_, runnable_id: string) {
+            return (v: any) => {
+                return doRequest('backendAsync', { runnable_id, v })
+            }
+        }
+    })
+
+export function waitJob(jobId: string) {
+    return doRequest('waitJob', { jobId })
+}
+
+export function getJob(jobId: string) {
+    return doRequest('getJob', { jobId })
+}
+`;
+}

package/esm/src/commands/dependencies/dependencies.js

@@ -0,0 +1,66 @@
+// deno-lint-ignore-file no-explicit-any
+import { requireLogin } from "../../core/auth.js";
+import { resolveWorkspace } from "../../core/context.js";
+import { colors, Command, log } from "../../../deps.js";
+import * as wmill from "../../../gen/services.gen.js";
+import fs from "node:fs";
+import { generateHash } from "../../utils/utils.js";
+import { checkifMetadataUptodate, updateMetadataGlobalLock, workspaceDependenciesPathToLanguageAndFilename } from "../../utils/metadata.js";
+async function push(opts, filePath, language, name) {
+    const workspace = await resolveWorkspace(opts);
+    await requireLogin(opts);
+    if (!fs.existsSync(filePath)) {
+        throw new Error(`File not found: ${filePath}`);
+    }
+    const content = fs.readFileSync(filePath, "utf8");
+    // Use the existing pushWorkspaceDependencies function
+    await pushWorkspaceDependencies(workspace.workspaceId, filePath, null, content);
+}
+const command = new Command()
+    .alias("deps")
+    .description("workspace dependencies related commands")
+    .command("push", "Push workspace dependencies from a local file")
+    .arguments("<file_path:string>")
+    .option("--language <language:string>", "Programming language (python3, typescript, go, php). If not specified, will be inferred from file extension.")
+    .option("--name <name:string>", "Name for the dependencies. If not specified, creates workspace default dependencies.")
+    .action(push);
+export async function pushWorkspaceDependencies(workspace, path, _befObj, newDependenciesContent) {
+    try {
+        let res = workspaceDependenciesPathToLanguageAndFilename(path);
+        if (!res) {
+            throw new Error(`Unknown workspace dependencies file format: ${path}`);
+        }
+        let { language, name } = res;
+        // TODO: include workspace?
+        // Generate hash for workspace dependencies content and metadata
+        const contentHash = await generateHash(newDependenciesContent + path);
+        // Check if dependencies are up-to-date using wmill-lock.yaml tracking
+        const isUpToDate = await checkifMetadataUptodate(path, contentHash, undefined);
+        if (isUpToDate) {
+            const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`;
+            log.info(colors.green(`${displayName} for ${language} are up-to-date, skipping push`));
+            return;
+        }
+        log.info(colors.yellow(`Pushing ${name ? 'named' : 'workspace default'} dependencies for ${language}...`));
+        await wmill.createWorkspaceDependencies({
+            workspace,
+            requestBody: {
+                name,
+                content: newDependenciesContent,
+                language,
+                workspace_id: workspace,
+                // Description is not supported in cli, it will use old description
+                description: undefined
+            }
+        });
+        // Update wmill-lock.yaml with new hash after successful push
+        await updateMetadataGlobalLock(path, contentHash);
+        const displayName = name ? `named dependencies "${name}"` : `workspace default dependencies`;
+        log.info(colors.green(`Successfully pushed ${displayName} for ${language}`));
+    }
+    catch (error) {
+        log.error(colors.red(`Failed to push workspace dependencies: ${error.message}`));
+        throw error;
+    }
+}
+export default command;

package/esm/src/commands/dev/dev.js

@@ -103,7 +103,7 @@ async function dev(opts) {
             ws.on("message", (message) => {
                 let data;
                 try {
-                    data = JSON.parse(message);
+                    data = JSON.parse(message.toString());
                 }
                 catch (e) {
                     console.log("Received invalid JSON: " + message + " " + e);

package/esm/src/commands/flow/flow.js

@@ -8,10 +8,10 @@ import { requireLogin } from "../../core/auth.js";
 import { resolveWorkspace, validatePath } from "../../core/context.js";
 import { resolve, track_job } from "../script/script.js";
 import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.js";
-import { generateFlowLockInternal } from "../../utils/metadata.js";
 import { mergeConfigWithConfigFile } from "../../core/conf.js";
 import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.js";
 import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.js";
+import { generateFlowLockInternal } from "./flow_metadata.js";
 const alreadySynced = [];
 export async function pushFlow(workspace, remotePath, localPath, message) {
     if (alreadySynced.includes(localPath)) {
@@ -150,13 +150,12 @@ async function run(opts, path) {
     log.info(JSON.stringify(jobInfo.result ?? {}, null, 2));
 }
 async function generateLocks(opts, folder) {
-    const useRawReqs = opts.useRawRequirements || dntShim.Deno.env.get("USE_RAW_REQUIREMENTS") === "true";
     const workspace = await resolveWorkspace(opts);
     await requireLogin(opts);
     opts = await mergeConfigWithConfigFile(opts);
     if (folder) {
         // read script metadata file
-        await generateFlowLockInternal(folder, false, workspace, opts, undefined, undefined, useRawReqs);
+        await generateFlowLockInternal(folder, false, workspace, opts);
     }
     else {
         const ignore = await ignoreF(opts);
@@ -168,7 +167,7 @@ async function generateLocks(opts, folder) {
         }, false, {})).map((x) => x.substring(0, x.lastIndexOf(SEP)));
         let hasAny = false;
         for (const folder of elems) {
-            const candidate = await generateFlowLockInternal(folder, true, workspace, opts, undefined, undefined, useRawReqs);
+            const candidate = await generateFlowLockInternal(folder, true, workspace, opts);
             if (candidate) {
                 hasAny = true;
                 log.info(colors.yellow.bold(`~ ${candidate}`));
@@ -188,7 +187,7 @@ async function generateLocks(opts, folder) {
             return;
         }
         for (const folder of elems) {
-            await generateFlowLockInternal(folder, false, workspace, opts, undefined, undefined, useRawReqs);
+            await generateFlowLockInternal(folder, false, workspace, opts);
         }
     }
 }

package/esm/src/commands/flow/flow_metadata.js

@@ -0,0 +1,140 @@
+import * as dntShim from "../../../_dnt.shims.js";
+import { SEP, colors, log, path, yamlParseFile, yamlStringify, } from "../../../deps.js";
+import { readLockfile, checkifMetadataUptodate, blueColor, clearGlobalLock, updateMetadataGlobalLock, LockfileGenerationError, getRawWorkspaceDependencies, } from "../../utils/metadata.js";
+import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.js";
+import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.js";
+import { exts, } from "../script/script.js";
+import { FSFSElement } from "../sync/sync.js";
+import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.js";
+import { workspaceDependenciesLanguages } from "../../utils/script_common.js";
+const TOP_HASH = "__flow_hash";
+async function generateFlowHash(rawWorkspaceDependencies, folder, defaultTs) {
+    const elems = await FSFSElement(path.join(dntShim.Deno.cwd(), folder), [], true);
+    const hashes = {};
+    for await (const f of elems.getChildren()) {
+        if (exts.some((e) => f.path.endsWith(e))) {
+            // Embed workspace dependencies into hash
+            hashes[f.path] = await generateHash((await f.getContentText()) + JSON.stringify(rawWorkspaceDependencies));
+        }
+    }
+    return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
+}
+export async function generateFlowLockInternal(folder, dryRun, workspace, opts, justUpdateMetadataLock, noStaleMessage) {
+    if (folder.endsWith(SEP)) {
+        folder = folder.substring(0, folder.length - 1);
+    }
+    const remote_path = folder
+        .replaceAll(SEP, "/")
+        .substring(0, folder.length - ".flow".length);
+    if (!justUpdateMetadataLock && !noStaleMessage) {
+        log.info(`Generating lock for flow ${folder} at ${remote_path}`);
+    }
+    // Always get out-of-sync workspace dependencies
+    const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
+    let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
+    const conf = await readLockfile();
+    if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
+        if (!noStaleMessage) {
+            log.info(colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`));
+        }
+        return;
+    }
+    else if (dryRun) {
+        return remote_path;
+    }
+    if (Object.keys(rawWorkspaceDependencies).length > 0) {
+        log.info((await blueColor())(`Found workspace dependencies (${workspaceDependenciesLanguages
+            .map((l) => l.filename)
+            .join("/")}) for ${folder}, using them`));
+    }
+    const flowValue = (await yamlParseFile(folder + SEP + "flow.yaml"));
+    if (!justUpdateMetadataLock) {
+        const changedScripts = [];
+        //find hashes that do not correspond to previous hashes
+        for (const [path, hash] of Object.entries(hashes)) {
+            if (path == TOP_HASH) {
+                continue;
+            }
+            if (!(await checkifMetadataUptodate(folder, hash, conf, path))) {
+                changedScripts.push(path);
+            }
+        }
+        log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
+        await replaceInlineScripts(flowValue.value.modules, async (path) => await dntShim.Deno.readTextFile(folder + SEP + path), log, folder + SEP, SEP, changedScripts);
+        //removeChangedLocks
+        flowValue.value = await updateFlow(workspace, flowValue.value, remote_path, rawWorkspaceDependencies);
+        const inlineScripts = extractInlineScriptsForFlows(flowValue.value.modules, {}, SEP, opts.defaultTs);
+        inlineScripts.forEach((s) => {
+            writeIfChanged(dntShim.Deno.cwd() + SEP + folder + SEP + s.path, s.content);
+        });
+        // Overwrite `flow.yaml` with the new lockfile references
+        writeIfChanged(dntShim.Deno.cwd() + SEP + folder + SEP + "flow.yaml", yamlStringify(flowValue));
+    }
+    hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
+    await clearGlobalLock(folder);
+    for (const [path, hash] of Object.entries(hashes)) {
+        await updateMetadataGlobalLock(folder, hash, path);
+    }
+    log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
+}
+export async function updateFlow(workspace, flow_value, remotePath, rawWorkspaceDependencies) {
+    let rawResponse;
+    if (Object.keys(rawWorkspaceDependencies).length > 0) {
+        log.info(colors.blue("Using raw workspace dependencies for flow dependencies"));
+        // generate the script lock running a dependency job in Windmill and update it inplace
+        const extraHeaders = getHeaders();
+        rawResponse = await fetch(`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, {
+            method: "POST",
+            headers: {
+                Cookie: `token=${workspace.token}`,
+                "Content-Type": "application/json",
+                ...extraHeaders,
+            },
+            body: JSON.stringify({
+                flow_value,
+                path: remotePath,
+                use_local_lockfiles: true,
+                raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
+                    ? rawWorkspaceDependencies
+                    : null,
+            }),
+        });
+    }
+    else {
+        // Standard dependency resolution on the server
+        const extraHeaders = getHeaders();
+        rawResponse = await fetch(`${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/flow_dependencies`, {
+            method: "POST",
+            headers: {
+                Cookie: `token=${workspace.token}`,
+                "Content-Type": "application/json",
+                ...extraHeaders,
+            },
+            body: JSON.stringify({
+                flow_value,
+                path: remotePath,
+            }),
+        });
+    }
+    let responseText = "reading response failed";
+    try {
+        const res = (await rawResponse.json());
+        if (rawResponse.status != 200) {
+            const msg = res?.["error"]?.["message"];
+            if (msg) {
+                throw new LockfileGenerationError(`Failed to generate lockfile: ${msg}`);
+            }
+            throw new LockfileGenerationError(`Failed to generate lockfile: ${rawResponse.statusText}, ${responseText}`);
+        }
+        return res.updated_flow_value;
+    }
+    catch (e) {
+        try {
+            responseText = await rawResponse.text();
+        }
+        catch {
+            responseText = "";
+        }
+        throw new Error(`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`);
+    }
+}