windmill-cli 1.743.0 → 1.745.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/esm/main.js +1071 -89
  2. package/package.json +1 -1
package/esm/main.js CHANGED
@@ -16784,7 +16784,7 @@ var init_OpenAPI = __esm(() => {
16784
16784
  PASSWORD: undefined,
16785
16785
  TOKEN: getEnv3("WM_TOKEN"),
16786
16786
  USERNAME: undefined,
16787
- VERSION: "1.743.0",
16787
+ VERSION: "1.745.0",
16788
16788
  WITH_CREDENTIALS: true,
16789
16789
  interceptors: {
16790
16790
  request: new Interceptors,
@@ -17128,6 +17128,7 @@ __export(exports_services_gen, {
17128
17128
  star: () => star,
17129
17129
  signS3Objects: () => signS3Objects,
17130
17130
  setupCustomInstanceDb: () => setupCustomInstanceDb,
17131
+ setWsSpecific: () => setWsSpecific,
17131
17132
  setWorkspaceSlackOauthConfig: () => setWorkspaceSlackOauthConfig,
17132
17133
  setWorkspaceEncryptionKey: () => setWorkspaceEncryptionKey,
17133
17134
  setWebsocketTriggerMode: () => setWebsocketTriggerMode,
@@ -17369,6 +17370,8 @@ __export(exports_services_gen, {
17369
17370
  listAuditLogs: () => listAuditLogs,
17370
17371
  listAssetsByUsage: () => listAssetsByUsage,
17371
17372
  listAssets: () => listAssets,
17373
+ listAssetSchemas: () => listAssetSchemas,
17374
+ listAssetPartitions: () => listAssetPartitions,
17372
17375
  listAssetDispatchEdges: () => listAssetDispatchEdges,
17373
17376
  listApps: () => listApps,
17374
17377
  listAppPathsFromWorkspaceRunnable: () => listAppPathsFromWorkspaceRunnable,
@@ -17536,6 +17539,7 @@ __export(exports_services_gen, {
17536
17539
  getFlowAllLogs: () => getFlowAllLogs,
17537
17540
  getEmailTrigger: () => getEmailTrigger,
17538
17541
  getDraftForUser: () => getDraftForUser,
17542
+ getDevWorkspace: () => getDevWorkspace,
17539
17543
  getDeployTo: () => getDeployTo,
17540
17544
  getDependentsAmounts: () => getDependentsAmounts,
17541
17545
  getDependents: () => getDependents,
@@ -17651,6 +17655,7 @@ __export(exports_services_gen, {
17651
17655
  disconnectSlack: () => disconnectSlack,
17652
17656
  disconnectAccount: () => disconnectAccount,
17653
17657
  diffRawScriptsWithDeployed: () => diffRawScriptsWithDeployed,
17658
+ detachDevWorkspace: () => detachDevWorkspace,
17654
17659
  deleteWorkspaceSlackOauthConfig: () => deleteWorkspaceSlackOauthConfig,
17655
17660
  deleteWorkspaceDependencies: () => deleteWorkspaceDependencies,
17656
17661
  deleteWorkspace: () => deleteWorkspace,
@@ -17782,6 +17787,7 @@ __export(exports_services_gen, {
17782
17787
  batchReRunJobs: () => batchReRunJobs,
17783
17788
  backendVersion: () => backendVersion,
17784
17789
  backendUptodate: () => backendUptodate,
17790
+ attachDevWorkspace: () => attachDevWorkspace,
17785
17791
  assignGhesInstallation: () => assignGhesInstallation,
17786
17792
  archiveWorkspaceDependencies: () => archiveWorkspaceDependencies,
17787
17793
  archiveWorkspace: () => archiveWorkspace,
@@ -18183,6 +18189,34 @@ var backendVersion = () => {
18183
18189
  body: data3.requestBody,
18184
18190
  mediaType: "application/json"
18185
18191
  });
18192
+ }, attachDevWorkspace = (data3) => {
18193
+ return request(OpenAPI, {
18194
+ method: "POST",
18195
+ url: "/w/{workspace}/workspaces/attach_dev_workspace",
18196
+ path: {
18197
+ workspace: data3.workspace
18198
+ },
18199
+ body: data3.requestBody,
18200
+ mediaType: "application/json"
18201
+ });
18202
+ }, detachDevWorkspace = (data3) => {
18203
+ return request(OpenAPI, {
18204
+ method: "POST",
18205
+ url: "/w/{workspace}/workspaces/detach_dev_workspace",
18206
+ path: {
18207
+ workspace: data3.workspace
18208
+ },
18209
+ body: data3.requestBody,
18210
+ mediaType: "application/json"
18211
+ });
18212
+ }, getDevWorkspace = (data3) => {
18213
+ return request(OpenAPI, {
18214
+ method: "GET",
18215
+ url: "/w/{workspace}/workspaces/get_dev_workspace",
18216
+ path: {
18217
+ workspace: data3.workspace
18218
+ }
18219
+ });
18186
18220
  }, existsWorkspace = (data3) => {
18187
18221
  return request(OpenAPI, {
18188
18222
  method: "POST",
@@ -19820,6 +19854,16 @@ var backendVersion = () => {
19820
19854
  path: data3.path
19821
19855
  }
19822
19856
  });
19857
+ }, setWsSpecific = (data3) => {
19858
+ return request(OpenAPI, {
19859
+ method: "POST",
19860
+ url: "/w/{workspace}/workspaces/set_ws_specific",
19861
+ path: {
19862
+ workspace: data3.workspace
19863
+ },
19864
+ body: data3.requestBody,
19865
+ mediaType: "application/json"
19866
+ });
19823
19867
  }, setPublicAppRateLimit = (data3) => {
19824
19868
  return request(OpenAPI, {
19825
19869
  method: "POST",
@@ -25353,6 +25397,28 @@ var backendVersion = () => {
25353
25397
  workspace: data3.workspace
25354
25398
  }
25355
25399
  });
25400
+ }, listAssetPartitions = (data3) => {
25401
+ return request(OpenAPI, {
25402
+ method: "GET",
25403
+ url: "/w/{workspace}/assets/partitions",
25404
+ path: {
25405
+ workspace: data3.workspace
25406
+ },
25407
+ query: {
25408
+ path: data3.path
25409
+ }
25410
+ });
25411
+ }, listAssetSchemas = (data3) => {
25412
+ return request(OpenAPI, {
25413
+ method: "GET",
25414
+ url: "/w/{workspace}/assets/asset_schemas",
25415
+ path: {
25416
+ workspace: data3.workspace
25417
+ },
25418
+ query: {
25419
+ path: data3.path
25420
+ }
25421
+ });
25356
25422
  }, listVolumes = (data3) => {
25357
25423
  return request(OpenAPI, {
25358
25424
  method: "GET",
@@ -25542,7 +25608,7 @@ var init_auth = __esm(async () => {
25542
25608
  });
25543
25609
 
25544
25610
  // src/core/constants.ts
25545
- var WM_FORK_PREFIX = "wm-fork", VERSION = "1.743.0";
25611
+ var WM_FORK_PREFIX = "wm-fork", VERSION = "1.745.0";
25546
25612
 
25547
25613
  // src/utils/git.ts
25548
25614
  var exports_git = {};
@@ -26514,6 +26580,10 @@ async function createWorkspaceFork2(opts, workspaceName, workspaceId = undefined
26514
26580
  info(colors.blue(`Creating forked workspace: ${workspaceName}...`));
26515
26581
  const trueWorkspaceId = `${WM_FORK_PREFIX}-${workspaceId}`;
26516
26582
  validateForkWorkspaceId(trueWorkspaceId);
26583
+ const effectiveName = opts.createWorkspaceName ?? workspaceName ?? trueWorkspaceId;
26584
+ if (effectiveName.length > 50) {
26585
+ throw new Error(`Fork workspace name is too long (${effectiveName.length} chars; max 50). Choose a shorter name.`);
26586
+ }
26517
26587
  let alreadyExists = false;
26518
26588
  try {
26519
26589
  alreadyExists = await existsWorkspace({
@@ -28557,7 +28627,7 @@ var init_workspace = __esm(async () => {
28557
28627
  ]);
28558
28628
  command2 = new Command().alias("profile").description("workspace related commands").action(list3).command("switch").complete("workspace", async () => (await allWorkspaces()).map((x) => x.name)).description("Switch to another workspace").arguments("<workspace_name:string:workspace>").action(switchC).command("add").description("Add a workspace").arguments("[workspace_name:string] [workspace_id:string] [remote:string]").option("-c --create", "Create the workspace if it does not exist").option("--create-workspace-name <workspace_name:string>", "Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.").option("--create-username <username:string>", "Specify your own username in the newly created workspace. Ignored if --create is not specified, the workspace already exists or automatic username creation is enabled on the instance.", {
28559
28629
  default: "admin"
28560
- }).action(add).command("remove").description("Remove a workspace").arguments("<workspace_name:string>").action(remove).command("whoami").description("Show the currently active user").action(whoami2).command("list").description("List local workspace profiles").action(list3).command("list-remote").description("List workspaces on the remote server that you have access to").option("--as-superadmin", "List ALL workspaces on the instance (requires the token to belong to a superadmin/devops user)").action(listRemote).command("list-forks").description("List forked workspaces on the remote server").action(listForks).command("bind").description("Create or update a workspace entry in wmill.yaml from the active profile").option("--workspace <name:string>", "Workspace name (default: current branch or workspaceId)").option("--branch <branch:string>", "Git branch to associate (default: workspace name)").action((opts) => bind(opts, true)).command("unbind").description("Remove baseUrl and workspaceId from a workspace entry").option("--workspace <name:string>", "Workspace to unbind").action((opts) => bind(opts, false)).command("fork").description("Create a forked workspace").arguments("[workspace_name:string] [workspace_id:string]").option("--create-workspace-name <workspace_name:string>", "Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.").option("--color <color:string>", "Workspace color (hex code, e.g. #ff0000)").option("--datatable-behavior <behavior:string>", "How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt)").option("--from-branch <branch:string>", "Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on <branch> (its bound workspace is the parent) and rename the current branch onto wm-fork/<branch>/<id>. Usually unneeded — from a working branch `wmill workspace fork` offers this interactively; from a base branch it creates a fresh fork branch.").option("-y --yes", "Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for.").action(createWorkspaceFork2).command("delete-fork").description("Delete a forked workspace and git branch").arguments("<fork_name:string>").option("-y --yes", "Skip confirmation prompt").action(deleteWorkspaceFork).command("merge").description("Compare and deploy changes between a fork and its parent workspace").option("--direction <direction:string>", "Deploy direction: to-parent or to-fork").option("--all", "Deploy all changed items including conflicts").option("--skip-conflicts", "Skip items modified in both workspaces").option("--include <items:string>", "Comma-separated kind:path items to include (e.g. script:f/test/main,flow:f/my/flow)").option("--exclude <items:string>", "Comma-separated kind:path items to exclude").option("--preserve-on-behalf-of", "Preserve original on_behalf_of/permissioned_as values").option("-y --yes", "Non-interactive mode (deploy without prompts)").action(mergeWorkspaces).command("connect-slack").description("Non-interactively connect Slack to the active workspace using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: workspace_settings fields, g/slack group, f/slack_bot folder, and the encrypted bot token variable + resource at f/slack_bot/bot_token.").option("--bot-token <bot_token:string>", "Slack bot token (xoxb-...)", { required: true }).option("--team-id <team_id:string>", "Slack team id", { required: true }).option("--team-name <team_name:string>", "Slack team name", { required: true }).action(connectSlack2).command("disconnect-slack").description("Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push.").action(disconnectSlack2);
28630
+ }).action(add).command("remove").description("Remove a workspace").arguments("<workspace_name:string>").action(remove).command("whoami").description("Show the currently active user").action(whoami2).command("list").description("List local workspace profiles").action(list3).command("list-remote").description("List workspaces on the remote server that you have access to").option("--as-superadmin", "List ALL workspaces on the instance (requires the token to belong to a superadmin/devops user)").action(listRemote).command("list-forks").description("List forked workspaces on the remote server").action(listForks).command("bind").description("Create or update a workspace entry in wmill.yaml from the active profile").option("--workspace <name:string>", "Workspace name (default: current branch or workspaceId)").option("--branch <branch:string>", "Git branch to associate (default: workspace name)").action((opts) => bind(opts, true)).command("unbind").description("Remove baseUrl and workspaceId from a workspace entry").option("--workspace <name:string>", "Workspace to unbind").action((opts) => bind(opts, false)).command("fork").description("Create a forked workspace").arguments("[workspace_name:string] [workspace_id:string]").option("--create-workspace-name <workspace_name:string>", "Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id.").option("--color <color:string>", "Workspace color (hex code, e.g. #ff0000)").option("--datatable-behavior <behavior:string>", "How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt)").option("--from-branch <branch:string>", "Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on <branch> (its bound workspace is the parent) and rename the current branch onto wm-fork/<branch>/<id>. Usually unneeded — from a working branch `wmill workspace fork` offers this interactively; from a base branch it creates a fresh fork branch.").option("-y --yes", "Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for.").action(createWorkspaceFork2).command("delete-fork").description("Delete a forked workspace").arguments("<fork_name:string>").option("-y --yes", "Skip confirmation prompt").action(deleteWorkspaceFork).command("merge").description("Compare and deploy changes between a fork and its parent workspace").option("--direction <direction:string>", "Deploy direction: to-parent or to-fork").option("--all", "Deploy all changed items including conflicts").option("--skip-conflicts", "Skip items modified in both workspaces").option("--include <items:string>", "Comma-separated kind:path items to include (e.g. script:f/test/main,flow:f/my/flow)").option("--exclude <items:string>", "Comma-separated kind:path items to exclude").option("--preserve-on-behalf-of", "Preserve original on_behalf_of/permissioned_as values").option("-y --yes", "Non-interactive mode (deploy without prompts)").action(mergeWorkspaces).command("connect-slack").description("Non-interactively connect Slack to the active workspace using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: workspace_settings fields, g/slack group, f/slack_bot folder, and the encrypted bot token variable + resource at f/slack_bot/bot_token.").option("--bot-token <bot_token:string>", "Slack bot token (xoxb-...)", { required: true }).option("--team-id <team_id:string>", "Slack team id", { required: true }).option("--team-name <team_name:string>", "Slack team name", { required: true }).action(connectSlack2).command("disconnect-slack").description("Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push.").action(disconnectSlack2);
28561
28631
  workspace_default = command2;
28562
28632
  });
28563
28633
 
@@ -85832,11 +85902,20 @@ inspect asset-driven pipelines (scripts marked \`// pipeline\`, wired by \`// on
85832
85902
  - \`--json\` - Output as JSON (for piping to jq)
85833
85903
  - \`pipeline show <folder:string>\` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal
85834
85904
  - \`--json\` - Output the raw asset graph as JSON
85905
+ - \`--local\` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace — no deploy needed.
85835
85906
  - \`pipeline run <folder:string>\` - run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s)
85836
85907
  - \`--from <script:string>\` - Start script (short name or path). Defaults to the folder's sole schedule/manual root.
85837
85908
  - \`--to <node:string>\` - End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream.
85838
85909
  - \`--dry-run\` - Print the topological run plan without executing.
85839
85910
  - \`--json\` - Output the plan as JSON (for piping to jq).
85911
+ - \`--local\` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files.
85912
+ - \`--upload <binding:string>\` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable.
85913
+ - \`pipeline docs <folder:string>\` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop
85914
+ - \`--local\` - Build the graph from local working-tree files instead of the deployed workspace.
85915
+ - \`pipeline dev [folder:string]\` - Live-preview a data pipeline from local files: watch an \`f/<folder>\` of \`// pipeline\` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).
85916
+ - \`--port <port:number>\` - Port for the dev WebSocket server.
85917
+ - \`--no-open\` - Do not open the browser automatically.
85918
+ - \`--frontend <origin:string>\` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.
85840
85919
 
85841
85920
  ### protection-rules
85842
85921
 
@@ -86168,7 +86247,7 @@ workspace related commands
86168
86247
  - \`--datatable-behavior <behavior:string>\` - How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt)
86169
86248
  - \`--from-branch <branch:string>\` - Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on <branch> (its bound workspace is the parent) and rename the current branch onto wm-fork/<branch>/<id>. Usually unneeded — from a working branch \`wmill workspace fork\` offers this interactively; from a base branch it creates a fresh fork branch.
86170
86249
  - \`-y --yes\` - Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for.
86171
- - \`workspace delete-fork <fork_name:string>\` - Delete a forked workspace and git branch
86250
+ - \`workspace delete-fork <fork_name:string>\` - Delete a forked workspace
86172
86251
  - \`-y --yes\` - Skip confirmation prompt
86173
86252
  - \`workspace merge\` - Compare and deploy changes between a fork and its parent workspace
86174
86253
  - \`--direction <direction:string>\` - Deploy direction: to-parent or to-fork
@@ -94754,14 +94833,16 @@ var scriptNodeId = (path23) => `${SCRIPT_PREFIX}${path23}`;
94754
94833
  var isScriptNode = (id) => id.startsWith(SCRIPT_PREFIX);
94755
94834
  var scriptPathOf = (id) => id.slice(SCRIPT_PREFIX.length);
94756
94835
  var assetNodeId = (kind, path23) => `${kind}:${path23}`;
94757
- var EVENT_TRIGGER_KINDS = new Set([
94836
+ var NON_AUTORUN_TRIGGER_KINDS = new Set([
94758
94837
  "kafka",
94759
94838
  "mqtt",
94760
94839
  "nats",
94761
94840
  "postgres",
94762
94841
  "sqs",
94763
94842
  "gcp",
94764
- "email"
94843
+ "email",
94844
+ "webhook",
94845
+ "data_upload"
94765
94846
  ]);
94766
94847
  function assetUriToNodeId(uri) {
94767
94848
  const m3 = uri.match(/^([a-z0-9_]+):\/\/(.+)$/i);
@@ -94823,6 +94904,26 @@ function closure(adj, start) {
94823
94904
  }
94824
94905
  var descendants = (dag, n2) => closure(dag.down, n2);
94825
94906
  var ancestors = (dag, n2) => closure(dag.up, n2);
94907
+ function reachableCutting(dag, starts, barriers) {
94908
+ const seen = new Set;
94909
+ const queue = [];
94910
+ for (const s2 of starts) {
94911
+ if (barriers.has(s2) || seen.has(s2))
94912
+ continue;
94913
+ seen.add(s2);
94914
+ queue.push(s2);
94915
+ }
94916
+ while (queue.length > 0) {
94917
+ const n2 = queue.shift();
94918
+ for (const next of dag.down.get(n2) ?? []) {
94919
+ if (barriers.has(next) || seen.has(next))
94920
+ continue;
94921
+ seen.add(next);
94922
+ queue.push(next);
94923
+ }
94924
+ }
94925
+ return seen;
94926
+ }
94826
94927
  function boundedSet(dag, start, ends) {
94827
94928
  const downSet = new Set(descendants(dag, start));
94828
94929
  downSet.add(start);
@@ -94847,7 +94948,7 @@ function boundedSet(dag, start, ends) {
94847
94948
  function validStarts(g2) {
94848
94949
  const subscribers = new Set;
94849
94950
  const scheduleScripts = new Set;
94850
- const eventScripts = new Set;
94951
+ const nonAutorunScripts = new Set;
94851
94952
  for (const t2 of g2.triggers ?? []) {
94852
94953
  if (t2.runnable_kind !== "script")
94853
94954
  continue;
@@ -94855,8 +94956,8 @@ function validStarts(g2) {
94855
94956
  subscribers.add(t2.runnable_path);
94856
94957
  else if (t2.trigger_kind === "schedule")
94857
94958
  scheduleScripts.add(t2.runnable_path);
94858
- else if (EVENT_TRIGGER_KINDS.has(t2.trigger_kind))
94859
- eventScripts.add(t2.runnable_path);
94959
+ else if (NON_AUTORUN_TRIGGER_KINDS.has(t2.trigger_kind))
94960
+ nonAutorunScripts.add(t2.runnable_path);
94860
94961
  }
94861
94962
  const out = new Set;
94862
94963
  for (const r2 of g2.runnables ?? []) {
@@ -94865,11 +94966,20 @@ function validStarts(g2) {
94865
94966
  const p3 = r2.path;
94866
94967
  if (scheduleScripts.has(p3))
94867
94968
  out.add(scriptNodeId(p3));
94868
- else if (!subscribers.has(p3) && !eventScripts.has(p3))
94969
+ else if (!subscribers.has(p3) && !nonAutorunScripts.has(p3))
94869
94970
  out.add(scriptNodeId(p3));
94870
94971
  }
94871
94972
  return out;
94872
94973
  }
94974
+ function nonAutorunTriggerScripts(g2) {
94975
+ const out = new Set;
94976
+ for (const t2 of g2.triggers ?? []) {
94977
+ if (t2.runnable_kind === "script" && NON_AUTORUN_TRIGGER_KINDS.has(t2.trigger_kind)) {
94978
+ out.add(scriptNodeId(t2.runnable_path));
94979
+ }
94980
+ }
94981
+ return out;
94982
+ }
94873
94983
  function scriptsOf(nodes) {
94874
94984
  const out = [];
94875
94985
  for (const id of nodes)
@@ -94931,14 +95041,716 @@ function topoOrder(g2, scripts) {
94931
95041
  return { order, cyclic };
94932
95042
  }
94933
95043
 
95044
+ // src/commands/pipeline/localGraph.ts
95045
+ init_script_common();
95046
+ await __promiseAll([
95047
+ init_metadata(),
95048
+ init_script(),
95049
+ init_conf()
95050
+ ]);
95051
+ import * as fs17 from "node:fs";
95052
+ import * as path23 from "node:path";
95053
+ import process25 from "node:process";
95054
+ function workspaceRoot() {
95055
+ const yaml = getWmillYamlPath();
95056
+ return yaml ? path23.dirname(yaml) : process25.cwd();
95057
+ }
95058
+ function wasmFnForLanguage(language) {
95059
+ switch (language) {
95060
+ case "bun":
95061
+ case "deno":
95062
+ case "nativets":
95063
+ return "parse_assets_ts";
95064
+ case "bunnative":
95065
+ return "parse_assets_ts";
95066
+ case "python3":
95067
+ return "parse_assets_py";
95068
+ case "duckdb":
95069
+ case "postgresql":
95070
+ case "mysql":
95071
+ case "bigquery":
95072
+ case "snowflake":
95073
+ case "mssql":
95074
+ case "oracledb":
95075
+ return "parse_assets_sql";
95076
+ default:
95077
+ return;
95078
+ }
95079
+ }
95080
+ var ASSET_WASM_PKG = "windmill-parser-wasm-asset";
95081
+ function commentPrefix(language) {
95082
+ if (language === "python3" || language === "bash" || language === "ansible" || language === "ruby" || language === "rlang" || language === "nu" || language === "powershell")
95083
+ return "#";
95084
+ if (language === "postgresql" || language === "mysql" || language === "bigquery" || language === "snowflake" || language === "mssql" || language === "oracledb" || language === "duckdb")
95085
+ return "--";
95086
+ return "//";
95087
+ }
95088
+ var NATIVE_KINDS = new Set([
95089
+ "schedule",
95090
+ "webhook",
95091
+ "email",
95092
+ "kafka",
95093
+ "mqtt",
95094
+ "nats",
95095
+ "postgres",
95096
+ "sqs",
95097
+ "gcp",
95098
+ "data_upload"
95099
+ ]);
95100
+ function fallbackParse(content, language) {
95101
+ const raw = commentPrefix(language);
95102
+ const p3 = raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
95103
+ const out = { in_pipeline: false, triggers: [] };
95104
+ for (const line of content.split(`
95105
+ `)) {
95106
+ const trimmed = line.trim();
95107
+ if (trimmed === "")
95108
+ continue;
95109
+ if (!trimmed.startsWith(raw))
95110
+ break;
95111
+ if (line.match(new RegExp(`^\\s*${p3}\\s*pipeline\\s*$`))) {
95112
+ out.in_pipeline = true;
95113
+ continue;
95114
+ }
95115
+ const tag = line.match(new RegExp(`^\\s*${p3}\\s*tag\\s+(\\S+)\\s*$`));
95116
+ if (tag) {
95117
+ out.tag = tag[1];
95118
+ continue;
95119
+ }
95120
+ const on = line.match(new RegExp(`^\\s*${p3}\\s*on\\s+(.+?)\\s*$`));
95121
+ if (!on)
95122
+ continue;
95123
+ const rest = on[1].trim();
95124
+ const firstTok = rest.split(/\s+/)[0];
95125
+ const uri = firstTok.match(/^([a-z0-9_]+):\/\/(.+)$/i);
95126
+ if (uri) {
95127
+ const prefix = uri[1].toLowerCase();
95128
+ const kind = prefix === "s3" ? "s3object" : prefix;
95129
+ out.triggers.push({ kind: "asset", asset_kind: kind, path: uri[2] });
95130
+ } else if (NATIVE_KINDS.has(firstTok) && rest === firstTok) {
95131
+ out.triggers.push({ kind: firstTok });
95132
+ }
95133
+ }
95134
+ return out;
95135
+ }
95136
+ function volumeCommentPrefix(language) {
95137
+ switch (language) {
95138
+ case "python3":
95139
+ case "bash":
95140
+ case "powershell":
95141
+ case "ansible":
95142
+ case "ruby":
95143
+ case "rlang":
95144
+ return "#";
95145
+ case "deno":
95146
+ case "bun":
95147
+ case "bunnative":
95148
+ case "nativets":
95149
+ case "go":
95150
+ return "//";
95151
+ default:
95152
+ return;
95153
+ }
95154
+ }
95155
+ function parseVolumeAnnotations(content, prefix) {
95156
+ const out = [];
95157
+ for (const line of content.split(`
95158
+ `)) {
95159
+ const trimmed = line.trim();
95160
+ if (trimmed === "")
95161
+ continue;
95162
+ if (!trimmed.startsWith(prefix))
95163
+ break;
95164
+ const after = trimmed.slice(prefix.length).trim();
95165
+ const m3 = after.match(/^volume:\s*(\S+)/);
95166
+ if (m3)
95167
+ out.push({ kind: "volume", path: m3[1], access_type: "rw" });
95168
+ }
95169
+ return out;
95170
+ }
95171
+ async function inferScriptAssets(content, language) {
95172
+ const out = await inferScriptAssetsBody(content, language);
95173
+ const vp = volumeCommentPrefix(language);
95174
+ if (vp) {
95175
+ const vols = parseVolumeAnnotations(content, vp);
95176
+ if (vols.length > 0)
95177
+ out.assets = [...out.assets ?? [], ...vols];
95178
+ }
95179
+ return out;
95180
+ }
95181
+ async function inferScriptAssetsBody(content, language) {
95182
+ const fn = wasmFnForLanguage(language);
95183
+ if (!fn)
95184
+ return fallbackParse(content, language);
95185
+ let raw;
95186
+ try {
95187
+ const mod = await loadParser(ASSET_WASM_PKG);
95188
+ raw = mod[fn](content);
95189
+ } catch {
95190
+ return fallbackParse(content, language);
95191
+ }
95192
+ if (raw.startsWith("err:"))
95193
+ return fallbackParse(content, language);
95194
+ try {
95195
+ return JSON.parse(raw);
95196
+ } catch {
95197
+ return fallbackParse(content, language);
95198
+ }
95199
+ }
95200
+ async function collectScripts(folderDir, root, defaultTs) {
95201
+ const out = [];
95202
+ function walk(dir) {
95203
+ let entries;
95204
+ try {
95205
+ entries = fs17.readdirSync(dir, { withFileTypes: true });
95206
+ } catch {
95207
+ return;
95208
+ }
95209
+ for (const e2 of entries) {
95210
+ if (e2.name.startsWith(".") || e2.name === "node_modules")
95211
+ continue;
95212
+ const abs = path23.join(dir, e2.name);
95213
+ if (e2.isDirectory()) {
95214
+ walk(abs);
95215
+ continue;
95216
+ }
95217
+ if (!e2.isFile())
95218
+ continue;
95219
+ const ext2 = exts.find((x3) => e2.name.endsWith(x3));
95220
+ if (!ext2)
95221
+ continue;
95222
+ const relFromRoot = path23.relative(root, abs).replaceAll("\\", "/");
95223
+ let language;
95224
+ try {
95225
+ language = inferContentTypeFromFilePath(abs, defaultTs);
95226
+ } catch {
95227
+ continue;
95228
+ }
95229
+ out.push({
95230
+ path: removeExtensionToPath(relFromRoot),
95231
+ content: fs17.readFileSync(abs, "utf-8"),
95232
+ language
95233
+ });
95234
+ }
95235
+ }
95236
+ walk(folderDir);
95237
+ out.sort((a2, b2) => a2.path.localeCompare(b2.path));
95238
+ return out;
95239
+ }
95240
+ async function buildLocalPipelineGraph(args) {
95241
+ const folderClean = args.folder.replace(/^f\//, "").replace(/\/$/, "");
95242
+ const folderDir = path23.join(args.root, "f", folderClean);
95243
+ const all = await collectScripts(folderDir, args.root, args.defaultTs);
95244
+ const runnables = [];
95245
+ const edges = [];
95246
+ const triggers = [];
95247
+ const assetSet = new Map;
95248
+ const pipelineScripts = [];
95249
+ for (const s2 of all) {
95250
+ const out = await inferScriptAssets(s2.content, s2.language);
95251
+ if (!out.in_pipeline)
95252
+ continue;
95253
+ pipelineScripts.push(out.tag ? { ...s2, tag: out.tag } : s2);
95254
+ const mat = out.materialize;
95255
+ const materialize_strategy = mat && !mat.manual ? mat.append ? "append" : mat.unique_key ? "merge" : "replace" : undefined;
95256
+ runnables.push({
95257
+ path: s2.path,
95258
+ usage_kind: "script",
95259
+ in_pipeline: true,
95260
+ ...out.partition ? { partition_kind: out.partition.kind } : {},
95261
+ ...out.freshness ? { freshness: out.freshness.duration } : {},
95262
+ ...out.tag ? { tag: out.tag } : {},
95263
+ ...out.retry ? { retry: out.retry } : {},
95264
+ ...out.data_tests && out.data_tests.length > 0 ? { data_tests: out.data_tests } : {},
95265
+ ...out.column_lineage && out.column_lineage.length > 0 ? { column_lineage: out.column_lineage } : {},
95266
+ ...mat ? { materialize_target: { kind: mat.target_kind, path: mat.target_path } } : {},
95267
+ ...materialize_strategy ? { materialize_strategy } : {}
95268
+ });
95269
+ for (const a2 of out.assets ?? []) {
95270
+ assetSet.set(`${a2.kind}:${a2.path}`, { kind: a2.kind, path: a2.path });
95271
+ edges.push({
95272
+ runnable_kind: "script",
95273
+ runnable_path: s2.path,
95274
+ asset_kind: a2.kind,
95275
+ asset_path: a2.path,
95276
+ access_type: a2.access_type
95277
+ });
95278
+ }
95279
+ if (mat) {
95280
+ assetSet.set(`${mat.target_kind}:${mat.target_path}`, {
95281
+ kind: mat.target_kind,
95282
+ path: mat.target_path
95283
+ });
95284
+ const hasWrite = edges.some((e2) => e2.runnable_path === s2.path && e2.asset_kind === mat.target_kind && e2.asset_path === mat.target_path && (e2.access_type === "w" || e2.access_type === "rw"));
95285
+ if (!hasWrite) {
95286
+ edges.push({
95287
+ runnable_kind: "script",
95288
+ runnable_path: s2.path,
95289
+ asset_kind: mat.target_kind,
95290
+ asset_path: mat.target_path,
95291
+ access_type: "w"
95292
+ });
95293
+ }
95294
+ }
95295
+ for (const t2 of out.triggers ?? []) {
95296
+ if (t2.kind === "asset") {
95297
+ const at = t2;
95298
+ assetSet.set(`${at.asset_kind}:${at.path}`, {
95299
+ kind: at.asset_kind,
95300
+ path: at.path
95301
+ });
95302
+ triggers.push({
95303
+ trigger_kind: "asset",
95304
+ asset_kind: at.asset_kind,
95305
+ asset_path: at.path,
95306
+ runnable_kind: "script",
95307
+ runnable_path: s2.path
95308
+ });
95309
+ } else {
95310
+ triggers.push({
95311
+ trigger_kind: t2.kind,
95312
+ runnable_kind: "script",
95313
+ runnable_path: s2.path
95314
+ });
95315
+ }
95316
+ }
95317
+ }
95318
+ return {
95319
+ graph: {
95320
+ runnables,
95321
+ assets: [...assetSet.values()],
95322
+ edges,
95323
+ triggers
95324
+ },
95325
+ scripts: pipelineScripts
95326
+ };
95327
+ }
95328
+
95329
+ // src/commands/pipeline/pipeline.ts
95330
+ await __promiseAll([
95331
+ init_conf(),
95332
+ init_codebase(),
95333
+ init_metadata()
95334
+ ]);
95335
+ import { readFile as readFile3 } from "node:fs/promises";
95336
+
95337
+ // src/commands/pipeline/dev.ts
95338
+ init_mod3();
95339
+ init_colors2();
95340
+ init_open();
95341
+ init_wrapper();
95342
+ init_log();
95343
+ init_port_probe();
95344
+ init_config();
95345
+ await __promiseAll([
95346
+ init_auth(),
95347
+ init_context(),
95348
+ init_conf(),
95349
+ init_codebase()
95350
+ ]);
95351
+ import * as http4 from "node:http";
95352
+ import * as fs18 from "node:fs";
95353
+ import * as path24 from "node:path";
95354
+ import { createHash as createHash3, randomBytes as randomBytes2 } from "node:crypto";
95355
+ import process26 from "node:process";
95356
+ var PORT2 = 3201;
95357
+ var LISTEN_HOST = "127.0.0.1";
95358
+ async function stableWsToken(remote, workspaceId, root, folder, port) {
95359
+ const key = createHash3("sha256").update(`${remote}\x00${workspaceId}\x00${root}\x00${folder}\x00${port}`).digest("hex").slice(0, 32);
95360
+ let tokenFile;
95361
+ try {
95362
+ tokenFile = path24.join(await getConfigDirPath(), `pipeline-dev-${key}.token`);
95363
+ const existing = fs18.readFileSync(tokenFile, "utf-8").trim();
95364
+ if (existing)
95365
+ return existing;
95366
+ } catch {}
95367
+ const token = randomBytes2(24).toString("base64url");
95368
+ if (tokenFile) {
95369
+ try {
95370
+ fs18.writeFileSync(tokenFile, token, { mode: 384 });
95371
+ } catch {}
95372
+ }
95373
+ return token;
95374
+ }
95375
+ function resolveFolder(root, folderArg) {
95376
+ if (folderArg)
95377
+ return folderArg.replace(/^f\//, "").replace(/\/$/, "");
95378
+ const rel = path24.relative(root, process26.cwd()).replaceAll("\\", "/");
95379
+ if (rel === "" || rel.startsWith(".."))
95380
+ return;
95381
+ const segs = rel.split("/");
95382
+ if (segs[0] === "f" && segs[1])
95383
+ return segs[1];
95384
+ return;
95385
+ }
95386
+ async function dev3(opts, folderArg) {
95387
+ const root = workspaceRoot();
95388
+ const folder = resolveFolder(root, folderArg);
95389
+ if (!folder) {
95390
+ error(colors.red("Could not determine the pipeline folder. Pass it explicitly " + "(`wmill pipeline dev <folder>`) or run from inside an `f/<folder>` directory."));
95391
+ process26.exit(1);
95392
+ }
95393
+ const folderDir = path24.join(root, "f", folder);
95394
+ if (!fs18.existsSync(folderDir)) {
95395
+ error(colors.red(`Folder not found on disk: ${folderDir}`));
95396
+ process26.exit(1);
95397
+ }
95398
+ const workspace = await resolveWorkspace(opts);
95399
+ await requireLogin(opts);
95400
+ const merged = await mergeConfigWithConfigFile(opts);
95401
+ const codebases = await listSyncCodebases(merged);
95402
+ let tempScriptRefs;
95403
+ try {
95404
+ const { buildPreviewTempScriptRefs: buildPreviewTempScriptRefs2 } = await init_generate_metadata().then(() => exports_generate_metadata);
95405
+ tempScriptRefs = await buildPreviewTempScriptRefs2(workspace, merged, codebases, { kind: "all" });
95406
+ } catch {}
95407
+ const EMPTY_GRAPH = { runnables: [], assets: [], edges: [], triggers: [] };
95408
+ async function buildBundle() {
95409
+ const { graph, scripts } = await buildLocalPipelineGraph({
95410
+ root,
95411
+ folder,
95412
+ defaultTs: merged.defaultTs
95413
+ });
95414
+ return {
95415
+ type: "pipeline",
95416
+ folder,
95417
+ graph,
95418
+ scripts,
95419
+ temp_script_refs: tempScriptRefs
95420
+ };
95421
+ }
95422
+ let current;
95423
+ try {
95424
+ current = await buildBundle();
95425
+ } catch (e2) {
95426
+ error(colors.red(`Initial graph build failed: ${e2.message}`));
95427
+ current = { type: "pipeline", folder, graph: EMPTY_GRAPH, scripts: [], temp_script_refs: tempScriptRefs };
95428
+ }
95429
+ info(colors.blue(`Watching f/${folder} — ${current.scripts.length} pipeline script(s)`));
95430
+ const clients = new Set;
95431
+ function broadcast() {
95432
+ const msg = JSON.stringify(current);
95433
+ for (const ws of clients) {
95434
+ if (ws.readyState === import_websocket.default.OPEN)
95435
+ ws.send(msg);
95436
+ }
95437
+ }
95438
+ let timer;
95439
+ const watcher = fs18.watch(folderDir, { recursive: true });
95440
+ watcher.on("change", (_ev, filename) => {
95441
+ if (filename && filename.toString().endsWith(".lock"))
95442
+ return;
95443
+ if (timer)
95444
+ clearTimeout(timer);
95445
+ timer = setTimeout(async () => {
95446
+ timer = undefined;
95447
+ try {
95448
+ current = await buildBundle();
95449
+ info(colors.cyan(`↻ rebuilt graph (${current.scripts.length} scripts)`));
95450
+ broadcast();
95451
+ } catch (e2) {
95452
+ error(colors.red(`Failed to rebuild pipeline graph: ${e2.message}`));
95453
+ }
95454
+ }, 150);
95455
+ });
95456
+ watcher.on("error", (e2) => error(colors.red(`Watcher error: ${e2.message}`)));
95457
+ const port = await resolveBindPort(opts.port ?? PORT2, "wmill pipeline dev", {
95458
+ info: (m3) => info(m3),
95459
+ warn: (m3) => warn(m3)
95460
+ });
95461
+ const server = http4.createServer((_req, res) => {
95462
+ res.writeHead(200);
95463
+ res.end();
95464
+ });
95465
+ const wsToken = await stableWsToken(workspace.remote, workspace.workspaceId, root, folder, port);
95466
+ const wss = new import_websocket_server.default({
95467
+ server,
95468
+ verifyClient: (info2) => {
95469
+ try {
95470
+ const u2 = new URL(info2.req.url ?? "", "http://localhost");
95471
+ return u2.searchParams.get("token") === wsToken;
95472
+ } catch {
95473
+ return false;
95474
+ }
95475
+ }
95476
+ });
95477
+ wss.on("connection", (ws) => {
95478
+ clients.add(ws);
95479
+ try {
95480
+ ws.send(JSON.stringify(current));
95481
+ } catch {}
95482
+ ws.on("close", () => clients.delete(ws));
95483
+ ws.on("error", () => clients.delete(ws));
95484
+ });
95485
+ const pageBase = (opts.frontend ?? workspace.remote).replace(/\/?$/, "/");
95486
+ const url = `${pageBase}pipeline_dev?workspace=${workspace.workspaceId}&wm_token=${workspace.token}&folder=${encodeURIComponent(folder)}&port=${port}&ws_token=${encodeURIComponent(wsToken)}`;
95487
+ server.listen(port, LISTEN_HOST, () => {
95488
+ info(colors.green.bold(`\uD83D\uDE80 Pipeline dev server on ws://localhost:${port}/ws`));
95489
+ info(colors.gray(`Open: ${url}`));
95490
+ if (opts.open !== false) {
95491
+ open_default(url).catch((e2) => warn(colors.yellow(`Failed to open browser: ${e2.message}`)));
95492
+ }
95493
+ });
95494
+ process26.on("SIGINT", () => {
95495
+ info(colors.yellow(`
95496
+ \uD83D\uDED1 Shutting down…`));
95497
+ watcher.close();
95498
+ for (const ws of clients)
95499
+ ws.close();
95500
+ server.close();
95501
+ process26.exit(0);
95502
+ });
95503
+ }
95504
+ var command41 = new Command().description("Live-preview a data pipeline from local files: watch an `f/<folder>` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).").arguments("[folder:string]").option("--port <port:number>", "Port for the dev WebSocket server.").option("--no-open", "Do not open the browser automatically.").option("--frontend <origin:string>", "Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.").action(dev3);
95505
+ var dev_default3 = command41;
95506
+
95507
+ // src/commands/pipeline/docs.ts
95508
+ init_gen();
95509
+ init_services_gen();
95510
+ init_log();
95511
+ init_colors2();
95512
+ await __promiseAll([
95513
+ init_auth(),
95514
+ init_context(),
95515
+ init_conf()
95516
+ ]);
95517
+ import { writeFile as writeFile25 } from "node:fs/promises";
95518
+ import { existsSync as existsSync16, readFileSync as readFileSync7 } from "node:fs";
95519
+ import * as path25 from "node:path";
95520
+ var ASSET_KINDS = "s3object,ducklake,datatable,volume";
95521
+ function assetUri(kind, p3) {
95522
+ const prefix = kind === "s3object" ? "s3" : kind;
95523
+ return `${prefix}://${p3}`;
95524
+ }
95525
+ async function fetchDeployedGraph(workspaceId, folder) {
95526
+ const res = await fetch(`${OpenAPI.BASE}/w/${workspaceId}/assets/graph?folder=${encodeURIComponent(folder)}&asset_kinds=${ASSET_KINDS}`, { headers: { Authorization: `Bearer ${OpenAPI.TOKEN}` } });
95527
+ if (!res.ok) {
95528
+ throw new Error(`GET assets/graph -> ${res.status}: ${await res.text()}`);
95529
+ }
95530
+ return await res.json();
95531
+ }
95532
+ function generatePipelineMarkdown(folder, graph, datatableSchemas, local) {
95533
+ const writesByScript = new Map;
95534
+ const readsByScript = new Map;
95535
+ for (const e2 of graph.edges) {
95536
+ if (e2.runnable_kind !== "script")
95537
+ continue;
95538
+ const uri = assetUri(e2.asset_kind, e2.asset_path);
95539
+ if (e2.access_type === "w" || e2.access_type === "rw") {
95540
+ (writesByScript.get(e2.runnable_path) ?? writesByScript.set(e2.runnable_path, []).get(e2.runnable_path)).push(uri);
95541
+ }
95542
+ if (e2.access_type === "r" || e2.access_type === "rw" || e2.access_type === undefined) {
95543
+ (readsByScript.get(e2.runnable_path) ?? readsByScript.set(e2.runnable_path, []).get(e2.runnable_path)).push(uri);
95544
+ }
95545
+ }
95546
+ const onByScript = new Map;
95547
+ const nativeByScript = new Map;
95548
+ for (const t2 of graph.triggers) {
95549
+ if (t2.runnable_kind !== "script")
95550
+ continue;
95551
+ if (t2.trigger_kind === "asset") {
95552
+ const at = t2;
95553
+ (onByScript.get(at.runnable_path) ?? onByScript.set(at.runnable_path, []).get(at.runnable_path)).push(assetUri(at.asset_kind, at.asset_path));
95554
+ } else {
95555
+ (nativeByScript.get(t2.runnable_path) ?? nativeByScript.set(t2.runnable_path, []).get(t2.runnable_path)).push(t2.trigger_kind);
95556
+ }
95557
+ }
95558
+ const scripts = graph.runnables.filter((r2) => r2.usage_kind === "script").map((r2) => r2.path).sort();
95559
+ let md = `# Pipeline \`f/${folder}\`
95560
+
95561
+ ${local ? "_Built from local working-tree files (`// pipeline` scripts)._" : "_Built from the deployed workspace asset graph._"}
95562
+
95563
+ A data pipeline is a folder of scripts marked \`// pipeline\` and wired by asset
95564
+ annotations. A script subscribes to upstream data with \`// on <asset-uri>\` and
95565
+ produces data by reading/writing assets in its body (\`datatable://\`,
95566
+ \`ducklake://\`, \`s3://\`, \`volume://\`). The cascade runs a producer, then every
95567
+ downstream subscriber, in topological order.
95568
+
95569
+ - **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"}
95570
+
95571
+ ## Scripts
95572
+
95573
+ `;
95574
+ for (const s2 of scripts) {
95575
+ const on = onByScript.get(s2) ?? [];
95576
+ const native = nativeByScript.get(s2) ?? [];
95577
+ const writes = [...new Set(writesByScript.get(s2) ?? [])].sort();
95578
+ const reads = [...new Set(readsByScript.get(s2) ?? [])].sort();
95579
+ md += `### \`${s2}\`
95580
+
95581
+ `;
95582
+ if (native.length)
95583
+ md += `- **Triggers:** ${native.map((n2) => `\`${n2}\``).join(", ")}
95584
+ `;
95585
+ if (on.length)
95586
+ md += `- **On (subscribes to):** ${on.map((u2) => `\`${u2}\``).join(", ")}
95587
+ `;
95588
+ if (reads.length)
95589
+ md += `- **Reads:** ${reads.map((u2) => `\`${u2}\``).join(", ")}
95590
+ `;
95591
+ if (writes.length)
95592
+ md += `- **Writes:** ${writes.map((u2) => `\`${u2}\``).join(", ")}
95593
+ `;
95594
+ if (!native.length && !on.length && !reads.length && !writes.length) {
95595
+ md += `- _No declared triggers or asset IO._
95596
+ `;
95597
+ }
95598
+ md += `
95599
+ `;
95600
+ }
95601
+ const referencedDatatables = new Set(graph.assets.filter((a2) => a2.kind === "datatable").map((a2) => a2.path.split("/")[0]));
95602
+ const relevant = (datatableSchemas ?? []).filter((dt) => referencedDatatables.has(dt.datatable_name));
95603
+ if (relevant.length > 0) {
95604
+ md += `## Datatable schemas
95605
+
95606
+ `;
95607
+ for (const dt of relevant) {
95608
+ md += `### Datatable \`${dt.datatable_name}\`
95609
+
95610
+ `;
95611
+ if (dt.error) {
95612
+ md += `> ⚠️ ${dt.error}
95613
+
95614
+ `;
95615
+ continue;
95616
+ }
95617
+ for (const [schemaName, tables3] of Object.entries(dt.schemas ?? {})) {
95618
+ for (const [tableName, columns] of Object.entries(tables3)) {
95619
+ const ref = schemaName === "public" ? `${dt.datatable_name}/${tableName}` : `${dt.datatable_name}/${schemaName}:${tableName}`;
95620
+ md += `- \`datatable://${ref}\`
95621
+ `;
95622
+ for (const [col, type] of Object.entries(columns)) {
95623
+ md += ` - \`${col}\`: ${type}
95624
+ `;
95625
+ }
95626
+ }
95627
+ }
95628
+ md += `
95629
+ `;
95630
+ }
95631
+ }
95632
+ md += `## Working with this pipeline
95633
+
95634
+ - **Inspect the graph:** \`wmill pipeline show ${folder} --local\`
95635
+ - **Run the cascade locally (no deploy):** \`wmill pipeline run ${folder} --local\`
95636
+ (optionally \`--from <script> --to <script>\` to bound it, \`--dry-run\` to preview the plan)
95637
+ - **Live visual preview while editing:** \`wmill pipeline dev ${folder}\`
95638
+ - **Deploy:** \`wmill sync push\`
95639
+
95640
+ Mark a script as part of this pipeline with a bare \`// pipeline\` comment
95641
+ (\`#\` for Python, \`--\` for SQL). Add \`// on <asset-uri>\` lines to subscribe it to
95642
+ upstream assets.
95643
+
95644
+ ---
95645
+ *Generated by \`wmill pipeline docs\`.*
95646
+ `;
95647
+ return md;
95648
+ }
95649
+ async function generatePipelineDocs(opts, folder) {
95650
+ const workspace = await resolveWorkspace(opts);
95651
+ await requireLogin(opts);
95652
+ const f3 = folder.replace(/^f\//, "").replace(/\/$/, "");
95653
+ if (f3.split("/").includes("..")) {
95654
+ error(colors.red(`Invalid folder '${folder}': '..' path segments are not allowed.`));
95655
+ return;
95656
+ }
95657
+ const root = workspaceRoot();
95658
+ const merged = await mergeConfigWithConfigFile(opts);
95659
+ const graph = opts.local ? (await buildLocalPipelineGraph({ root, folder: f3, defaultTs: merged.defaultTs })).graph : await fetchDeployedGraph(workspace.workspaceId, f3);
95660
+ if (graph.runnables.length === 0) {
95661
+ if (!opts.local) {
95662
+ try {
95663
+ const localCount = (await buildLocalPipelineGraph({ root, folder: f3, defaultTs: merged.defaultTs })).graph.runnables.length;
95664
+ if (localCount > 0) {
95665
+ info(`No deployed pipeline scripts in f/${f3}, but ${localCount} \`// pipeline\` script(s) exist in the local working tree — run \`wmill pipeline docs ${f3} --local\` to document those.`);
95666
+ return;
95667
+ }
95668
+ } catch {}
95669
+ }
95670
+ info(`No pipeline scripts in f/${f3}.`);
95671
+ return;
95672
+ }
95673
+ let datatableSchemas = [];
95674
+ try {
95675
+ datatableSchemas = await listDataTableSchemas({ workspace: workspace.workspaceId });
95676
+ } catch (err) {
95677
+ warn(colors.yellow(`Could not fetch datatable schemas: ${err.message}`));
95678
+ }
95679
+ const md = generatePipelineMarkdown(f3, graph, datatableSchemas, !!opts.local);
95680
+ const folderDir = path25.join(root, "f", f3);
95681
+ await writeFile25(path25.join(folderDir, "PIPELINE.md"), md, "utf-8");
95682
+ const written = ["PIPELINE.md"];
95683
+ const pointers = [
95684
+ ["AGENTS.md", `See @PIPELINE.md for this pipeline's graph, assets, and how to run it.
95685
+ `],
95686
+ ["CLAUDE.md", `Instructions are in @PIPELINE.md
95687
+ `]
95688
+ ];
95689
+ for (const [name, content] of pointers) {
95690
+ const p3 = path25.join(folderDir, name);
95691
+ if (existsSync16(p3)) {
95692
+ let existing;
95693
+ try {
95694
+ existing = readFileSync7(p3, "utf-8");
95695
+ } catch {
95696
+ continue;
95697
+ }
95698
+ if (existing.trim() !== content.trim()) {
95699
+ warn(colors.yellow(`Kept existing f/${f3}/${name} (hand-written — not the generated pointer)`));
95700
+ continue;
95701
+ }
95702
+ }
95703
+ await writeFile25(p3, content, "utf-8");
95704
+ written.push(name);
95705
+ }
95706
+ info(colors.green(`✓ Wrote ${written.join(", ")} to f/${f3}`));
95707
+ }
95708
+
95709
+ // src/commands/pipeline/pipelineUpload.ts
95710
+ import { basename as basename10 } from "node:path";
95711
+ function parseUploadBinding(spec) {
95712
+ const eq = spec.indexOf("=");
95713
+ const left = eq < 0 ? "" : spec.slice(0, eq).trim();
95714
+ const source = eq < 0 ? "" : spec.slice(eq + 1).trim();
95715
+ if (!left || !source) {
95716
+ throw new Error(`--upload '${spec}' must be <script>[:<param>]=<file-or-s3-uri>`);
95717
+ }
95718
+ const colon = left.indexOf(":");
95719
+ if (colon < 0)
95720
+ return { scriptTok: left, source };
95721
+ const scriptTok = left.slice(0, colon).trim();
95722
+ const param = left.slice(colon + 1).trim();
95723
+ if (!scriptTok || !param) {
95724
+ throw new Error(`--upload '${spec}' must be <script>[:<param>]=<file-or-s3-uri>`);
95725
+ }
95726
+ return { scriptTok, param, source };
95727
+ }
95728
+ function s3ObjectParams(schema) {
95729
+ const props = schema?.properties ?? {};
95730
+ return Object.keys(props).filter((k2) => props[k2]?.format === "resource-s3_object");
95731
+ }
95732
+ function devUploadKey(scriptPath, param, source) {
95733
+ return `wmilldev/pipeline/${scriptPath}/${param}/${basename10(source)}`;
95734
+ }
95735
+ function parseS3Uri(source) {
95736
+ const m3 = source.match(/^s3:\/\/([^/]*)\/(.*)$/);
95737
+ if (!m3)
95738
+ return { s3: source.slice("s3://".length) };
95739
+ const storage = m3[1] || undefined;
95740
+ return storage ? { s3: m3[2], storage } : { s3: m3[2] };
95741
+ }
95742
+ function s3Arg(param, obj) {
95743
+ return { [param]: obj };
95744
+ }
95745
+
94934
95746
  // src/commands/pipeline/pipeline.ts
94935
- async function apiGet(path23) {
94936
- const response = await fetch(`${OpenAPI.BASE}${path23}`, {
95747
+ async function apiGet(path26) {
95748
+ const response = await fetch(`${OpenAPI.BASE}${path26}`, {
94937
95749
  headers: { Authorization: `Bearer ${OpenAPI.TOKEN}` }
94938
95750
  });
94939
95751
  if (!response.ok) {
94940
95752
  const body = await response.text();
94941
- throw new Error(`GET ${path23} -> ${response.status}: ${body}`);
95753
+ throw new Error(`GET ${path26} -> ${response.status}: ${body}`);
94942
95754
  }
94943
95755
  return await response.json();
94944
95756
  }
@@ -94956,10 +95768,10 @@ async function list18(opts) {
94956
95768
  new Table2().header(["Folder", "Scripts"]).padding(2).border(true).body(items.map((p3) => [`f/${p3.folder}`, String(p3.script_count)])).render();
94957
95769
  }
94958
95770
  }
94959
- var ASSET_KINDS = "s3object,ducklake,datatable,volume";
94960
- function assetUri(kind, path23) {
95771
+ var ASSET_KINDS2 = "s3object,ducklake,datatable,volume";
95772
+ function assetUri2(kind, path26) {
94961
95773
  const prefix = kind === "s3object" ? "s3" : kind;
94962
- return `${prefix}://${path23}`;
95774
+ return `${prefix}://${path26}`;
94963
95775
  }
94964
95776
  function shortName(scriptPath) {
94965
95777
  return scriptPath.split("/").pop() ?? scriptPath;
@@ -94970,22 +95782,88 @@ function pushTo(map, key, val) {
94970
95782
  async function show2(opts, folder) {
94971
95783
  if (opts.json)
94972
95784
  setSilent(true);
94973
- const workspace = await resolveWorkspace(opts);
94974
- await requireLogin(opts);
94975
95785
  const f3 = folder.replace(/^f\//, "").replace(/\/$/, "");
94976
- const graph = await apiGet(`/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f3)}&asset_kinds=${ASSET_KINDS}`);
95786
+ let graph;
95787
+ let enrich;
95788
+ if (opts.local) {
95789
+ const merged = await mergeConfigWithConfigFile(opts);
95790
+ graph = (await buildLocalPipelineGraph({ root: workspaceRoot(), folder: f3, defaultTs: merged.defaultTs })).graph;
95791
+ } else {
95792
+ const workspace = await resolveWorkspace(opts);
95793
+ await requireLogin(opts);
95794
+ graph = await apiGet(`/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f3)}&asset_kinds=${ASSET_KINDS2}`);
95795
+ enrich = (nativeByScript, roots) => enrichRootMarkers(workspace.workspaceId, graph, nativeByScript, roots);
95796
+ }
94977
95797
  if (opts.json) {
94978
95798
  console.log(JSON.stringify(graph));
94979
95799
  return;
94980
95800
  }
94981
95801
  if (graph.runnables.length === 0) {
94982
- info(`No pipeline scripts in f/${f3}. Mark scripts with a \`// pipeline\` comment and push them.`);
95802
+ info(opts.local ? `No \`// pipeline\` scripts found in f/${f3} locally. Mark scripts with a \`// pipeline\` comment.` : `No pipeline scripts in f/${f3}. Mark scripts with a \`// pipeline\` comment and push them.`);
94983
95803
  return;
94984
95804
  }
95805
+ await renderGraph(graph, f3, enrich);
95806
+ }
95807
+ var MARKER_KINDS = new Set(["data_upload", "webhook", "email"]);
95808
+ function recoverHeaderMarkers(content) {
95809
+ const found = [];
95810
+ for (const line of content.split(`
95811
+ `)) {
95812
+ const trimmed = line.trim();
95813
+ if (trimmed === "")
95814
+ continue;
95815
+ if (!trimmed.startsWith("//") && !trimmed.startsWith("--") && !trimmed.startsWith("#"))
95816
+ break;
95817
+ const m3 = line.match(/^\s*(?:\/\/|--|#)\s*on\s+(\w+)\s*$/);
95818
+ if (m3 && MARKER_KINDS.has(m3[1]) && !found.includes(m3[1]))
95819
+ found.push(m3[1]);
95820
+ }
95821
+ return found;
95822
+ }
95823
+ async function enrichRootMarkers(workspaceId, graph, nativeByScript, roots) {
95824
+ await Promise.all(roots.map(async (p3) => {
95825
+ const r2 = graph.runnables.find((x3) => x3.path === p3);
95826
+ if (r2?.usage_kind !== "script")
95827
+ return;
95828
+ try {
95829
+ const script = await getScriptByPath({ workspace: workspaceId, path: p3 });
95830
+ const existing = nativeByScript.get(p3) ?? [];
95831
+ for (const kind of recoverHeaderMarkers(script.content ?? "")) {
95832
+ if (!existing.some((t2) => t2.kind === kind))
95833
+ existing.push({ kind });
95834
+ }
95835
+ if (existing.length > 0)
95836
+ nativeByScript.set(p3, existing);
95837
+ } catch {}
95838
+ }));
95839
+ }
95840
+ async function enrichDeployedNonAutorunTriggers(workspaceId, graph) {
95841
+ const scripts = (graph.runnables ?? []).filter((r2) => r2.usage_kind === "script");
95842
+ await Promise.all(scripts.map(async (r2) => {
95843
+ const known = new Set((graph.triggers ?? []).filter((t2) => t2.runnable_path === r2.path && MARKER_KINDS.has(t2.trigger_kind)).map((t2) => t2.trigger_kind));
95844
+ let script;
95845
+ try {
95846
+ script = await getScriptByPath({ workspace: workspaceId, path: r2.path });
95847
+ } catch (e2) {
95848
+ throw new Error(`Could not verify triggers for ${r2.path} (${e2?.body ?? e2?.message ?? e2}) — ` + `aborting so an input-only entrypoint isn't run with empty args; retry once resolved.`);
95849
+ }
95850
+ for (const kind of recoverHeaderMarkers(script.content ?? "")) {
95851
+ if (known.has(kind))
95852
+ continue;
95853
+ known.add(kind);
95854
+ graph.triggers.push({
95855
+ trigger_kind: kind,
95856
+ runnable_kind: "script",
95857
+ runnable_path: r2.path
95858
+ });
95859
+ }
95860
+ }));
95861
+ }
95862
+ async function renderGraph(graph, f3, enrich) {
94985
95863
  const writesByScript = new Map;
94986
95864
  for (const e2 of graph.edges) {
94987
95865
  if (e2.access_type === "w" || e2.access_type === "rw") {
94988
- const uri = assetUri(e2.asset_kind, e2.asset_path);
95866
+ const uri = assetUri2(e2.asset_kind, e2.asset_path);
94989
95867
  pushTo(writesByScript, e2.runnable_path, uri);
94990
95868
  }
94991
95869
  }
@@ -94995,7 +95873,7 @@ async function show2(opts, folder) {
94995
95873
  for (const t2 of graph.triggers) {
94996
95874
  if (t2.trigger_kind === "asset") {
94997
95875
  const at = t2;
94998
- const uri = assetUri(at.asset_kind, at.asset_path);
95876
+ const uri = assetUri2(at.asset_kind, at.asset_path);
94999
95877
  pushTo(subsByAsset, uri, t2.runnable_path);
95000
95878
  pushTo(subsByScript, t2.runnable_path, uri);
95001
95879
  } else {
@@ -95046,33 +95924,8 @@ async function show2(opts, folder) {
95046
95924
  });
95047
95925
  }
95048
95926
  const roots = graph.runnables.map((r2) => r2.path).filter((p3) => !subsByScript.get(p3)?.length).sort();
95049
- const MARKER_KINDS = ["data_upload", "webhook", "email"];
95050
- await Promise.all(roots.map(async (p3) => {
95051
- const r2 = graph.runnables.find((x3) => x3.path === p3);
95052
- if (r2?.usage_kind !== "script")
95053
- return;
95054
- try {
95055
- const script = await getScriptByPath({
95056
- workspace: workspace.workspaceId,
95057
- path: p3
95058
- });
95059
- const existing = nativeByScript.get(p3) ?? [];
95060
- for (const line of (script.content ?? "").split(`
95061
- `)) {
95062
- const m3 = line.match(/^\s*(?:\/\/|--|#)\s*on\s+(\w+)\s*$/);
95063
- if (!m3)
95064
- continue;
95065
- const kind = m3[1];
95066
- if (!MARKER_KINDS.includes(kind))
95067
- continue;
95068
- if (!existing.some((t2) => t2.kind === kind)) {
95069
- existing.push({ kind });
95070
- }
95071
- }
95072
- if (existing.length > 0)
95073
- nativeByScript.set(p3, existing);
95074
- } catch {}
95075
- }));
95927
+ if (enrich)
95928
+ await enrich(nativeByScript, roots);
95076
95929
  const scriptCount = graph.runnables.length;
95077
95930
  const assetCount = graph.assets.length;
95078
95931
  info(colors.bold(`Pipeline f/${f3}`) + colors.dim(` — ${scriptCount} script${scriptCount === 1 ? "" : "s"} · ${assetCount} asset${assetCount === 1 ? "" : "s"}`));
@@ -95106,14 +95959,101 @@ async function waitJob(workspace, id) {
95106
95959
  }
95107
95960
  throw new Error(`Timed out waiting for job ${id}`);
95108
95961
  }
95962
+ async function resolveUploadArgs(bindings, scriptPath, workspaceId, local, localScripts) {
95963
+ let schema;
95964
+ const loadSchema = async () => {
95965
+ if (schema)
95966
+ return schema;
95967
+ if (local) {
95968
+ const ls = localScripts?.get(scriptPath);
95969
+ if (!ls) {
95970
+ throw new Error(`No local file for ${scriptPath} to infer its S3Object parameter.`);
95971
+ }
95972
+ schema = (await inferSchema(ls.language, ls.content, {}, scriptPath)).schema;
95973
+ } else {
95974
+ schema = (await getScriptByPath({ workspace: workspaceId, path: scriptPath })).schema;
95975
+ }
95976
+ return schema;
95977
+ };
95978
+ const args = {};
95979
+ for (const binding of bindings) {
95980
+ let param = binding.param;
95981
+ if (!param) {
95982
+ const params = s3ObjectParams(await loadSchema());
95983
+ if (params.length === 1)
95984
+ param = params[0];
95985
+ else if (params.length === 0) {
95986
+ throw new Error(`${scriptPath} declares no S3Object parameter to bind --upload to.`);
95987
+ } else {
95988
+ throw new Error(`${scriptPath} has multiple S3Object parameters (${params.join(", ")}) — pick one with --upload ${binding.scriptTok}:<param>=${binding.source}`);
95989
+ }
95990
+ }
95991
+ if (param in args) {
95992
+ throw new Error(`--upload binds ${scriptPath}:${param} more than once.`);
95993
+ }
95994
+ let obj;
95995
+ if (binding.source.startsWith("s3://")) {
95996
+ obj = parseS3Uri(binding.source);
95997
+ } else {
95998
+ const buf = await readFile3(binding.source);
95999
+ const key = devUploadKey(scriptPath, param, binding.source);
96000
+ await fileUpload({
96001
+ workspace: workspaceId,
96002
+ fileKey: key,
96003
+ requestBody: new Blob([buf])
96004
+ });
96005
+ info(colors.gray(` ↑ ${binding.source} → s3://${key}`));
96006
+ obj = { s3: key };
96007
+ }
96008
+ Object.assign(args, s3Arg(param, obj));
96009
+ }
96010
+ return args;
96011
+ }
95109
96012
  async function run5(opts, folder) {
95110
96013
  if (opts.json)
95111
96014
  setSilent(true);
95112
96015
  const workspace = await resolveWorkspace(opts);
95113
96016
  await requireLogin(opts);
95114
96017
  const f3 = folder.replace(/^f\//, "").replace(/\/$/, "");
95115
- const graph = await apiGet(`/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f3)}&asset_kinds=${ASSET_KINDS}`);
95116
- const starts = validStarts(graph);
96018
+ let graph;
96019
+ let localScripts;
96020
+ let tempScriptRefs;
96021
+ if (opts.local) {
96022
+ const merged = await mergeConfigWithConfigFile(opts);
96023
+ const built = await buildLocalPipelineGraph({
96024
+ root: workspaceRoot(),
96025
+ folder: f3,
96026
+ defaultTs: merged.defaultTs
96027
+ });
96028
+ graph = built.graph;
96029
+ localScripts = new Map(built.scripts.map((s2) => [s2.path, s2]));
96030
+ try {
96031
+ const codebases = await listSyncCodebases(merged);
96032
+ const { buildPreviewTempScriptRefs: buildPreviewTempScriptRefs2 } = await init_generate_metadata().then(() => exports_generate_metadata);
96033
+ tempScriptRefs = await buildPreviewTempScriptRefs2(workspace, merged, codebases, { kind: "all" });
96034
+ } catch {}
96035
+ } else {
96036
+ graph = await apiGet(`/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f3)}&asset_kinds=${ASSET_KINDS2}`);
96037
+ await enrichDeployedNonAutorunTriggers(workspace.workspaceId, graph);
96038
+ }
96039
+ const boundBindingsByPath = new Map;
96040
+ const boundNodeIds = new Set;
96041
+ for (const spec of opts.upload ?? []) {
96042
+ const binding = parseUploadBinding(spec);
96043
+ const id = resolveToken(graph, binding.scriptTok);
96044
+ if (!id || !id.startsWith("script:")) {
96045
+ const matches = graph.runnables.filter((r2) => r2.usage_kind === "script" && (r2.path.split("/").pop() ?? r2.path) === binding.scriptTok);
96046
+ if (matches.length > 1) {
96047
+ throw new Error(`--upload '${binding.scriptTok}' matches multiple scripts (${matches.map((r2) => r2.path).sort().join(", ")}) — use the full path.`);
96048
+ }
96049
+ throw new Error(`--upload '${binding.scriptTok}' matched no script in f/${f3}.`);
96050
+ }
96051
+ const p3 = scriptPathOf(id);
96052
+ (boundBindingsByPath.get(p3) ?? boundBindingsByPath.set(p3, []).get(p3)).push(binding);
96053
+ boundNodeIds.add(id);
96054
+ }
96055
+ const starts = new Set([...validStarts(graph), ...boundNodeIds]);
96056
+ let runAll = false;
95117
96057
  let start;
95118
96058
  if (opts.from) {
95119
96059
  const resolved = resolveToken(graph, opts.from);
@@ -95132,8 +96072,10 @@ async function run5(opts, folder) {
95132
96072
  start = [...starts][0];
95133
96073
  } else if (starts.size === 0) {
95134
96074
  throw new Error(`No schedule or manual root in f/${f3} to start a bounded run from.`);
96075
+ } else if ((opts.to ?? []).length === 0) {
96076
+ runAll = true;
95135
96077
  } else {
95136
- throw new Error(`f/${f3} has ${starts.size} possible starts — pass --from <script>. Candidates: ${scriptsOf(starts).sort().join(", ")}`);
96078
+ throw new Error(`f/${f3} has ${starts.size} possible starts — pass --from <script> when using --to. Candidates: ${scriptsOf(starts).sort().join(", ")}`);
95137
96079
  }
95138
96080
  const toTokens = (opts.to ?? []).flatMap((t2) => t2.split(",")).map((t2) => t2.trim()).filter(Boolean);
95139
96081
  const ends = [];
@@ -95154,34 +96096,50 @@ async function run5(opts, folder) {
95154
96096
  }
95155
96097
  const idLabel = (id) => id.startsWith("script:") ? scriptPathOf(id) : id;
95156
96098
  const dag = buildLineageDag(graph);
96099
+ const barriers = new Set([...nonAutorunTriggerScripts(graph)].filter((id) => !starts.has(id)));
95157
96100
  let selectedScripts;
95158
96101
  let reachableEnds = [];
95159
96102
  let droppedEnds = [];
95160
- if (ends.length === 0) {
95161
- const all = new Set(descendants(dag, start));
95162
- all.add(start);
95163
- selectedScripts = new Set(scriptsOf(all));
96103
+ if (runAll) {
96104
+ selectedScripts = new Set(scriptsOf(reachableCutting(dag, starts, barriers)));
96105
+ } else if (ends.length === 0) {
96106
+ selectedScripts = new Set(scriptsOf(reachableCutting(dag, [start], barriers)));
95164
96107
  } else {
95165
96108
  const res = boundedSet(dag, start, ends);
95166
- reachableEnds = res.reachableEnds;
95167
96109
  droppedEnds = res.droppedEnds;
95168
96110
  for (const d3 of droppedEnds) {
95169
96111
  warn(`end '${idLabel(d3)}' is not downstream of the start — ignored.`);
95170
96112
  }
95171
- selectedScripts = new Set(scriptsOf(res.nodes));
96113
+ const reachable = reachableCutting(dag, [start], barriers);
96114
+ selectedScripts = new Set(scriptsOf([...res.nodes].filter((n2) => reachable.has(n2))));
96115
+ for (const e2 of res.reachableEnds) {
96116
+ if (reachable.has(e2)) {
96117
+ reachableEnds.push(e2);
96118
+ } else {
96119
+ droppedEnds.push(e2);
96120
+ warn(`end '${idLabel(e2)}' is only reachable through a skipped input/event handler — not run (bind it with --upload to include it).`);
96121
+ }
96122
+ }
95172
96123
  }
95173
96124
  const { order, cyclic } = topoOrder(graph, selectedScripts);
95174
96125
  if (cyclic.length > 0) {
95175
96126
  warn(`Skipping ${cyclic.length} script(s) on a dependency cycle: ${cyclic.sort().join(", ")}`);
95176
96127
  }
96128
+ const orderSet = new Set(order);
96129
+ const boundInOrder = [...boundBindingsByPath.keys()].filter((p3) => orderSet.has(p3)).sort();
96130
+ for (const p3 of boundBindingsByPath.keys()) {
96131
+ if (!orderSet.has(p3))
96132
+ warn(`--upload for ${p3} is unused — it isn't in the run selection.`);
96133
+ }
95177
96134
  if (opts.json) {
95178
96135
  console.log(JSON.stringify({
95179
- start: scriptPathOf(start),
96136
+ start: runAll ? null : scriptPathOf(start),
95180
96137
  ends: ends.map(idLabel),
95181
96138
  reachableEnds: reachableEnds.map(idLabel),
95182
96139
  droppedEnds: droppedEnds.map(idLabel),
95183
96140
  order,
95184
- cyclic
96141
+ cyclic,
96142
+ uploads: boundInOrder
95185
96143
  }));
95186
96144
  }
95187
96145
  if (order.length === 0) {
@@ -95191,32 +96149,56 @@ async function run5(opts, folder) {
95191
96149
  }
95192
96150
  if (opts.dryRun) {
95193
96151
  if (!opts.json) {
95194
- info(colors.bold(`Bounded run plan — ${order.length} script${order.length === 1 ? "" : "s"}`) + colors.dim(` (from ${shortName(scriptPathOf(start))})`));
95195
- order.forEach((p3, i) => info(` ${i + 1}. ${p3}`));
96152
+ info(colors.bold(`Bounded run plan — ${order.length} script${order.length === 1 ? "" : "s"}`) + colors.dim(runAll ? ` (whole pipeline)` : ` (from ${shortName(scriptPathOf(start))})`));
96153
+ order.forEach((p3, i) => info(` ${i + 1}. ${p3}${boundBindingsByPath.has(p3) ? colors.dim(" (← --upload)") : ""}`));
95196
96154
  }
95197
96155
  return;
95198
96156
  }
95199
- for (const path23 of order) {
96157
+ const uploadArgs = new Map;
96158
+ for (const nodePath2 of boundInOrder) {
96159
+ uploadArgs.set(nodePath2, await resolveUploadArgs(boundBindingsByPath.get(nodePath2), nodePath2, workspace.workspaceId, opts.local, localScripts));
96160
+ }
96161
+ for (const nodePath2 of order) {
95200
96162
  if (!opts.json)
95201
- info(colors.gray(`▶ running ${path23}…`));
95202
- const id = await runScriptByPath({
95203
- workspace: workspace.workspaceId,
95204
- path: path23,
95205
- requestBody: { _wmill_skip_asset_dispatch: true }
95206
- });
96163
+ info(colors.gray(`▶ running ${nodePath2}…`));
96164
+ const nodeArgs = { ...uploadArgs.get(nodePath2) ?? {}, _wmill_skip_asset_dispatch: true };
96165
+ let id;
96166
+ if (opts.local) {
96167
+ const ls = localScripts.get(nodePath2);
96168
+ if (!ls) {
96169
+ throw new Error(`No local file for ${nodePath2} — is it a \`// pipeline\` script in f/${f3}?`);
96170
+ }
96171
+ id = await runScriptPreview({
96172
+ workspace: workspace.workspaceId,
96173
+ requestBody: {
96174
+ content: ls.content,
96175
+ language: ls.language,
96176
+ path: nodePath2,
96177
+ args: nodeArgs,
96178
+ temp_script_refs: tempScriptRefs,
96179
+ ...ls.tag ? { tag: ls.tag } : {}
96180
+ }
96181
+ });
96182
+ } else {
96183
+ id = await runScriptByPath({
96184
+ workspace: workspace.workspaceId,
96185
+ path: nodePath2,
96186
+ requestBody: nodeArgs
96187
+ });
96188
+ }
95207
96189
  const ok = await waitJob(workspace.workspaceId, id);
95208
96190
  if (!ok) {
95209
- throw new Error(`Bounded run failed at ${path23} (job ${id}).`);
96191
+ throw new Error(`Bounded run failed at ${nodePath2} (job ${id}).`);
95210
96192
  }
95211
96193
  if (!opts.json)
95212
- info(colors.green(` ✓ ${path23}`));
96194
+ info(colors.green(` ✓ ${nodePath2}`));
95213
96195
  }
95214
96196
  if (!opts.json) {
95215
96197
  info(colors.green.bold(`Bounded run complete — ${order.length} script(s) succeeded.`));
95216
96198
  }
95217
96199
  }
95218
- var command41 = new Command().description("inspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on <spec>` annotations)").command("list", "list pipeline folders in the workspace").option("--json", "Output as JSON (for piping to jq)").action(list18).command("show", "render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal").arguments("<folder:string>").option("--json", "Output the raw asset graph as JSON").action(show2).command("run", "run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s)").arguments("<folder:string>").option("--from <script:string>", "Start script (short name or path). Defaults to the folder's sole schedule/manual root.").option("--to <node:string>", "End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream.", { collect: true }).option("--dry-run", "Print the topological run plan without executing.").option("--json", "Output the plan as JSON (for piping to jq).").action(run5);
95219
- var pipeline_default = command41;
96200
+ var command42 = new Command().description("inspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on <spec>` annotations)").command("list", "list pipeline folders in the workspace").option("--json", "Output as JSON (for piping to jq)").action(list18).command("show", "render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal").arguments("<folder:string>").option("--json", "Output the raw asset graph as JSON").option("--local", "Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace — no deploy needed.").action(show2).command("run", "run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s)").arguments("<folder:string>").option("--from <script:string>", "Start script (short name or path). Defaults to the folder's sole schedule/manual root.").option("--to <node:string>", "End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream.", { collect: true }).option("--dry-run", "Print the topological run plan without executing.").option("--json", "Output the plan as JSON (for piping to jq).").option("--local", "Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files.").option("--upload <binding:string>", "Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable.", { collect: true }).action(run5).command("docs", "generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop").arguments("<folder:string>").option("--local", "Build the graph from local working-tree files instead of the deployed workspace.").action(generatePipelineDocs).command("dev", dev_default3);
96201
+ var pipeline_default = command42;
95220
96202
 
95221
96203
  // src/commands/ducklake/ducklake.ts
95222
96204
  init_mod3();
@@ -95246,8 +96228,8 @@ async function run6(opts, sql) {
95246
96228
  const name = opts.name ?? DEFAULT_DUCKLAKE_NAME;
95247
96229
  await runCatalogQuery(opts, "ducklake", name, sql);
95248
96230
  }
95249
- var command42 = new Command().description("ducklake related commands").command("list", "list all ducklakes in the workspace").option("--json", "Output as JSON (for piping to jq)").action(list19).command("run", "run a SQL query on a ducklake").arguments("<sql:string>").option("-n --name <name:string>", "Ducklake name (default: main)").option("-s --silent", "Output only the final result as JSON. Useful for scripting.").action(run6);
95250
- var ducklake_default = command42;
96231
+ var command43 = new Command().description("ducklake related commands").command("list", "list all ducklakes in the workspace").option("--json", "Output as JSON (for piping to jq)").action(list19).command("run", "run a SQL query on a ducklake").arguments("<sql:string>").option("-n --name <name:string>", "Ducklake name (default: main)").option("-s --silent", "Output only the final result as JSON. Useful for scripting.").action(run6);
96232
+ var ducklake_default = command43;
95251
96233
 
95252
96234
  // src/commands/object-storage/object-storage.ts
95253
96235
  init_mod3();
@@ -95262,8 +96244,8 @@ await __promiseAll([
95262
96244
  init_utils()
95263
96245
  ]);
95264
96246
  import { Buffer as Buffer5 } from "node:buffer";
95265
- import { readFile as readFile3, writeFile as writeFile25 } from "node:fs/promises";
95266
- import { basename as basename10 } from "node:path";
96247
+ import { readFile as readFile4, writeFile as writeFile26 } from "node:fs/promises";
96248
+ import { basename as basename11 } from "node:path";
95267
96249
  function formatBytes(n2) {
95268
96250
  if (n2 == null)
95269
96251
  return "-";
@@ -95326,7 +96308,7 @@ More results available. Use --marker '${result2.next_marker}' to paginate.`);
95326
96308
  async function upload(opts, localPath, fileKey) {
95327
96309
  const workspace = await resolveWorkspace(opts);
95328
96310
  await requireLogin(opts);
95329
- const buf = await readFile3(localPath);
96311
+ const buf = await readFile4(localPath);
95330
96312
  const blob = new Blob([buf], { type: opts.contentType ?? "application/octet-stream" });
95331
96313
  await fileUpload({
95332
96314
  workspace: workspace.workspaceId,
@@ -95364,8 +96346,8 @@ async function download(opts, fileKey, outputPath) {
95364
96346
  process.stdout.write(buf);
95365
96347
  return;
95366
96348
  }
95367
- const dest = outputPath ?? basename10(fileKey);
95368
- await writeFile25(dest, buf);
96349
+ const dest = outputPath ?? basename11(fileKey);
96350
+ await writeFile26(dest, buf);
95369
96351
  info(colors.green(`Downloaded ${fileKey} -> ${dest}`));
95370
96352
  }
95371
96353
  async function del(opts, fileKey) {
@@ -95448,8 +96430,8 @@ async function preview3(opts, fileKey) {
95448
96430
  `);
95449
96431
  }
95450
96432
  }
95451
- var command43 = new Command().alias("s3").description("Object storage (S3) related commands. Operates on the workspace's default object storage; use --storage to target a configured secondary storage.").action(listStorages).command("list", "List configured object storages for the workspace (default + secondary).").option("--json", "Output as JSON (for piping to jq)").action(listStorages).command("files", "List files in an object storage. Optionally filter by prefix.").alias("ls").arguments("[prefix:string]").option("--json", "Output as JSON (for piping to jq)").option("--max-keys <maxKeys:number>", "Page size (default 100)").option("--marker <marker:string>", "Pagination marker from a previous response").option("--storage <storage:string>", "Secondary storage name (omit for the workspace default)").action(listFiles).command("upload", "Upload a local file to object storage at the given file key.").arguments("<local_path:string> <file_key:string>").option("--storage <storage:string>", "Secondary storage name").option("--content-type <contentType:string>", "Content-Type header to set on the object").option("--content-disposition <contentDisposition:string>", "Content-Disposition header to set on the object").action(upload).command("download", "Download an object to a local file (or stdout). Default output path is the basename of the file key in the current directory.").arguments("<file_key:string> [output_path:string]").option("--storage <storage:string>", "Secondary storage name").option("--stdout", "Write file contents to stdout instead of a file").action(download).command("delete", "Delete an object from object storage. Prompts for confirmation unless --yes is set.").arguments("<file_key:string>").option("--storage <storage:string>", "Secondary storage name").option("--yes", "Skip the confirmation prompt").action(del).command("move", "Move an object within the same storage (rename or relocate by key).").arguments("<src_file_key:string> <dest_file_key:string>").option("--storage <storage:string>", "Secondary storage name").action(move).command("info", "Show metadata (size, mime, last-modified) for an object.").arguments("<file_key:string>").option("--json", "Output as JSON (for piping to jq)").option("--storage <storage:string>", "Secondary storage name").action(info2).command("preview", "Preview the contents of an object (text/CSV). Use --bytes-from / --bytes-length to peek at a slice of binary files.").arguments("<file_key:string>").option("--storage <storage:string>", "Secondary storage name").option("--mime <mime:string>", "Override the detected mime type (e.g. text/csv)").option("--bytes-from <bytesFrom:number>", "Start offset in bytes").option("--bytes-length <bytesLength:number>", "Number of bytes to read").option("--csv-separator <csvSeparator:string>", "CSV column separator (default ,)").option("--csv-header", "Treat the first CSV row as a header").action(preview3);
95452
- var object_storage_default = command43;
96433
+ var command44 = new Command().alias("s3").description("Object storage (S3) related commands. Operates on the workspace's default object storage; use --storage to target a configured secondary storage.").action(listStorages).command("list", "List configured object storages for the workspace (default + secondary).").option("--json", "Output as JSON (for piping to jq)").action(listStorages).command("files", "List files in an object storage. Optionally filter by prefix.").alias("ls").arguments("[prefix:string]").option("--json", "Output as JSON (for piping to jq)").option("--max-keys <maxKeys:number>", "Page size (default 100)").option("--marker <marker:string>", "Pagination marker from a previous response").option("--storage <storage:string>", "Secondary storage name (omit for the workspace default)").action(listFiles).command("upload", "Upload a local file to object storage at the given file key.").arguments("<local_path:string> <file_key:string>").option("--storage <storage:string>", "Secondary storage name").option("--content-type <contentType:string>", "Content-Type header to set on the object").option("--content-disposition <contentDisposition:string>", "Content-Disposition header to set on the object").action(upload).command("download", "Download an object to a local file (or stdout). Default output path is the basename of the file key in the current directory.").arguments("<file_key:string> [output_path:string]").option("--storage <storage:string>", "Secondary storage name").option("--stdout", "Write file contents to stdout instead of a file").action(download).command("delete", "Delete an object from object storage. Prompts for confirmation unless --yes is set.").arguments("<file_key:string>").option("--storage <storage:string>", "Secondary storage name").option("--yes", "Skip the confirmation prompt").action(del).command("move", "Move an object within the same storage (rename or relocate by key).").arguments("<src_file_key:string> <dest_file_key:string>").option("--storage <storage:string>", "Secondary storage name").action(move).command("info", "Show metadata (size, mime, last-modified) for an object.").arguments("<file_key:string>").option("--json", "Output as JSON (for piping to jq)").option("--storage <storage:string>", "Secondary storage name").action(info2).command("preview", "Preview the contents of an object (text/CSV). Use --bytes-from / --bytes-length to peek at a slice of binary files.").arguments("<file_key:string>").option("--storage <storage:string>", "Secondary storage name").option("--mime <mime:string>", "Override the detected mime type (e.g. text/csv)").option("--bytes-from <bytesFrom:number>", "Start offset in bytes").option("--bytes-length <bytesLength:number>", "Number of bytes to read").option("--csv-separator <csvSeparator:string>", "CSV column separator (default ,)").option("--csv-header", "Treat the first CSV row as a header").action(preview3);
96434
+ var object_storage_default = command44;
95453
96435
 
95454
96436
  // src/main.ts
95455
96437
  await init_context();
@@ -95469,7 +96451,7 @@ async function checkVersionSafe(cmd) {
95469
96451
  }
95470
96452
  mainCommand.version(`${currentVersion} (New version available: ${latestVersion}. Run '${mainCommand.getName()} upgrade' to upgrade to the latest version!)`);
95471
96453
  }
95472
- var command44 = new Command().name("wmill").action(() => info(`Welcome to Windmill CLI ${VERSION}. Use -h for help.`)).description("Windmill CLI").globalOption("--workspace <workspace:string>", "Specify the target workspace. This overrides the default workspace.").globalOption("--debug --verbose", "Show debug/verbose logs").globalOption("--show-diffs", "Show diff informations when syncing (may show sensitive informations)").globalOption("--token <token:string>", "Specify an API token. This will override any stored token.").globalOption("--base-url <baseUrl:string>", "Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used.").globalOption("--config-dir <configDir:string>", "Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location.").env("HEADERS <headers:string>", `Specify headers to use for all requests. e.g: "HEADERS='h1: v1, h2: v2'"`).version(VERSION).versionOption(false).helpOption("-h, --help", "Show this help.", {
96454
+ var command45 = new Command().name("wmill").action(() => info(`Welcome to Windmill CLI ${VERSION}. Use -h for help.`)).description("Windmill CLI").globalOption("--workspace <workspace:string>", "Specify the target workspace. This overrides the default workspace.").globalOption("--debug --verbose", "Show debug/verbose logs").globalOption("--show-diffs", "Show diff informations when syncing (may show sensitive informations)").globalOption("--token <token:string>", "Specify an API token. This will override any stored token.").globalOption("--base-url <baseUrl:string>", "Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used.").globalOption("--config-dir <configDir:string>", "Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location.").env("HEADERS <headers:string>", `Specify headers to use for all requests. e.g: "HEADERS='h1: v1, h2: v2'"`).version(VERSION).versionOption(false).helpOption("-h, --help", "Show this help.", {
95473
96455
  action: async function() {
95474
96456
  const self2 = this;
95475
96457
  const long = self2.getRawArgs().includes(`--${self2.getHelpOption()?.name}`);
@@ -95511,20 +96493,20 @@ var command44 = new Command().name("wmill").action(() => info(`Welcome to Windmi
95511
96493
  error(e2);
95512
96494
  info("Try running with sudo and otherwise check the result of the command: npm uninstall windmill-cli && npm install -g windmill-cli");
95513
96495
  })).command("completions", new Command().description("Generate shell completions.").command("bash", new Command().description("Generate bash completions.").action(() => {
95514
- process.stdout.write(generateShellCompletions(command44, "bash") + `
96496
+ process.stdout.write(generateShellCompletions(command45, "bash") + `
95515
96497
  `);
95516
96498
  })).command("zsh", new Command().description("Generate zsh completions.").action(() => {
95517
- process.stdout.write(generateShellCompletions(command44, "zsh") + `
96499
+ process.stdout.write(generateShellCompletions(command45, "zsh") + `
95518
96500
  `);
95519
96501
  })).command("fish", new Command().description("Generate fish completions.").action(() => {
95520
- process.stdout.write(generateShellCompletions(command44, "fish") + `
96502
+ process.stdout.write(generateShellCompletions(command45, "fish") + `
95521
96503
  `);
95522
96504
  })));
95523
96505
  async function main2() {
95524
96506
  try {
95525
96507
  const args = process.argv.slice(2);
95526
96508
  if (args.length === 0) {
95527
- command44.showHelp();
96509
+ command45.showHelp();
95528
96510
  }
95529
96511
  const LOG_LEVEL = args.includes("--verbose") || args.includes("--debug") ? "DEBUG" : "INFO";
95530
96512
  setShowDiffs(args.includes("--show-diffs"));
@@ -95544,7 +96526,7 @@ async function main2() {
95544
96526
  const { warnIfTsconfigStale: warnIfTsconfigStale2 } = await init_tsconfig().then(() => exports_tsconfig);
95545
96527
  await warnIfTsconfigStale2().catch(() => {});
95546
96528
  }
95547
- await command44.parse(args);
96529
+ await command45.parse(args);
95548
96530
  } catch (e2) {
95549
96531
  if (e2 && typeof e2 === "object" && "name" in e2 && e2.name === "ApiError") {
95550
96532
  const body = e2.body;
@@ -95580,7 +96562,7 @@ if (isMain()) {
95580
96562
  process.stdin.destroy();
95581
96563
  });
95582
96564
  }
95583
- var main_default = command44;
96565
+ var main_default = command45;
95584
96566
  export {
95585
96567
  add as workspaceAdd,
95586
96568
  workspace_default as workspace,