wesl-packager 0.6.46 → 0.6.49

package/bin/wesl-packager.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env -S node --experimental-strip-types
+import "../src/main.ts";

package/package.json

@@ -1,28 +1,26 @@
 {
   "name": "wesl-packager",
-  "version": "0.6.46",
+  "version": "0.6.49",
   "type": "module",
-  "bin": "bin/wesl-packager",
+  "bin": "bin/wesl-packager.js",
   "files": [
-    "bin"
+    "bin",
+    "src"
   ],
   "repository": "github:wgsl-tooling-wg/wesl-js",
   "dependencies": {
     "@biomejs/js-api": "^1.0.0",
     "@biomejs/wasm-nodejs": "^2.0.6",
     "yargs": "^18.0.0",
-    "wesl": "0.6.46"
+    "wesl": "0.6.49",
+    "wesl-tooling": "0.6.12"
   },
   "devDependencies": {
     "dependent_package": "x",
     "dir-compare": "^5.0.0",
-    "picocolors": "^1.1.1",
-    "rollup": "^4.44.1",
-    "wesl-tooling": "x"
+    "picocolors": "^1.1.1"
   },
   "scripts": {
-    "build": "tsdown",
-    "build:watch": "tsdown --watch",
     "test": "cross-env FORCE_COLOR=1 vitest",
     "test:once": "vitest run",
     "typecheck": "tsgo"

package/src/PackageWesl.ts

@@ -0,0 +1,219 @@
+import fs, { mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { Biome, Distribution } from "@biomejs/js-api";
+import { noSuffix, sanitizePackageName, type WeslBundle } from "wesl";
+import { weslBundleDeclUrl } from "wesl/bundle-decl";
+import { loadModules, parseDependencies, zip } from "wesl-tooling";
+import type { CliArgs } from "./PackagerCli.ts";
+
+const weslBundleDecl = await readFile(new URL(weslBundleDeclUrl), "utf8");
+
+const { biome, projectKey: biomeKey } = await setupBiome();
+
+/** write weslBundle .js and .d.ts files for this shader */
+export async function packageWgsl(args: CliArgs): Promise<void> {
+  const { projectDir, outDir, multiBundle, baseDir, src } = args;
+  const modules = await loadModules(projectDir, baseDir, src);
+  const pkgJsonPath = path.join(projectDir, "package.json");
+  const { name } = await loadPackageFields(pkgJsonPath);
+  const edition = "unstable_2025_1";
+
+  if (args.multiBundle) {
+    await writeMultiBundle(modules, name, edition, projectDir, outDir);
+  } else {
+    const allDeps = parseDependencies(modules, projectDir);
+    const sanitized = sanitizePackageName(name);
+    const deps = filterSelfDeps(allDeps, sanitized);
+    const bundle: WeslBundle = { name: sanitized, edition, modules };
+    await writeJsBundle(bundle, deps, outDir);
+  }
+  await writeTypeScriptDts(outDir);
+  if (args.updatePackageJson) {
+    await updatePackageJson(projectDir, outDir, multiBundle);
+  }
+}
+
+/** add an 'exports' entry to package.json for the wesl bundles */
+async function updatePackageJson(
+  projectDir: string,
+  outDir: string,
+  multiBundle: boolean,
+): Promise<void> {
+  const pkgJsonPath = path.join(projectDir, "package.json");
+  const pkgJsonString = await fs.readFile(pkgJsonPath, { encoding: "utf8" });
+  const pkgJson = JSON.parse(pkgJsonString);
+  const exports: Record<string, any> = {};
+
+  const distDir = path.relative(projectDir, outDir);
+  if (multiBundle) {
+    exports["./*"] = {
+      types: `./${distDir}/weslBundle.d.ts`,
+      import: `./${distDir}/*/weslBundle.js`,
+    };
+  } else {
+    exports["."] = {
+      types: `./${distDir}/weslBundle.d.ts`,
+      import: `./${distDir}/weslBundle.js`,
+    };
+  }
+
+  const newPkgJson = insertExports(pkgJson, exports);
+  const jsonString = JSON.stringify(newPkgJson, null, 2).concat("\n");
+  await fs.writeFile(pkgJsonPath, jsonString);
+}
+
+/** insert the exports field into the package.json */
+function insertExports(pkgJson: any, exports: Record<string, any>): any {
+  // insert the export entries into the existing package.json, after these fields (for a clean format)
+  const exportsAfter = [
+    "name",
+    "description",
+    "version",
+    "private",
+    "author",
+    "type",
+    "bin",
+    "files",
+    "repository",
+    "homepage",
+    "scripts",
+  ];
+  const entries = Object.entries(pkgJson);
+  const index = entries.findLastIndex(([key]) => exportsAfter.includes(key));
+
+  entries.splice(index + 1, 0, ["exports", exports]);
+  return Object.fromEntries(entries);
+}
+
+/** create one bundle per source module */
+async function writeMultiBundle(
+  modules: Record<string, string>,
+  name: string,
+  edition: string,
+  projectDir: string,
+  outDir: string,
+): Promise<void> {
+  const sanitized = sanitizePackageName(name);
+  for (const [moduleName, moduleSrc] of Object.entries(modules)) {
+    const oneModule = { [moduleName]: moduleSrc };
+    const moduleBundle: WeslBundle = {
+      name: sanitized,
+      edition,
+      modules: oneModule,
+    };
+    const dependencies = parseDependencies(oneModule, projectDir);
+    const bundleDirRelative = noSuffix(moduleName);
+    const bundleDir = path.join(outDir, bundleDirRelative);
+    await writeJsBundle(moduleBundle, dependencies, bundleDir);
+  }
+}
+
+/** Write a weslBundle.js containing the bundled shader sources */
+async function writeJsBundle(
+  weslBundle: WeslBundle,
+  dependencies: string[],
+  outDir: string,
+): Promise<void> {
+  await mkdir(outDir, { recursive: true });
+
+  const depNames = dependencies.map(dep => dep.replaceAll("/", "_"));
+  const depsWithNames = zip(dependencies, depNames);
+
+  const imports = depsWithNames
+    .map(([dep, depName]) => {
+      return `import ${depName} from "${dep}";`;
+    })
+    .join("\n");
+  const importsStr = imports ? `${imports}\n` : "";
+
+  const bundleString = bundleToJsString(weslBundle, depNames);
+
+  const outString = `
+    ${importsStr}
+    export const weslBundle = ${bundleString}
+
+    export default weslBundle;
+  `;
+
+  const outPath = path.join(outDir, "weslBundle.js");
+  const formatted = biome.formatContent(biomeKey, outString, {
+    filePath: "b.js",
+  });
+  await fs.writeFile(outPath, formatted.content);
+}
+
+/** Write weslBundle.d.ts containing the type definitions for a WeslBundle */
+async function writeTypeScriptDts(outDir: string): Promise<void> {
+  // TODO could we use /// <reference types="wesl"> to get the type of WeslBundle?
+  const constDecl = `
+    export declare const weslBundle: WeslBundle;
+    export default weslBundle;
+  `;
+  const declText = weslBundleDecl + constDecl;
+  const formatted = biome.formatContent(biomeKey, declText, {
+    filePath: "t.d.ts",
+  });
+
+  const outPath = path.join(outDir, "weslBundle.d.ts");
+  await fs.writeFile(outPath, formatted.content);
+}
+
+/** @return the bundle plus dependencies as a JavaScript string */
+function bundleToJsString(bundle: WeslBundle, dependencies: string[]): string {
+  const { name, edition, modules } = bundle;
+  const depsObj = dependencies.length ? { dependencies: 99 } : {};
+  const obj = { name, edition, modules, ...depsObj };
+  const jsonString = JSON.stringify(obj, null, 2);
+  if (dependencies.length) {
+    const dependenciesStr = `: [${dependencies.join(", ")}]`;
+    const result = jsonString.replace(": 99", dependenciesStr);
+    return result;
+  } else {
+    return jsonString;
+  }
+}
+
+/** Filter out self-dependencies from the dependency list */
+function filterSelfDeps(deps: string[], packageName: string): string[] {
+  return deps.filter(
+    dep => dep !== packageName && !dep.startsWith(`${packageName}/`),
+  );
+}
+
+interface PkgFields {
+  name: string;
+}
+
+/** parse and extract fields from package.json that we care about
+ * (the name of the package) */
+async function loadPackageFields(pkgJsonPath: string): Promise<PkgFields> {
+  const pkgJsonString = await fs.readFile(pkgJsonPath, { encoding: "utf8" });
+  const pkgJson = JSON.parse(pkgJsonString);
+  const { name } = pkgJson;
+  verifyField("name", name);
+
+  function verifyField(field: string, value: any): void {
+    if (value === undefined) {
+      console.error(`no '${field}' field found in "${pkgJsonPath}"`);
+      throw new Error("package.json incomplete");
+    }
+  }
+  return { name };
+}
+
+/** setup biome to use as a formatter */
+async function setupBiome(): Promise<{ biome: Biome; projectKey: number }> {
+  const biome = await Biome.create({
+    distribution: Distribution.NODE,
+  });
+  const { projectKey } = biome.openProject();
+  biome.applyConfiguration(projectKey, {
+    formatter: {
+      enabled: true,
+      indentStyle: "space",
+      lineEnding: "lf",
+      expand: "auto",
+    },
+  });
+  return { biome, projectKey };
+}

package/src/PackagerCli.ts

@@ -0,0 +1,62 @@
+import { versionFromPackageJson } from "wesl-tooling";
+import yargs from "yargs";
+import { packageWgsl } from "./PackageWesl.ts";
+
+export type CliArgs = Awaited<ReturnType<typeof parseArgs>>;
+let cliArgs: CliArgs;
+
+export async function packagerCli(rawArgs: string[]): Promise<void> {
+  cliArgs = await parseArgs(rawArgs);
+  await packageWgsl(cliArgs);
+}
+
+async function parseArgs(args: string[]) {
+  const projectDir = new URL("..", import.meta.url).href;
+  const appVersion = await versionFromPackageJson(projectDir);
+  return yargs(args)
+    .command("$0", "create an npm package from WGSL/WESL files")
+    .version(appVersion)
+    .option("src", {
+      type: "string",
+      describe:
+        "WGSL/WESL files to bundle in the package (glob syntax, defaults to wesl.toml or shaders/**/*.w[eg]sl)",
+    })
+    .option("rootDir", {
+      deprecated: true,
+      type: "string",
+      describe: "use --baseDir instead",
+    })
+    .option("baseDir", {
+      deprecated: true,
+      type: "string",
+      describe: "root directory for shaders (defaults to wesl.toml or shaders)",
+    })
+    .option("projectDir", {
+      type: "string",
+      default: ".",
+      describe: "directory containing package.json and wesl.toml",
+    })
+    .option("multiBundle", {
+      type: "boolean",
+      default: false,
+      describe: "make a shader bundle for each source file",
+    })
+    .option("exportName", {
+      type: "string",
+      default: ".",
+      describe:
+        "package.json export name for consolidated bundle (ignored for multiBundle)",
+    })
+    .option("updatePackageJson", {
+      type: "boolean",
+      default: false,
+      describe: "add 'exports' entries into package.json",
+    })
+    .option("outDir", {
+      type: "string",
+      default: "dist",
+      describe: "where to put bundled output files (relative to projectDir)",
+    })
+    .help()
+    .parse();
+}

package/src/main.ts

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+import { exit } from "node:process";
+import { hideBin } from "yargs/helpers";
+import { packagerCli } from "./PackagerCli.ts";
+
+const rawArgs = hideBin(process.argv);
+
+const [major] = process.versions.node.split(".").map(Number);
+if (major < 22) {
+  console.error(
+    `Please upgrade node to version 22 or higher. (The current node version is ${process.version})`,
+  );
+  exit(1);
+}
+
+packagerCli(rawArgs).catch(e => {
+  console.error(e);
+  exit(1);
+});

package/src/test/ExpectDirMatch.ts

@@ -0,0 +1,119 @@
+import fs from "node:fs";
+import path from "node:path";
+import { diffChars, diffLines } from "diff";
+import {
+  compareSync,
+  type Difference,
+  type Options,
+  type Result,
+} from "dir-compare";
+import pico from "picocolors";
+import { expect } from "vitest";
+
+export const defaultOptions: Options = {
+  compareContent: true,
+  excludeFilter: "node_modules, .gitignore",
+  compareFileSync: normalizeCRLFCompare,
+};
+
+/**
+ * Compare two directories recursively and asserts that their contents match.
+ * Logs any differences found (extra, missing, or changed files).
+ * Fails the test if any differences are detected.
+ *
+ * @param resultDir - Path to the directory containing actual results.
+ * @param expectDir - Path to the directory containing expected results.
+ * @param options - Optional dir-compare options to customize comparison.
+ */
+export function expectDirMatch(
+  expectDir: string,
+  resultDir: string,
+  options?: Options,
+): void {
+  const compareOpts = { ...defaultOptions, ...options };
+  const compareResult: Result = compareSync(resultDir, expectDir, compareOpts);
+  const diffs = compareResult.diffSet!.filter(r => r.state !== "equal");
+  diffs.forEach(logDiff);
+  if (diffs.length > 0) {
+    expect.fail(`${resultDir} and ${expectDir} do not match`);
+  }
+}
+
+/** print a difference between two files or directories to the error log */
+function logDiff(diff: Difference): void {
+  const { name1, name2, path1, path2, state, relativePath } = diff;
+  const relative = relativePath.endsWith("/")
+    ? relativePath
+    : `${relativePath}/`;
+  if (state === "left") {
+    console.error(`Extra in result: ${relative}${name1}`);
+  } else if (state === "right") {
+    console.error(`Missing in result: ${relative}${name2}`);
+  } else {
+    const resultPath = path.join(path1!, name1!);
+    const expectPath = path.join(path2!, name2!);
+    console.error(`\n--- File ${relative}${name1} is different: ---`);
+    logFileDiff(resultPath, expectPath);
+  }
+}
+
+/** error log a difference between two files */
+function logFileDiff(resultPath: string, expectPath: string): void {
+  const resultStr = fs.readFileSync(resultPath, "utf8");
+  const expectStr = fs.readFileSync(expectPath, "utf8");
+  const diffResult = diffChars(resultStr, expectStr);
+  if (diffResult) {
+    if (process.env.FORCE_COLOR) {
+      const messages: string[] = [];
+      diffResult.forEach(part => {
+        const { value, added, removed } = part;
+        let spaced = value;
+        if (value.trim() === "") {
+          spaced = value
+            .replaceAll(" ", "█")
+            .replaceAll("\n", "\\n")
+            .replaceAll("\t", "→");
+        }
+
+        let colored = spaced;
+        if (added) {
+          colored = pico.bold(pico.green(spaced));
+        } else if (removed) {
+          colored = pico.bold(pico.red(spaced));
+        }
+        messages.push(colored);
+      });
+      console.error(messages.join(""));
+    } else {
+      const diffResult = diffLines(resultStr, expectStr);
+      diffResult.forEach(part => {
+        const { value, added, removed } = part;
+        let prefix = "";
+        if (added) {
+          prefix = "+ ";
+        } else if (removed) {
+          prefix = "- ";
+        }
+        console.error(prefix + value);
+      });
+    }
+  } else {
+    console.log(`'${resultPath}' and '${expectPath}' match`);
+  }
+}
+
+/** compare but convert \r\n to \n on both path1 and path2 */
+function normalizeCRLFCompare(
+  path1: string,
+  stat1: fs.Stats,
+  path2: string,
+  stat2: fs.Stats,
+  _options: Options,
+): boolean {
+  if (stat1.isFile() && stat2.isFile()) {
+    const content1 = fs.readFileSync(path1, "utf8").replace(/\r\n/g, "\n");
+    const content2 = fs.readFileSync(path2, "utf8").replace(/\r\n/g, "\n");
+    return content1 === content2;
+  }
+  return false;
+}

package/src/test/PackageWesl.test.ts

@@ -0,0 +1,103 @@
+import fs, { mkdir, readFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path, { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { rimraf } from "rimraf";
+import { expect, test } from "vitest";
+import { packagerCli } from "../PackagerCli.ts";
+import { expectDirMatch } from "./ExpectDirMatch.ts";
+
+const testDir = dirname(fileURLToPath(import.meta.url));
+
+test("package two wgsl files into one bundle", async () => {
+  const projectDir = path.join(testDir, "wesl-package");
+  const distDir = path.join(projectDir, "dist");
+  const srcDir = path.join(projectDir, "src");
+  await rimraf(distDir);
+  await mkdir(distDir);
+  await packageCli(
+    `--projectDir ${projectDir} 
+     --baseDir ${srcDir} 
+     --updatePackageJson false
+     --src ${srcDir}/*.w[eg]sl
+     --outDir ${distDir}`,
+  );
+  const contents = await readFile(path.join(distDir, "weslBundle.js"), "utf8");
+  const normalized = contents.replace(/\r\n/g, "\n"); // normalize line endings to LF
+  expect(normalized).toMatchInlineSnapshot(`
+    "export const weslBundle = {
+      name: "test_wesl_package",
+      edition: "unstable_2025_1",
+      modules: {
+        "util.wgsl": "fn foo() {}",
+        "lib.wesl": "import package::util;\\n",
+      },
+    };
+
+    export default weslBundle;
+    "
+  `);
+});
+
+/** set to true for manual review of test results */
+const saveTestDir = false;
+
+test("package multi ", async () => {
+  // create a copy of multi_package in a temporary directory so we can mutate it
+  let workDir: string;
+  if (saveTestDir) {
+    workDir = "/tmp/testing_multi";
+    await rimraf(workDir);
+    await mkdir(workDir);
+  } else {
+    workDir = await fs.mkdtemp(
+      path.join(tmpdir(), "wesl-packager-test-multi-"),
+    );
+  }
+
+  try {
+    const multiDir = path.join(testDir, "multi_package");
+    await fs.cp(multiDir, workDir, {
+      recursive: true,
+      preserveTimestamps: true,
+    });
+    replaceInFile(path.join(workDir, "package.json"), {
+      multi_pkg_src: "multi_pkg",
+    });
+
+    const distDir = path.join(workDir, "dist");
+    // run wesl-packager in temporary directory
+    await packageCli(
+      `--projectDir ${workDir}
+      --src ${workDir}/shaders/**/*.wesl
+      --updatePackageJson
+      --multiBundle
+      --baseDir ${workDir}/shaders
+      --outDir ${distDir}`,
+    );
+
+    const expectDir = path.join(testDir, "../../../test_pkg/multi_pkg");
+    expectDirMatch(workDir, expectDir);
+  } finally {
+    if (!saveTestDir) {
+      await rimraf(workDir);
+    }
+  }
+});
+
+function packageCli(argsLine: string): Promise<void> {
+  return packagerCli(argsLine.split(/\s+/));
+}
+
+/** Rewrite a file, replacing specified strings.  */
+async function replaceInFile(
+  filePath: string,
+  replacements: Record<string, string>,
+): Promise<void> {
+  let content = await readFile(filePath, "utf8");
+  // content = content.replace(/\r\n/g, "\n"); // normalize line endings to LF
+  for (const [search, replace] of Object.entries(replacements)) {
+    content = content.replaceAll(search, replace);
+  }
+  await fs.writeFile(filePath, content, "utf8");
+}

package/src/test/multi_package/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "multi_pkg_src",
+  "version": "0.0.1",
+  "private": true,
+  "dependencies": {
+    "dependent_package": "workspace:*"
+  }
+}

package/src/test/multi_package/shaders/dir/nested.wesl

@@ -0,0 +1 @@
+fn nest() {} 

package/src/test/multi_package/shaders/second.wesl

@@ -0,0 +1 @@
+fn two() { }

package/src/test/multi_package/shaders/transitive.wesl

@@ -0,0 +1,3 @@
+import dependent_package::dep;
+
+fn toDep() { dep(); }

package/src/test/wesl-package/dist/weslBundle.d.ts

@@ -0,0 +1,20 @@
+export interface WeslBundle {
+  /** npm package name sanitized to be a valid WESL identifier
+   * (@ removed, / ==> __, - ==> _) */
+  name: string;
+
+  /** WESL edition of the code e.g. unstable_2025_1 */
+  edition: string;
+
+  /** map of WESL/WGSL modules:
+   *    keys are file paths, relative to package root (e.g. "./lib.wgsl")
+   *    values are WESL/WGSL code strings
+   */
+  modules: Record<string, string>;
+
+  /** packages referenced by this package */
+  dependencies?: WeslBundle[];
+}
+
+export declare const weslBundle: WeslBundle;
+export default weslBundle;

package/src/test/wesl-package/dist/weslBundle.js

@@ -0,0 +1,10 @@
+export const weslBundle = {
+  name: "test_wesl_package",
+  edition: "unstable_2025_1",
+  modules: {
+    "util.wgsl": "fn foo() {}",
+    "lib.wesl": "import package::util;\n",
+  },
+};
+
+export default weslBundle;

package/src/test/wesl-package/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "test-wesl-package",
+  "private": true,
+  "exports": {
+    ".": {
+      "import": "./dist/weslBundle.js",
+      "types": "./dist/weslBundle.d.ts"
+    }
+  }
+}

package/src/test/wesl-package/src/lib.wesl

@@ -0,0 +1 @@
+import package::util;

package/src/test/wesl-package/src/util.wgsl

@@ -0,0 +1 @@
+fn foo() {}