wrangler 0.0.0-e22f9d7 → 0.0.0-e2a26a0

package/src/publish.ts

@@ -1,33 +1,36 @@
 import assert from "node:assert";
-import path from "node:path";
 import { readFile } from "node:fs/promises";
-import esbuild from "esbuild";
-import { execa } from "execa";
+import path from "node:path";
+import { URLSearchParams } from "node:url";
+import * as esbuild from "esbuild";
+import { execaCommand } from "execa";
 import tmp from "tmp-promise";
-import type { CfWorkerInit } from "./api/worker";
 import { toFormData } from "./api/form_data";
 import { fetchResult } from "./cfetch";
-import type { Config } from "./config";
 import makeModuleCollector from "./module-collection";
 import { syncAssets } from "./sites";
+import type { CfWorkerInit } from "./api/worker";
+import type { Config } from "./config";
+import type { AssetPaths } from "./sites";
+import type { Metafile } from "esbuild";
 
 type CfScriptFormat = undefined | "modules" | "service-worker";
 
 type Props = {
   config: Config;
-  format?: CfScriptFormat;
-  script?: string;
-  name?: string;
-  env?: string;
-  compatibilityDate?: string;
-  compatibilityFlags?: string[];
-  public?: string;
-  site?: string;
-  triggers?: (string | number)[];
-  routes?: (string | number)[];
-  legacyEnv?: boolean;
+  format: CfScriptFormat | undefined;
+  script: string | undefined;
+  name: string | undefined;
+  env: string | undefined;
+  compatibilityDate: string | undefined;
+  compatibilityFlags: string[] | undefined;
+  assetPaths: AssetPaths | undefined;
+  triggers: (string | number)[] | undefined;
+  routes: (string | number)[] | undefined;
+  legacyEnv: boolean | undefined;
   jsxFactory: undefined | string;
   jsxFragment: undefined | string;
+  experimentalPublic: boolean;
 };
 
 function sleep(ms: number) {
@@ -35,10 +38,10 @@ function sleep(ms: number) {
 }
 
 export default async function publish(props: Props): Promise<void> {
-  if (props.public && props.format === "service-worker") {
+  if (props.experimentalPublic && props.format === "service-worker") {
     // TODO: check config too
     throw new Error(
-      "You cannot use the service worker format with a public directory."
+      "You cannot publish in the service worker format with a public directory."
     );
   }
   // TODO: warn if git/hg has uncommitted changes
@@ -76,12 +79,31 @@ export default async function publish(props: Props): Promise<void> {
     'You need to provide a name when publishing a worker. Either pass it as a cli arg with `--name <name>` or in your config file as `name = "<name>"`'
   );
 
+  if (config.site?.["entry-point"]) {
+    console.warn(
+      "Deprecation notice: The `site.entry-point` config field is no longer used.\n" +
+        "The entry-point should be specified via the command line (e.g. `wrangler publish path/to/script`) or the `build.upload.main` config field.\n" +
+        "Please remove the `site.entry-point` field from the `wrangler.toml` file."
+    );
+  }
+
+  assert(
+    !config.site || config.site.bucket,
+    "A [site] definition requires a `bucket` field with a path to the site's public directory."
+  );
+
   let file: string;
   if (props.script) {
-    file = props.script;
+    // If the script name comes from the command line it is relative to the current working directory.
+    file = path.resolve(props.script);
   } else {
-    assert(build?.upload?.main, "missing main file");
-    file = path.join(path.dirname(__path__), build.upload.main);
+    // If the script name comes from the config, then it is relative to the wrangler.toml file.
+    if (build?.upload?.main === undefined) {
+      throw new Error(
+        "Missing entry-point: The entry-point should be specified via the command line (e.g. `wrangler publish path/to/script`) or the `build.upload.main` config field."
+      );
+    }
+    file = path.resolve(path.dirname(__path__), build.upload.main);
   }
 
   if (props.legacyEnv) {
@@ -90,12 +112,11 @@ export default async function publish(props: Props): Promise<void> {
   const envName = props.env ?? "production";
 
   const destination = await tmp.dir({ unsafeCleanup: true });
-
   if (props.config.build?.command) {
     // TODO: add a deprecation message here?
     console.log("running:", props.config.build.command);
-    const buildCommandPieces = props.config.build.command.split(" ");
-    await execa(buildCommandPieces[0], buildCommandPieces.slice(1), {
+    await execaCommand(props.config.build.command, {
+      shell: true,
       stdout: "inherit",
       stderr: "inherit",
       ...(props.config.build?.cwd && { cwd: props.config.build.cwd }),
@@ -104,22 +125,22 @@ export default async function publish(props: Props): Promise<void> {
 
   const moduleCollector = makeModuleCollector();
   const result = await esbuild.build({
-    ...(props.public
+    ...(props.experimentalPublic
       ? {
           stdin: {
             contents: (
               await readFile(
-                path.join(__dirname, "../static-asset-facade.js"),
+                path.join(__dirname, "../templates/static-asset-facade.js"),
                 "utf8"
               )
-            ).replace("__ENTRY_POINT__", path.join(process.cwd(), file)),
+            ).replace("__ENTRY_POINT__", file),
             sourcefile: "static-asset-facade.js",
             resolveDir: path.dirname(file),
           },
+          nodePaths: [path.join(__dirname, "../vendor")],
         }
       : { entryPoints: [file] }),
     bundle: true,
-    nodePaths: props.public ? [path.join(__dirname, "../vendor")] : undefined,
     outdir: destination.path,
     external: ["__STATIC_CONTENT_MANIFEST"],
     format: "esm",
@@ -128,6 +149,9 @@ export default async function publish(props: Props): Promise<void> {
     conditions: ["worker", "browser"],
     loader: {
       ".js": "jsx",
+      ".html": "text",
+      ".pem": "text",
+      ".txt": "text",
     },
     plugins: [moduleCollector.plugin],
     ...(jsxFactory && { jsxFactory }),
@@ -137,21 +161,25 @@ export default async function publish(props: Props): Promise<void> {
   // result.metafile is defined because of the `metafile: true` option above.
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   const metafile = result.metafile!;
-  const expectedEntryPoint = props.public
-    ? path.join(path.dirname(file), "static-asset-facade.js")
-    : file;
-  const outputEntry = Object.entries(metafile.outputs).find(
-    ([, { entryPoint }]) => entryPoint === expectedEntryPoint
+  const entryPoints = Object.entries(metafile.outputs).filter(
+    ([_path, output]) => output.entryPoint !== undefined
   );
-  if (outputEntry === undefined) {
-    throw new Error(
-      `Cannot find entry-point "${expectedEntryPoint}" in generated bundle.`
-    );
-  }
+  assert(
+    entryPoints.length > 0,
+    `Cannot find entry-point "${file}" in generated bundle.` +
+      listEntryPoints(entryPoints)
+  );
+  assert(
+    entryPoints.length < 2,
+    "More than one entry-point found for generated bundle." +
+      listEntryPoints(entryPoints)
+  );
+  const entryPointExports = entryPoints[0][1].exports;
+  const resolvedEntryPointPath = entryPoints[0][0];
   const { format } = props;
   const bundle = {
-    type: outputEntry[1].exports.length > 0 ? "esm" : "commonjs",
-    exports: outputEntry[1].exports,
+    type: entryPointExports.length > 0 ? "esm" : "commonjs",
+    exports: entryPointExports,
   };
 
   // TODO: instead of bundling the facade with the worker, we should just bundle the worker and expose it as a module.
@@ -168,7 +196,7 @@ export default async function publish(props: Props): Promise<void> {
     return;
   }
 
-  const content = await readFile(outputEntry[0], { encoding: "utf-8" });
+  let content = await readFile(resolvedEntryPointPath, { encoding: "utf-8" });
   await destination.cleanup();
 
   // if config.migrations
@@ -210,13 +238,16 @@ export default async function publish(props: Props): Promise<void> {
     }
   }
 
-  const assetPath = props.public || props.site || props.config.site?.bucket; // TODO: allow both
-  const assets = assetPath
-    ? await syncAssets(accountId, scriptName, assetPath, false)
-    : { manifest: undefined, namespace: undefined };
+  const assets = await syncAssets(
+    accountId,
+    scriptName,
+    props.assetPaths,
+    false,
+    props.env
+  );
 
   const bindings: CfWorkerInit["bindings"] = {
-    kv_namespaces: envRootObj.kv_namespaces?.concat(
+    kv_namespaces: (envRootObj.kv_namespaces || []).concat(
       assets.namespace
         ? { binding: "__STATIC_CONTENT", id: assets.namespace }
         : []
@@ -226,17 +257,24 @@ export default async function publish(props: Props): Promise<void> {
     services: envRootObj.experimental_services,
   };
 
+  const workerType = bundle.type === "esm" ? "esm" : "commonjs";
+  if (workerType !== "esm" && assets.manifest) {
+    content = `const __STATIC_CONTENT_MANIFEST = ${JSON.stringify(
+      assets.manifest
+    )};\n${content}`;
+  }
+
   const worker: CfWorkerInit = {
     name: scriptName,
     main: {
-      name: path.basename(outputEntry[0]),
+      name: path.basename(resolvedEntryPointPath),
       content: content,
-      type: bundle.type === "esm" ? "esm" : "commonjs",
+      type: workerType,
     },
     bindings,
     ...(migrations && { migrations }),
     modules: moduleCollector.modules.concat(
-      assets.manifest
+      assets.manifest && workerType === "esm"
         ? {
             name: "__STATIC_CONTENT_MANIFEST",
             content: JSON.stringify(assets.manifest),
@@ -262,12 +300,12 @@ export default async function publish(props: Props): Promise<void> {
 
   // Upload the script so it has time to propagate.
   const { available_on_subdomain } = await fetchResult(
-    `${workerUrl}?available_on_subdomain=true`,
+    workerUrl,
     {
       method: "PUT",
-      // @ts-expect-error: TODO: fix this type error!
       body: toFormData(worker),
-    }
+    },
+    new URLSearchParams({ available_on_subdomains: "true" })
   );
 
   const uploadMs = Date.now() - start;
@@ -362,3 +400,11 @@ export default async function publish(props: Props): Promise<void> {
     console.log(" ", target);
   }
 }
+
+function listEntryPoints(
+  outputs: [string, ValueOf<Metafile["outputs"]>][]
+): string {
+  return outputs.map(([_input, output]) => output.entryPoint).join("\n");
+}
+
+type ValueOf<T> = T[keyof T];

package/src/sites.tsx

@@ -1,13 +1,36 @@
-import crypto from "node:crypto";
-import { createReadStream } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
-import { readdir, readFile } from "node:fs/promises";
-import { fetchResult } from "./cfetch";
-import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
+import ignore from "ignore";
+import { XXHash64 } from "xxhash-addon";
+import {
+  createNamespace,
+  listNamespaceKeys,
+  listNamespaces,
+  putBulkKeyValue,
+} from "./kv";
+import type { Config } from "./config";
+
+/** Paths to always ignore. */
+const ALWAYS_IGNORE = ["node_modules"];
+const HIDDEN_FILES_TO_INCLUDE = [
+  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
+];
 
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
+    // Skip files that we never want to process.
+    if (ALWAYS_IGNORE.some((p) => file.name === p)) {
+      continue;
+    }
+    // Skip hidden files (starting with .) except for some special ones
+    if (
+      file.name.startsWith(".") &&
+      !HIDDEN_FILES_TO_INCLUDE.some((p) => file.name === p)
+    ) {
+      continue;
+    }
+    // TODO: follow symlinks??
     if (file.isDirectory()) {
       yield* await getFilesInFolder(path.join(dirPath, file.name));
     } else {
@@ -16,27 +39,32 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   }
 }
 
-async function hashFileContent(filePath: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const hash = crypto.createHash("sha1");
-    const rs = createReadStream(filePath);
-    rs.on("error", reject);
-    rs.on("data", (chunk) => hash.update(chunk));
-    rs.on("end", () => resolve(hash.digest("hex")));
-  });
+/**
+ * Create a hash key for the given content using the xxhash algorithm.
+ *
+ * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key
+ * the most important thing here is to detect changes of a single file to invalidate the cache and
+ * it's impossible to serve two different files with the same name
+ */
+function hashFileContent(content: string): string {
+  const hasher = new XXHash64();
+  hasher.update(Buffer.from(content));
+  const hash = hasher.digest();
+  return hash.toString("hex").substring(0, 10);
 }
 
-async function hashFile(filePath: string): Promise<{
-  filePath: string;
-  hash: string;
-}> {
-  const extName = path.extname(filePath);
+/**
+ * Create a hashed asset key for the given asset.
+ *
+ * The key will change if the file path or content of the asset changes.
+ * The algorithm used here matches that of Wrangler 1.
+ */
+function hashAsset(filePath: string, content: string): string {
+  const extName = path.extname(filePath) || "";
   const baseName = path.basename(filePath, extName);
-  const hash = await hashFileContent(filePath);
-  return {
-    filePath: `${baseName}.${hash}${extName || ""}`,
-    hash,
-  };
+  const directory = path.dirname(filePath);
+  const hash = hashFileContent(content);
+  return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
 }
 
 async function createKVNamespaceIfNotAlreadyExisting(
@@ -52,41 +80,52 @@ async function createKVNamespaceIfNotAlreadyExisting(
   }
 
   // else we make the namespace
-  // TODO: use an export from ./kv
-  const json = await fetchResult<{ id: string }>(
-    `/accounts/${accountId}/storage/kv/namespaces`,
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ title }),
-    }
-  );
+  const id = await createNamespace(accountId, title);
+  console.log(`🌀 Created namespace for Workers Site "${title}"`);
 
   return {
     created: true,
-    id: json.id,
+    id,
   };
 }
 
+/**
+ * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
+ * the worker given by `scriptName`.
+ *
+ * @param accountId the account to upload to.
+ * @param scriptName the name of the worker whose assets we are uploading.
+ * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
+ * @param preview if true then upload to a "preview" KV namespace.
+ * @param _env (not implemented).
+ * @returns a promise for an object mapping the relative paths of the assets to the key of that
+ * asset in the KV namespace.
+ */
 export async function syncAssets(
   accountId: string,
   scriptName: string,
-  dirPath: string,
+  siteAssets: AssetPaths | undefined,
   preview: boolean,
-  _env?: string
-) {
-  const title = `__${scriptName}_sites_assets${preview ? "_preview" : ""}`;
+  env: string | undefined
+): Promise<{
+  manifest: { [filePath: string]: string } | undefined;
+  namespace: string | undefined;
+}> {
+  if (siteAssets === undefined) {
+    return { manifest: undefined, namespace: undefined };
+  }
+
+  const title = `__${scriptName}${env ? `-${env}` : ""}-workers_sites_assets${
+    preview ? "_preview" : ""
+  }`;
   const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
     title,
     accountId
   );
 
   // let's get all the keys in this namespace
-  const keys = new Set(
-    (await listNamespaceKeys(accountId, namespace)).map((x) => x.name)
-  );
+  const result = await listNamespaceKeys(accountId, namespace);
+  const keys = new Set(result.map((x) => x.name));
 
   const manifest = {};
   const upload: {
@@ -94,25 +133,108 @@ export async function syncAssets(
     value: string;
     base64: boolean;
   }[] = [];
+
+  const include = createPatternMatcher(siteAssets.includePatterns, false);
+  const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
   // TODO: this can be more efficient by parallelising
-  for await (const file of getFilesInFolder(dirPath)) {
-    // TODO: "exclude:" config
-    const { filePath } = await hashFile(file);
+  for await (const file of getFilesInFolder(siteAssets.baseDirectory)) {
+    if (!include(file)) {
+      continue;
+    }
+    if (exclude(file)) {
+      continue;
+    }
+
+    await validateAssetSize(file);
+    console.log(`reading ${file}...`);
+    const content = await readFile(file, "base64");
+
+    const assetKey = hashAsset(file, content);
+    validateAssetKey(assetKey);
+
     // now put each of the files into kv
-    if (!keys.has(filePath)) {
-      console.log(`uploading ${file}...`);
-      const content = await readFile(file, "base64");
-      if (content.length > 25 * 1024 * 1024) {
-        throw new Error(`File ${file} is too big, it should be under 25 mb.`);
-      }
+    if (!keys.has(assetKey)) {
+      console.log(`uploading as ${assetKey}...`);
       upload.push({
-        key: filePath,
+        key: assetKey,
         value: content,
         base64: true,
       });
+    } else {
+      console.log(`skipping - already uploaded`);
     }
-    manifest[path.relative(dirPath, file)] = filePath;
+    manifest[path.relative(siteAssets.baseDirectory, file)] = assetKey;
   }
   await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
   return { manifest, namespace };
 }
+
+function createPatternMatcher(
+  patterns: string[],
+  exclude: boolean
+): (filePath: string) => boolean {
+  if (patterns.length === 0) {
+    return (_filePath) => !exclude;
+  } else {
+    const ignorer = ignore().add(patterns);
+    return (filePath) => ignorer.test(filePath).ignored;
+  }
+}
+
+async function validateAssetSize(filePath: string) {
+  const { size } = await stat(filePath);
+  if (size > 25 * 1024 * 1024) {
+    throw new Error(
+      `File ${filePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
+    );
+  }
+}
+
+function validateAssetKey(assetKey: string) {
+  if (assetKey.length > 512) {
+    throw new Error(
+      `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",`
+    );
+  }
+}
+
+/**
+ * Convert a filePath to be safe to use as a relative URL.
+ *
+ * Primarily this involves converting Windows backslashes to forward slashes.
+ */
+function urlSafe(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+/**
+ * Information about the assets that should be uploaded
+ */
+export interface AssetPaths {
+  baseDirectory: string;
+  includePatterns: string[];
+  excludePatterns: string[];
+}
+
+/**
+ * Get an object that describes what assets to upload, if any.
+ *
+ * Uses the args (passed from the command line) if available,
+ * falling back to those defined in the config.
+ *
+ * // TODO: Support for environments
+ */
+export function getAssetPaths(
+  config: Config,
+  baseDirectory = config.site?.bucket,
+  includePatterns = config.site?.include ?? [],
+  excludePatterns = config.site?.exclude ?? []
+): undefined | AssetPaths {
+  return baseDirectory
+    ? {
+        baseDirectory,
+        includePatterns,
+        excludePatterns,
+      }
+    : undefined;
+}