wrangler 0.0.0-e1c615f → 0.0.0-e5d3690

package/src/publish.ts

@@ -1,7 +1,8 @@
 import assert from "node:assert";
 import path from "node:path";
 import { readFile } from "node:fs/promises";
-import esbuild from "esbuild";
+import * as esbuild from "esbuild";
+import type { Metafile } from "esbuild";
 import { execa } from "execa";
 import tmp from "tmp-promise";
 import type { CfWorkerInit } from "./api/worker";
@@ -9,23 +10,26 @@ import { toFormData } from "./api/form_data";
 import { fetchResult } from "./cfetch";
 import type { Config } from "./config";
 import makeModuleCollector from "./module-collection";
+import type { AssetPaths } from "./sites";
 import { syncAssets } from "./sites";
 
-type CfScriptFormat = void | "modules" | "service-worker";
+type CfScriptFormat = undefined | "modules" | "service-worker";
 
 type Props = {
   config: Config;
-  format?: CfScriptFormat;
-  script?: string;
-  name?: string;
-  env?: string;
-  public?: string;
-  site?: string;
-  triggers?: (string | number)[];
-  routes?: (string | number)[];
-  legacyEnv?: boolean;
-  jsxFactory: void | string;
-  jsxFragment: void | string;
+  format: CfScriptFormat | undefined;
+  script: string | undefined;
+  name: string | undefined;
+  env: string | undefined;
+  compatibilityDate: string | undefined;
+  compatibilityFlags: string[] | undefined;
+  assetPaths: AssetPaths | undefined;
+  triggers: (string | number)[] | undefined;
+  routes: (string | number)[] | undefined;
+  legacyEnv: boolean | undefined;
+  jsxFactory: undefined | string;
+  jsxFragment: undefined | string;
+  experimentalPublic: boolean;
 };
 
 function sleep(ms: number) {
@@ -33,10 +37,10 @@ function sleep(ms: number) {
 }
 
 export default async function publish(props: Props): Promise<void> {
-  if (props.public && props.format === "service-worker") {
+  if (props.experimentalPublic && props.format === "service-worker") {
     // TODO: check config too
     throw new Error(
-      "You cannot use the service worker format with a public directory."
+      "You cannot publish in the service worker format with a public directory."
     );
   }
   // TODO: warn if git/hg has uncommitted changes
@@ -48,6 +52,18 @@ export default async function publish(props: Props): Promise<void> {
     __path__,
   } = config;
 
+  const envRootObj =
+    props.env && config.env ? config.env[props.env] || {} : config;
+
+  assert(
+    envRootObj.compatibility_date || props.compatibilityDate,
+    "A compatibility_date is required when publishing. Add one to your wrangler.toml file, or pass it in your terminal as --compatibility_date. See https://developers.cloudflare.com/workers/platform/compatibility-dates for more information."
+  );
+
+  if (accountId === undefined) {
+    throw new Error("No account_id provided.");
+  }
+
   const triggers = props.triggers || config.triggers?.crons;
   const routes = props.routes || config.routes;
 
@@ -64,10 +80,12 @@ export default async function publish(props: Props): Promise<void> {
 
   let file: string;
   if (props.script) {
-    file = props.script;
+    // If the script name comes from the command line it is relative to the current working directory.
+    file = path.resolve(props.script);
   } else {
+    // If the script name comes from the config, then it is relative to the wrangler.toml file.
     assert(build?.upload?.main, "missing main file");
-    file = path.join(path.dirname(__path__), build.upload.main);
+    file = path.resolve(path.dirname(__path__), build.upload.main);
   }
 
   if (props.legacyEnv) {
@@ -76,7 +94,6 @@ export default async function publish(props: Props): Promise<void> {
   const envName = props.env ?? "production";
 
   const destination = await tmp.dir({ unsafeCleanup: true });
-
   if (props.config.build?.command) {
     // TODO: add a deprecation message here?
     console.log("running:", props.config.build.command);
@@ -90,7 +107,7 @@ export default async function publish(props: Props): Promise<void> {
 
   const moduleCollector = makeModuleCollector();
   const result = await esbuild.build({
-    ...(props.public
+    ...(props.experimentalPublic
       ? {
           stdin: {
             contents: (
@@ -98,14 +115,14 @@ export default async function publish(props: Props): Promise<void> {
                 path.join(__dirname, "../static-asset-facade.js"),
                 "utf8"
               )
-            ).replace("__ENTRY_POINT__", path.join(process.cwd(), file)),
+            ).replace("__ENTRY_POINT__", file),
             sourcefile: "static-asset-facade.js",
             resolveDir: path.dirname(file),
           },
+          nodePaths: [path.join(__dirname, "../vendor")],
         }
       : { entryPoints: [file] }),
     bundle: true,
-    nodePaths: props.public ? [path.join(__dirname, "../vendor")] : undefined,
     outdir: destination.path,
     external: ["__STATIC_CONTENT_MANIFEST"],
     format: "esm",
@@ -120,18 +137,28 @@ export default async function publish(props: Props): Promise<void> {
     ...(jsxFragment && { jsxFragment }),
   });
 
-  const chunks = Object.entries(result.metafile.outputs).find(
-    ([_path, { entryPoint }]) =>
-      entryPoint ===
-      (props.public
-        ? path.join(path.dirname(file), "static-asset-facade.js")
-        : file)
+  // result.metafile is defined because of the `metafile: true` option above.
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const metafile = result.metafile!;
+  const entryPoints = Object.entries(metafile.outputs).filter(
+    ([_path, output]) => output.entryPoint !== undefined
   );
-
+  assert(
+    entryPoints.length > 0,
+    `Cannot find entry-point "${file}" in generated bundle.` +
+      listEntryPoints(entryPoints)
+  );
+  assert(
+    entryPoints.length < 2,
+    "More than one entry-point found for generated bundle." +
+      listEntryPoints(entryPoints)
+  );
+  const entryPointExports = entryPoints[0][1].exports;
+  const resolvedEntryPointPath = entryPoints[0][0];
   const { format } = props;
   const bundle = {
-    type: chunks[1].exports.length > 0 ? "esm" : "commonjs",
-    exports: chunks[1].exports,
+    type: entryPointExports.length > 0 ? "esm" : "commonjs",
+    exports: entryPointExports,
   };
 
   // TODO: instead of bundling the facade with the worker, we should just bundle the worker and expose it as a module.
@@ -148,13 +175,13 @@ export default async function publish(props: Props): Promise<void> {
     return;
   }
 
-  const content = await readFile(chunks[0], { encoding: "utf-8" });
+  const content = await readFile(resolvedEntryPointPath, { encoding: "utf-8" });
   await destination.cleanup();
 
   // if config.migrations
   // get current migration tag
   let migrations;
-  if ("migrations" in config) {
+  if (config.migrations !== undefined) {
     const scripts = await fetchResult<{ id: string; migration_tag: string }[]>(
       `/accounts/${accountId}/workers/scripts`
     );
@@ -190,17 +217,13 @@ export default async function publish(props: Props): Promise<void> {
     }
   }
 
-  const assets =
-    props.public || props.site || props.config.site?.bucket // TODO: allow both
-      ? await syncAssets(
-          accountId,
-          scriptName,
-          props.public || props.site || props.config.site?.bucket,
-          false
-        )
-      : { manifest: undefined, namespace: undefined };
-
-  const envRootObj = props.env ? config.env[props.env] || {} : config;
+  const assets = await syncAssets(
+    accountId,
+    scriptName,
+    props.assetPaths,
+    false
+  );
+
   const bindings: CfWorkerInit["bindings"] = {
     kv_namespaces: envRootObj.kv_namespaces?.concat(
       assets.namespace
@@ -215,7 +238,7 @@ export default async function publish(props: Props): Promise<void> {
   const worker: CfWorkerInit = {
     name: scriptName,
     main: {
-      name: path.basename(chunks[0]),
+      name: path.basename(resolvedEntryPointPath),
       content: content,
       type: bundle.type === "esm" ? "esm" : "commonjs",
     },
@@ -248,12 +271,13 @@ export default async function publish(props: Props): Promise<void> {
 
   // Upload the script so it has time to propagate.
   const { available_on_subdomain } = await fetchResult(
-    `${workerUrl}?available_on_subdomain=true`,
+    workerUrl,
     {
       method: "PUT",
       // @ts-expect-error: TODO: fix this type error!
       body: toFormData(worker),
-    }
+    },
+    new URLSearchParams({ available_on_subdomains: "true" })
   );
 
   const uploadMs = Date.now() - start;
@@ -348,3 +372,11 @@ export default async function publish(props: Props): Promise<void> {
     console.log(" ", target);
   }
 }
+
+function listEntryPoints(
+  outputs: [string, ValueOf<Metafile["outputs"]>][]
+): string {
+  return outputs.map(([_input, output]) => output.entryPoint).join("\n");
+}
+
+type ValueOf<T> = T[keyof T];

package/src/sites.tsx

@@ -1,13 +1,32 @@
-import crypto from "node:crypto";
-import { createReadStream } from "node:fs";
 import * as path from "node:path";
-import { readdir, readFile } from "node:fs/promises";
+import { readdir, readFile, stat } from "node:fs/promises";
+import ignore from "ignore";
+import { XXHash64 } from "xxhash-addon";
 import { fetchResult } from "./cfetch";
+import type { Config } from "./config";
 import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
 
+/** Paths to always ignore. */
+const ALWAYS_IGNORE = ["node_modules"];
+const HIDDEN_FILES_TO_INCLUDE = [
+  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
+];
+
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
+    // Skip files that we never want to process.
+    if (ALWAYS_IGNORE.some((p) => file.name === p)) {
+      continue;
+    }
+    // Skip hidden files (starting with .) except for some special ones
+    if (
+      file.name.startsWith(".") &&
+      !HIDDEN_FILES_TO_INCLUDE.some((p) => file.name === p)
+    ) {
+      continue;
+    }
+    // TODO: follow symlinks??
     if (file.isDirectory()) {
       yield* await getFilesInFolder(path.join(dirPath, file.name));
     } else {
@@ -16,27 +35,32 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   }
 }
 
-async function hashFileContent(filePath: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const hash = crypto.createHash("sha1");
-    const rs = createReadStream(filePath);
-    rs.on("error", reject);
-    rs.on("data", (chunk) => hash.update(chunk));
-    rs.on("end", () => resolve(hash.digest("hex")));
-  });
+/**
+ * Create a hash key for the given content using the xxhash algorithm.
+ *
+ * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key
+ * the most important thing here is to detect changes of a single file to invalidate the cache and
+ * it's impossible to serve two different files with the same name
+ */
+function hashFileContent(content: string): string {
+  const hasher = new XXHash64();
+  hasher.update(Buffer.from(content));
+  const hash = hasher.digest();
+  return hash.toString("hex").substring(0, 10);
 }
 
-async function hashFile(filePath: string): Promise<{
-  filePath: string;
-  hash: string;
-}> {
-  const extName = path.extname(filePath);
+/**
+ * Create a hashed asset key for the given asset.
+ *
+ * The key will change if the file path or content of the asset changes.
+ * The algorithm used here matches that of Wrangler 1.
+ */
+function hashAsset(filePath: string, content: string): string {
+  const extName = path.extname(filePath) || "";
   const baseName = path.basename(filePath, extName);
-  const hash = await hashFileContent(filePath);
-  return {
-    filePath: `${baseName}.${hash}${extName || ""}`,
-    hash,
-  };
+  const directory = path.dirname(filePath);
+  const hash = hashFileContent(content);
+  return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
 }
 
 async function createKVNamespaceIfNotAlreadyExisting(
@@ -70,13 +94,32 @@ async function createKVNamespaceIfNotAlreadyExisting(
   };
 }
 
+/**
+ * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
+ * the worker given by `scriptName`.
+ *
+ * @param accountId the account to upload to.
+ * @param scriptName the name of the worker whose assets we are uploading.
+ * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
+ * @param preview if true then upload to a "preview" KV namespace.
+ * @param _env (not implemented).
+ * @returns a promise for an object mapping the relative paths of the assets to the key of that
+ * asset in the KV namespace.
+ */
 export async function syncAssets(
   accountId: string,
   scriptName: string,
-  dirPath: string,
+  siteAssets: AssetPaths | undefined,
   preview: boolean,
   _env?: string
-) {
+): Promise<{
+  manifest: { [filePath: string]: string } | undefined;
+  namespace: string | undefined;
+}> {
+  if (siteAssets === undefined) {
+    return { manifest: undefined, namespace: undefined };
+  }
+
   const title = `__${scriptName}_sites_assets${preview ? "_preview" : ""}`;
   const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
     title,
@@ -84,31 +127,117 @@ export async function syncAssets(
   );
 
   // let's get all the keys in this namespace
-  const keys = new Set(
-    (await listNamespaceKeys(accountId, namespace)).map((x) => x.name)
-  );
+  const result = await listNamespaceKeys(accountId, namespace);
+  const keys = new Set(result.map((x) => x.name));
 
   const manifest = {};
-  const upload = [];
+  const upload: {
+    key: string;
+    value: string;
+    base64: boolean;
+  }[] = [];
+
+  const include = createPatternMatcher(siteAssets.includePatterns, false);
+  const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
   // TODO: this can be more efficient by parallelising
-  for await (const file of getFilesInFolder(dirPath)) {
-    // TODO: "exclude:" config
-    const { filePath } = await hashFile(file);
+  for await (const file of getFilesInFolder(siteAssets.baseDirectory)) {
+    if (!include(file)) {
+      continue;
+    }
+    if (exclude(file)) {
+      continue;
+    }
+
+    await validateAssetSize(file);
+    console.log(`reading ${file}...`);
+    const content = await readFile(file, "base64");
+
+    const assetKey = hashAsset(file, content);
+    validateAssetKey(assetKey);
+
     // now put each of the files into kv
-    if (!keys.has(filePath)) {
-      console.log(`uploading ${file}...`);
-      const content = await readFile(file, "base64");
-      if (content.length > 25 * 1024 * 1024) {
-        throw new Error(`File ${file} is too big, it should be under 25 mb.`);
-      }
+    if (!keys.has(assetKey)) {
+      console.log(`uploading as ${assetKey}...`);
       upload.push({
-        key: filePath,
+        key: assetKey,
         value: content,
         base64: true,
       });
+    } else {
+      console.log(`skipping - already uploaded`);
     }
-    manifest[path.relative(dirPath, file)] = filePath;
+    manifest[path.relative(siteAssets.baseDirectory, file)] = assetKey;
   }
   await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
   return { manifest, namespace };
 }
+
+function createPatternMatcher(
+  patterns: string[],
+  exclude: boolean
+): (filePath: string) => boolean {
+  if (patterns.length === 0) {
+    return (_filePath) => !exclude;
+  } else {
+    const ignorer = ignore().add(patterns);
+    return (filePath) => ignorer.test(filePath).ignored;
+  }
+}
+
+async function validateAssetSize(filePath: string) {
+  const { size } = await stat(filePath);
+  if (size > 25 * 1024 * 1024) {
+    throw new Error(
+      `File ${filePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
+    );
+  }
+}
+
+function validateAssetKey(assetKey: string) {
+  if (assetKey.length > 512) {
+    throw new Error(
+      `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",`
+    );
+  }
+}
+
+/**
+ * Convert a filePath to be safe to use as a relative URL.
+ *
+ * Primarily this involves converting Windows backslashes to forward slashes.
+ */
+function urlSafe(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+/**
+ * Information about the assets that should be uploaded
+ */
+export interface AssetPaths {
+  baseDirectory: string;
+  includePatterns: string[];
+  excludePatterns: string[];
+}
+
+/**
+ * Get an object that describes what assets to upload, if any.
+ *
+ * Uses the args (passed from the command line) if available,
+ * falling back to those defined in the config.
+ *
+ * // TODO: Support for environments
+ */
+export function getAssetPaths(
+  config: Config,
+  baseDirectory = config.site?.bucket,
+  includePatterns = config.site?.include ?? [],
+  excludePatterns = config.site?.exclude ?? []
+): undefined | AssetPaths {
+  return baseDirectory
+    ? {
+        baseDirectory,
+        includePatterns,
+        excludePatterns,
+      }
+    : undefined;
+}