wrangler 0.0.0-e2a26a0 → 0.0.0-e322475

package/src/r2.ts

@@ -0,0 +1,50 @@
+import { fetchResult } from "./cfetch";
+
+/**
+ * Information about a bucket, returned from `listR2Buckets()`.
+ */
+export interface R2BucketInfo {
+  name: string;
+  creation_date: string;
+}
+
+/**
+ * Fetch a list of all the buckets under the given `accountId`.
+ */
+export async function listR2Buckets(
+  accountId: string
+): Promise<R2BucketInfo[]> {
+  const results = await fetchResult<{
+    buckets: R2BucketInfo[];
+  }>(`/accounts/${accountId}/r2/buckets`);
+  return results.buckets;
+}
+
+/**
+ * Create a bucket with the given `bucketName` within the account given by `accountId`.
+ *
+ * A 400 is returned if the account already owns a bucket with this name.
+ * A bucket must be explicitly deleted to be replaced.
+ */
+export async function createR2Bucket(
+  accountId: string,
+  bucketName: string
+): Promise<void> {
+  return await fetchResult<void>(
+    `/accounts/${accountId}/r2/buckets/${bucketName}`,
+    { method: "PUT" }
+  );
+}
+
+/**
+ * Delete a bucket with the given name
+ */
+export async function deleteR2Bucket(
+  accountId: string,
+  bucketName: string
+): Promise<void> {
+  return await fetchResult<void>(
+    `/accounts/${accountId}/r2/buckets/${bucketName}`,
+    { method: "DELETE" }
+  );
+}

package/src/selfsigned.d.ts

@@ -0,0 +1,29 @@
+declare module "selfsigned" {
+  import type { pki } from "node-forge";
+
+  export type Attributes = pki.CertificateField[];
+
+  export interface Options {
+    days?: number;
+    keySize?: number;
+    extensions?: unknown[];
+    algorithm?: string;
+    pkcs7?: boolean;
+    clientCertificate?: boolean;
+    clientCertificateCN?: string;
+  }
+
+  export interface Certificate {
+    private: string;
+    public: string;
+    cert: string;
+    fingerprint: string;
+  }
+
+  export function generate(attrs?: Attributes, options?: Options): Certificate;
+  export function generate(
+    attrs: Attributes,
+    options: Options,
+    callback: (err: Error, certificate: Certificate) => void
+  ): void;
+}

package/src/sites.tsx

@@ -1,33 +1,35 @@
+import assert from "node:assert";
 import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
 import ignore from "ignore";
-import { XXHash64 } from "xxhash-addon";
+import xxhash from "xxhash-wasm";
 import {
-  createNamespace,
-  listNamespaceKeys,
-  listNamespaces,
-  putBulkKeyValue,
+  createKVNamespace,
+  listKVNamespaceKeys,
+  listKVNamespaces,
+  putKVBulkKeyValue,
+  deleteKVBulkKeyValue,
 } from "./kv";
+import { logger } from "./logger";
 import type { Config } from "./config";
+import type { KeyValue } from "./kv";
+import type { XXHashAPI } from "xxhash-wasm";
 
 /** Paths to always ignore. */
-const ALWAYS_IGNORE = ["node_modules"];
-const HIDDEN_FILES_TO_INCLUDE = [
+const ALWAYS_IGNORE = new Set(["node_modules"]);
+const HIDDEN_FILES_TO_INCLUDE = new Set([
   ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
-];
+]);
 
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
     // Skip files that we never want to process.
-    if (ALWAYS_IGNORE.some((p) => file.name === p)) {
+    if (ALWAYS_IGNORE.has(file.name)) {
       continue;
     }
     // Skip hidden files (starting with .) except for some special ones
-    if (
-      file.name.startsWith(".") &&
-      !HIDDEN_FILES_TO_INCLUDE.some((p) => file.name === p)
-    ) {
+    if (file.name.startsWith(".") && !HIDDEN_FILES_TO_INCLUDE.has(file.name)) {
       continue;
     }
     // TODO: follow symlinks??
@@ -46,11 +48,8 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
  * the most important thing here is to detect changes of a single file to invalidate the cache and
  * it's impossible to serve two different files with the same name
  */
-function hashFileContent(content: string): string {
-  const hasher = new XXHash64();
-  hasher.update(Buffer.from(content));
-  const hash = hasher.digest();
-  return hash.toString("hex").substring(0, 10);
+function hashFileContent(hasher: XXHashAPI, content: string): string {
+  return hasher.h64ToString(content).substring(0, 10);
 }
 
 /**
@@ -59,11 +58,15 @@ function hashFileContent(content: string): string {
  * The key will change if the file path or content of the asset changes.
  * The algorithm used here matches that of Wrangler 1.
  */
-function hashAsset(filePath: string, content: string): string {
+function hashAsset(
+  hasher: XXHashAPI,
+  filePath: string,
+  content: string
+): string {
   const extName = path.extname(filePath) || "";
   const baseName = path.basename(filePath, extName);
   const directory = path.dirname(filePath);
-  const hash = hashFileContent(content);
+  const hash = hashFileContent(hasher, content);
   return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
 }
 
@@ -73,15 +76,15 @@ async function createKVNamespaceIfNotAlreadyExisting(
 ) {
   // check if it already exists
   // TODO: this is super inefficient, should be made better
-  const namespaces = await listNamespaces(accountId);
+  const namespaces = await listKVNamespaces(accountId);
   const found = namespaces.find((x) => x.title === title);
   if (found) {
     return { created: false, id: found.id };
   }
 
   // else we make the namespace
-  const id = await createNamespace(accountId, title);
-  console.log(`🌀 Created namespace for Workers Site "${title}"`);
+  const id = await createKVNamespace(accountId, title);
+  logger.log(`🌀 Created namespace for Workers Site "${title}"`);
 
   return {
     created: true,
@@ -97,16 +100,15 @@ async function createKVNamespaceIfNotAlreadyExisting(
  * @param scriptName the name of the worker whose assets we are uploading.
  * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
  * @param preview if true then upload to a "preview" KV namespace.
- * @param _env (not implemented).
  * @returns a promise for an object mapping the relative paths of the assets to the key of that
  * asset in the KV namespace.
  */
 export async function syncAssets(
-  accountId: string,
+  accountId: string | undefined,
   scriptName: string,
   siteAssets: AssetPaths | undefined,
   preview: boolean,
-  env: string | undefined
+  dryRun: boolean | undefined
 ): Promise<{
   manifest: { [filePath: string]: string } | undefined;
   namespace: string | undefined;
@@ -115,57 +117,113 @@ export async function syncAssets(
     return { manifest: undefined, namespace: undefined };
   }
 
-  const title = `__${scriptName}${env ? `-${env}` : ""}-workers_sites_assets${
+  if (dryRun) {
+    logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
+    return { manifest: undefined, namespace: undefined };
+  }
+  assert(accountId, "Missing accountId");
+
+  const title = `__${scriptName}-workers_sites_assets${
     preview ? "_preview" : ""
   }`;
+
   const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
     title,
     accountId
   );
 
   // let's get all the keys in this namespace
-  const result = await listNamespaceKeys(accountId, namespace);
-  const keys = new Set(result.map((x) => x.name));
+  const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
+  const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
+
+  const manifest: Record<string, string> = {};
 
-  const manifest = {};
-  const upload: {
-    key: string;
-    value: string;
-    base64: boolean;
-  }[] = [];
+  // A batch of uploads where each bucket has to be less than 100mb
+  const uploadBuckets: KeyValue[][] = [];
+  // The "live" bucket that we'll keep filling until it's just below 100mb
+  let uploadBucket: KeyValue[] = [];
+  // A size counter for the live bucket
+  let uploadBucketSize = 0;
 
   const include = createPatternMatcher(siteAssets.includePatterns, false);
   const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
-  // TODO: this can be more efficient by parallelising
-  for await (const file of getFilesInFolder(siteAssets.baseDirectory)) {
-    if (!include(file)) {
+  const hasher = await xxhash();
+
+  const assetDirectory = path.join(
+    siteAssets.baseDirectory,
+    siteAssets.assetDirectory
+  );
+  for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
+    const assetFile = path.relative(assetDirectory, absAssetFile);
+    if (!include(assetFile)) {
       continue;
     }
-    if (exclude(file)) {
+    if (exclude(assetFile)) {
       continue;
     }
 
-    await validateAssetSize(file);
-    console.log(`reading ${file}...`);
-    const content = await readFile(file, "base64");
-
-    const assetKey = hashAsset(file, content);
+    logger.log(`Reading ${assetFile}...`);
+    const content = await readFile(absAssetFile, "base64");
+    await validateAssetSize(absAssetFile, assetFile);
+    // while KV accepts files that are 25 MiB **before** b64 encoding
+    // the overall bucket size must be below 100 MiB **after** b64 encoding
+    const assetSize = Buffer.from(content).length;
+    const assetKey = hashAsset(hasher, assetFile, content);
     validateAssetKey(assetKey);
 
     // now put each of the files into kv
-    if (!keys.has(assetKey)) {
-      console.log(`uploading as ${assetKey}...`);
-      upload.push({
+    if (!namespaceKeys.has(assetKey)) {
+      logger.log(`Uploading as ${assetKey}...`);
+
+      // Check if adding this asset to the bucket would
+      // push it over the 100 MiB limit KV bulk API limit
+      if (uploadBucketSize + assetSize > 100 * 1024 * 1024) {
+        // If so, move the current bucket into the batch,
+        // and reset the counter/bucket
+        uploadBuckets.push(uploadBucket);
+        uploadBucketSize = 0;
+        uploadBucket = [];
+      }
+
+      // Update the bucket and the size counter
+      uploadBucketSize += assetSize;
+      uploadBucket.push({
         key: assetKey,
         value: content,
         base64: true,
       });
     } else {
-      console.log(`skipping - already uploaded`);
+      logger.log(`Skipping - already uploaded.`);
     }
-    manifest[path.relative(siteAssets.baseDirectory, file)] = assetKey;
+
+    // Remove the key from the set so we know what we've already uploaded
+    namespaceKeys.delete(assetKey);
+
+    // Prevent different manifest keys on windows
+    const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
+    manifest[manifestKey] = assetKey;
+  }
+
+  // Add the last (potentially only) bucket to the batch
+  uploadBuckets.push(uploadBucket);
+
+  // keys now contains all the files we're deleting
+  for (const key of namespaceKeys) {
+    logger.log(`Deleting ${key} from the asset store...`);
+  }
+
+  // upload each bucket in parallel
+  const bucketsToPut = [];
+  for (const bucket of uploadBuckets) {
+    bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
   }
-  await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
+  await Promise.all(bucketsToPut);
+
+  // then delete all the assets that aren't used anymore
+  await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
+
+  logger.log("↗️  Done syncing assets");
+
   return { manifest, namespace };
 }
 
@@ -181,11 +239,20 @@ function createPatternMatcher(
   }
 }
 
-async function validateAssetSize(filePath: string) {
-  const { size } = await stat(filePath);
+/**
+ * validate that the passed-in file is below 25 MiB
+ * **PRIOR** to base64 encoding. 25 MiB is a KV limit
+ * @param absFilePath
+ * @param relativeFilePath
+ */
+async function validateAssetSize(
+  absFilePath: string,
+  relativeFilePath: string
+): Promise<void> {
+  const { size } = await stat(absFilePath);
   if (size > 25 * 1024 * 1024) {
     throw new Error(
-      `File ${filePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
+      `File ${relativeFilePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
     );
   }
 }
@@ -211,8 +278,23 @@ function urlSafe(filePath: string): string {
  * Information about the assets that should be uploaded
  */
 export interface AssetPaths {
+  /**
+   * Absolute path to the root of the project.
+   *
+   * This is the directory containing wrangler.toml or cwd if no config.
+   */
   baseDirectory: string;
+  /**
+   * The path to the assets directory, relative to the `baseDirectory`.
+   */
+  assetDirectory: string;
+  /**
+   * An array of patterns that match files that should be uploaded.
+   */
   includePatterns: string[];
+  /**
+   * An array of patterns that match files that should not be uploaded.
+   */
   excludePatterns: string[];
 }
 
@@ -222,17 +304,63 @@ export interface AssetPaths {
  * Uses the args (passed from the command line) if available,
  * falling back to those defined in the config.
  *
- * // TODO: Support for environments
+ * (This function corresponds to --assets/config.assets)
+ *
  */
 export function getAssetPaths(
   config: Config,
-  baseDirectory = config.site?.bucket,
+  assetDirectory: string | undefined
+): AssetPaths | undefined {
+  const baseDirectory = path.resolve(
+    path.dirname(config.configPath ?? "wrangler.toml")
+  );
+
+  assetDirectory ??=
+    typeof config.assets === "string"
+      ? config.assets
+      : config.assets !== undefined
+      ? config.assets.bucket
+      : undefined;
+
+  const includePatterns =
+    (typeof config.assets !== "string" && config.assets?.include) || [];
+
+  const excludePatterns =
+    (typeof config.assets !== "string" && config.assets?.exclude) || [];
+
+  return assetDirectory
+    ? {
+        baseDirectory,
+        assetDirectory,
+        includePatterns,
+        excludePatterns,
+      }
+    : undefined;
+}
+
+/**
+ * Get an object that describes what site assets to upload, if any.
+ *
+ * Uses the args (passed from the command line) if available,
+ * falling back to those defined in the config.
+ *
+ * (This function corresponds to --site/config.site)
+ *
+ */
+export function getSiteAssetPaths(
+  config: Config,
+  assetDirectory = config.site?.bucket,
   includePatterns = config.site?.include ?? [],
   excludePatterns = config.site?.exclude ?? []
-): undefined | AssetPaths {
-  return baseDirectory
+): AssetPaths | undefined {
+  const baseDirectory = path.resolve(
+    path.dirname(config.configPath ?? "wrangler.toml")
+  );
+
+  return assetDirectory
     ? {
         baseDirectory,
+        assetDirectory,
         includePatterns,
         excludePatterns,
       }