wrangler 0.0.0-e1c615f → 0.0.0-e3a458e

package/src/r2.ts

@@ -0,0 +1,50 @@
+import { fetchResult } from "./cfetch";
+
+/**
+ * Information about a bucket, returned from `listR2Buckets()`.
+ */
+export interface R2BucketInfo {
+  name: string;
+  creation_date: string;
+}
+
+/**
+ * Fetch a list of all the buckets under the given `accountId`.
+ */
+export async function listR2Buckets(
+  accountId: string
+): Promise<R2BucketInfo[]> {
+  const results = await fetchResult<{
+    buckets: R2BucketInfo[];
+  }>(`/accounts/${accountId}/r2/buckets`);
+  return results.buckets;
+}
+
+/**
+ * Create a bucket with the given `bucketName` within the account given by `accountId`.
+ *
+ * A 400 is returned if the account already owns a bucket with this name.
+ * A bucket must be explicitly deleted to be replaced.
+ */
+export async function createR2Bucket(
+  accountId: string,
+  bucketName: string
+): Promise<void> {
+  return await fetchResult<void>(
+    `/accounts/${accountId}/r2/buckets/${bucketName}`,
+    { method: "PUT" }
+  );
+}
+
+/**
+ * Delete a bucket with the given name
+ */
+export async function deleteR2Bucket(
+  accountId: string,
+  bucketName: string
+): Promise<void> {
+  return await fetchResult<void>(
+    `/accounts/${accountId}/r2/buckets/${bucketName}`,
+    { method: "DELETE" }
+  );
+}

package/src/reporting.ts

@@ -0,0 +1,131 @@
+import * as fs from "node:fs";
+import { readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import TOML from "@iarna/toml";
+import { RewriteFrames } from "@sentry/integrations";
+import {
+  captureException,
+  getCurrentHub,
+  startTransaction,
+  init,
+  Integrations,
+  setContext,
+} from "@sentry/node";
+import { execaSync } from "execa";
+import prompts from "prompts";
+import * as pkj from "../package.json";
+
+export function initReporting() {
+  init({
+    release: `${pkj.name}@${pkj.version}`,
+    initialScope: {
+      tags: { [pkj.name]: pkj.version },
+    },
+    beforeBreadcrumb: (_breadcrumb, _hint) => {
+      // eventually, we might want to actually start dropping our own breadcrumbs.
+      // until then, we want to just ignore all of them, since sentry will
+      // automatically include breadcrumbs for things like console output,
+      // which can include unsanitized data that the user logs
+      // from their worker during development
+      return null;
+    },
+    dsn: "https://5089b76bf8a64a9c949bf5c2b5e8003c@o51786.ingest.sentry.io/6190959",
+    tracesSampleRate: 1.0,
+    integrations: [
+      new RewriteFrames({
+        root: "",
+        prefix: "/",
+      }),
+      new Integrations.Http({ tracing: true }),
+    ],
+  });
+
+  setContext("System Information", {
+    OS: process.platform,
+    node: process.version,
+    npm: execaSync("npm", ["--version"]).stdout,
+    wrangler: pkj.version,
+  });
+}
+
+async function appendReportingDecision(userInput: "true" | "false") {
+  const homePath = path.join(os.homedir(), ".wrangler/config/");
+  const reportingConfigLocation = path.join(homePath, "reporting.toml");
+  try {
+    fs.mkdirSync(homePath, { recursive: true });
+    await writeFile(
+      reportingConfigLocation,
+      `error_tracking_opt = ${userInput} # Sentry \nerror_tracking_opt_date = ${new Date().toISOString()} # Sentry Date Decision \n`,
+      "utf-8"
+    );
+  } catch (err) {
+    console.warn(
+      `Unable to create wrangler's error reporting configuration file at ${reportingConfigLocation}: `,
+      err
+    );
+  }
+}
+
+function exceptionTransaction(error: Error, origin = "") {
+  const transaction = startTransaction({
+    op: origin,
+    name: error.name,
+  });
+  captureException(error);
+  transaction.finish();
+}
+
+export async function reportError(err: Error, origin = "") {
+  if (!process.stdout.isTTY) return await appendReportingDecision("false");
+
+  const errorTrackingOpt = await reportingPermission();
+
+  if (errorTrackingOpt === undefined) {
+    const userInput = await prompts.prompt({
+      type: "select",
+      name: "sentryDecision",
+      message: "Would you like to submit a report when an error occurs?",
+      choices: [
+        { title: "Always", value: true },
+        { title: "Only this time", value: "once" },
+        { title: "No", value: false },
+      ],
+      initial: 2,
+    });
+
+    if (userInput.sentryDecision === "once") {
+      exceptionTransaction(err, origin);
+      return;
+    }
+
+    userInput.sentryDecision
+      ? await appendReportingDecision("true")
+      : await appendReportingDecision("false");
+  }
+
+  if (!errorTrackingOpt) {
+    const sentryClient = getCurrentHub().getClient();
+    if (sentryClient !== undefined) sentryClient.getOptions().enabled = false;
+  }
+
+  exceptionTransaction(err, origin);
+}
+
+async function reportingPermission() {
+  if (
+    !fs.existsSync(path.join(os.homedir(), ".wrangler/config/reporting.toml"))
+  )
+    return undefined;
+
+  const reportingTOML = TOML.parse(
+    await readFile(path.join(os.homedir(), ".wrangler/config/reporting.toml"), {
+      encoding: "utf-8",
+    })
+  );
+  const { error_tracking_opt } = reportingTOML as {
+    error_tracking_opt: string | undefined;
+  };
+
+  return error_tracking_opt;
+}

package/src/selfsigned.d.ts

@@ -0,0 +1,29 @@
+declare module "selfsigned" {
+  import type { pki } from "node-forge";
+
+  export type Attributes = pki.CertificateField[];
+
+  export interface Options {
+    days?: number;
+    keySize?: number;
+    extensions?: unknown[];
+    algorithm?: string;
+    pkcs7?: boolean;
+    clientCertificate?: boolean;
+    clientCertificateCN?: string;
+  }
+
+  export interface Certificate {
+    private: string;
+    public: string;
+    cert: string;
+    fingerprint: string;
+  }
+
+  export function generate(attrs?: Attributes, options?: Options): Certificate;
+  export function generate(
+    attrs: Attributes,
+    options: Options,
+    callback: (err: Error, certificate: Certificate) => void
+  ): void;
+}

package/src/sites.tsx

@@ -1,13 +1,37 @@
-import crypto from "node:crypto";
-import { createReadStream } from "node:fs";
+import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
-import { readdir, readFile } from "node:fs/promises";
-import { fetchResult } from "./cfetch";
-import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
+import ignore from "ignore";
+import xxhash from "xxhash-wasm";
+import {
+  createNamespace,
+  getKeyValue,
+  listNamespaceKeys,
+  listNamespaces,
+  putBulkKeyValue,
+} from "./kv";
+import type { Config } from "./config";
+import type { KeyValue, NamespaceKeyInfo } from "./kv";
+import type { XXHashAPI } from "xxhash-wasm";
+
+/** Paths to always ignore. */
+const ALWAYS_IGNORE = new Set(["node_modules"]);
+const HIDDEN_FILES_TO_INCLUDE = new Set([
+  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
+]);
+const FIVE_MINUTES = 60 * 5; // expressed in seconds, since that's what the api expects
 
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
+    // Skip files that we never want to process.
+    if (ALWAYS_IGNORE.has(file.name)) {
+      continue;
+    }
+    // Skip hidden files (starting with .) except for some special ones
+    if (file.name.startsWith(".") && !HIDDEN_FILES_TO_INCLUDE.has(file.name)) {
+      continue;
+    }
+    // TODO: follow symlinks??
     if (file.isDirectory()) {
       yield* await getFilesInFolder(path.join(dirPath, file.name));
     } else {
@@ -16,27 +40,33 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   }
 }
 
-async function hashFileContent(filePath: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const hash = crypto.createHash("sha1");
-    const rs = createReadStream(filePath);
-    rs.on("error", reject);
-    rs.on("data", (chunk) => hash.update(chunk));
-    rs.on("end", () => resolve(hash.digest("hex")));
-  });
+/**
+ * Create a hash key for the given content using the xxhash algorithm.
+ *
+ * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key
+ * the most important thing here is to detect changes of a single file to invalidate the cache and
+ * it's impossible to serve two different files with the same name
+ */
+function hashFileContent(hasher: XXHashAPI, content: string): string {
+  return hasher.h64ToString(content).substring(0, 10);
 }
 
-async function hashFile(filePath: string): Promise<{
-  filePath: string;
-  hash: string;
-}> {
-  const extName = path.extname(filePath);
+/**
+ * Create a hashed asset key for the given asset.
+ *
+ * The key will change if the file path or content of the asset changes.
+ * The algorithm used here matches that of Wrangler 1.
+ */
+function hashAsset(
+  hasher: XXHashAPI,
+  filePath: string,
+  content: string
+): string {
+  const extName = path.extname(filePath) || "";
   const baseName = path.basename(filePath, extName);
-  const hash = await hashFileContent(filePath);
-  return {
-    filePath: `${baseName}.${hash}${extName || ""}`,
-    hash,
-  };
+  const directory = path.dirname(filePath);
+  const hash = hashFileContent(hasher, content);
+  return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
 }
 
 async function createKVNamespaceIfNotAlreadyExisting(
@@ -52,63 +82,192 @@ async function createKVNamespaceIfNotAlreadyExisting(
   }
 
   // else we make the namespace
-  // TODO: use an export from ./kv
-  const json = await fetchResult<{ id: string }>(
-    `/accounts/${accountId}/storage/kv/namespaces`,
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ title }),
-    }
-  );
+  const id = await createNamespace(accountId, title);
+  console.log(`🌀 Created namespace for Workers Site "${title}"`);
 
   return {
     created: true,
-    id: json.id,
+    id,
   };
 }
 
+/**
+ * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
+ * the worker given by `scriptName`.
+ *
+ * @param accountId the account to upload to.
+ * @param scriptName the name of the worker whose assets we are uploading.
+ * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
+ * @param preview if true then upload to a "preview" KV namespace.
+ * @returns a promise for an object mapping the relative paths of the assets to the key of that
+ * asset in the KV namespace.
+ */
 export async function syncAssets(
   accountId: string,
   scriptName: string,
-  dirPath: string,
-  preview: boolean,
-  _env?: string
-) {
-  const title = `__${scriptName}_sites_assets${preview ? "_preview" : ""}`;
+  siteAssets: AssetPaths | undefined,
+  preview: boolean
+): Promise<{
+  manifest: { [filePath: string]: string } | undefined;
+  namespace: string | undefined;
+}> {
+  if (siteAssets === undefined) {
+    return { manifest: undefined, namespace: undefined };
+  }
+
+  const title = `__${scriptName}-workers_sites_assets${
+    preview ? "_preview" : ""
+  }`;
   const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
     title,
     accountId
   );
 
   // let's get all the keys in this namespace
-  const keys = new Set(
-    (await listNamespaceKeys(accountId, namespace)).map((x) => x.name)
+  const result = await listNamespaceKeys(accountId, namespace);
+  const keyMap = result.reduce<Record<string, NamespaceKeyInfo>>(
+    (km, key) => Object.assign(km, { [key.name]: key }),
+    {}
   );
 
-  const manifest = {};
-  const upload = [];
-  // TODO: this can be more efficient by parallelising
-  for await (const file of getFilesInFolder(dirPath)) {
-    // TODO: "exclude:" config
-    const { filePath } = await hashFile(file);
+  const manifest: Record<string, string> = {};
+  const toUpload: KeyValue[] = [];
+
+  const include = createPatternMatcher(siteAssets.includePatterns, false);
+  const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
+  const hasher = await xxhash();
+
+  for await (const file of getFilesInFolder(siteAssets.baseDirectory)) {
+    if (!include(file)) {
+      continue;
+    }
+    if (exclude(file)) {
+      continue;
+    }
+
+    await validateAssetSize(file);
+    console.log(`reading ${file}...`);
+    const content = await readFile(file, "base64");
+
+    const assetKey = hashAsset(hasher, file, content);
+    validateAssetKey(assetKey);
+
     // now put each of the files into kv
-    if (!keys.has(filePath)) {
-      console.log(`uploading ${file}...`);
-      const content = await readFile(file, "base64");
-      if (content.length > 25 * 1024 * 1024) {
-        throw new Error(`File ${file} is too big, it should be under 25 mb.`);
-      }
-      upload.push({
-        key: filePath,
+    if (!(assetKey in keyMap) || keyMap[assetKey].expiration) {
+      console.log(`uploading as ${assetKey}...`);
+      toUpload.push({
+        key: assetKey,
         value: content,
         base64: true,
       });
+    } else {
+      console.log(`skipping - already uploaded`);
     }
-    manifest[path.relative(dirPath, file)] = filePath;
+    // remove the key from the set so we know we've seen it
+    delete keyMap[assetKey];
+    manifest[path.relative(siteAssets.baseDirectory, file)] = assetKey;
   }
-  await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
+
+  // `keyMap` now contains the assets that we need to expire
+
+  let toExpire: KeyValue[] = [];
+  for (const asset of Object.values(keyMap)) {
+    if (!asset.expiration) {
+      console.log(`expiring unused ${asset.name}...`);
+      toExpire.push({
+        key: asset.name,
+        value: "", // we'll fill all the values in one go
+        // we use expiration_ttl, since we can't trust the time
+        // that a deploy source might provide (eg - https://github.com/cloudflare/wrangler/issues/2224)
+        expiration_ttl: FIVE_MINUTES,
+        base64: true,
+      });
+    }
+  }
+
+  // TODO: batch these in groups if it causes problems
+  toExpire = await Promise.all(
+    toExpire.map(async (asset) => ({
+      ...asset,
+      // it would be great if we didn't have to do this fetch at all
+      value: await getKeyValue(accountId, namespace, asset.key),
+    }))
+  );
+
+  await putBulkKeyValue(
+    accountId,
+    namespace,
+    toUpload.concat(toExpire),
+    () => {}
+  );
   return { manifest, namespace };
 }
+
+function createPatternMatcher(
+  patterns: string[],
+  exclude: boolean
+): (filePath: string) => boolean {
+  if (patterns.length === 0) {
+    return (_filePath) => !exclude;
+  } else {
+    const ignorer = ignore().add(patterns);
+    return (filePath) => ignorer.test(filePath).ignored;
+  }
+}
+
+async function validateAssetSize(filePath: string) {
+  const { size } = await stat(filePath);
+  if (size > 25 * 1024 * 1024) {
+    throw new Error(
+      `File ${filePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
+    );
+  }
+}
+
+function validateAssetKey(assetKey: string) {
+  if (assetKey.length > 512) {
+    throw new Error(
+      `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",`
+    );
+  }
+}
+
+/**
+ * Convert a filePath to be safe to use as a relative URL.
+ *
+ * Primarily this involves converting Windows backslashes to forward slashes.
+ */
+function urlSafe(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+/**
+ * Information about the assets that should be uploaded
+ */
+export interface AssetPaths {
+  baseDirectory: string;
+  includePatterns: string[];
+  excludePatterns: string[];
+}
+
+/**
+ * Get an object that describes what assets to upload, if any.
+ *
+ * Uses the args (passed from the command line) if available,
+ * falling back to those defined in the config.
+ *
+ */
+export function getAssetPaths(
+  config: Config,
+  baseDirectory = config.site?.bucket,
+  includePatterns = config.site?.include ?? [],
+  excludePatterns = config.site?.exclude ?? []
+): undefined | AssetPaths {
+  return baseDirectory
+    ? {
+        baseDirectory,
+        includePatterns,
+        excludePatterns,
+      }
+    : undefined;
+}