wrangler 0.0.0-ece06ea → 0.0.0-ed08df4

package/src/r2.ts

@@ -0,0 +1,50 @@
+import { fetchResult } from "./cfetch";
+
+/**
+ * Information about a bucket, returned from `listR2Buckets()`.
+ */
+export interface R2BucketInfo {
+  name: string;
+  creation_date: string;
+}
+
+/**
+ * Fetch a list of all the buckets under the given `accountId`.
+ */
+export async function listR2Buckets(
+  accountId: string
+): Promise<R2BucketInfo[]> {
+  const results = await fetchResult<{
+    buckets: R2BucketInfo[];
+  }>(`/accounts/${accountId}/r2/buckets`);
+  return results.buckets;
+}
+
+/**
+ * Create a bucket with the given `bucketName` within the account given by `accountId`.
+ *
+ * A 400 is returned if the account already owns a bucket with this name.
+ * A bucket must be explicitly deleted to be replaced.
+ */
+export async function createR2Bucket(
+  accountId: string,
+  bucketName: string
+): Promise<void> {
+  return await fetchResult<void>(
+    `/accounts/${accountId}/r2/buckets/${bucketName}`,
+    { method: "PUT" }
+  );
+}
+
+/**
+ * Delete a bucket with the given name
+ */
+export async function deleteR2Bucket(
+  accountId: string,
+  bucketName: string
+): Promise<void> {
+  return await fetchResult<void>(
+    `/accounts/${accountId}/r2/buckets/${bucketName}`,
+    { method: "DELETE" }
+  );
+}

package/src/selfsigned.d.ts

@@ -0,0 +1,29 @@
+declare module "selfsigned" {
+  import type { pki } from "node-forge";
+
+  export type Attributes = pki.CertificateField[];
+
+  export interface Options {
+    days?: number;
+    keySize?: number;
+    extensions?: unknown[];
+    algorithm?: string;
+    pkcs7?: boolean;
+    clientCertificate?: boolean;
+    clientCertificateCN?: string;
+  }
+
+  export interface Certificate {
+    private: string;
+    public: string;
+    cert: string;
+    fingerprint: string;
+  }
+
+  export function generate(attrs?: Attributes, options?: Options): Certificate;
+  export function generate(
+    attrs: Attributes,
+    options: Options,
+    callback: (err: Error, certificate: Certificate) => void
+  ): void;
+}

package/src/sites.tsx

@@ -1,14 +1,37 @@
-import { readdir, readFile } from "node:fs/promises";
-import { createReadStream } from "node:fs";
-import cfetch from "./cfetch";
-import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
+import { readdir, readFile, stat } from "node:fs/promises";
+import * as path from "node:path";
+import ignore from "ignore";
+import xxhash from "xxhash-wasm";
+import {
+  createKVNamespace,
+  listKVNamespaceKeys,
+  listKVNamespaces,
+  putKVBulkKeyValue,
+  deleteKVBulkKeyValue,
+} from "./kv";
+import { logger } from "./logger";
+import type { Config } from "./config";
+import type { KeyValue } from "./kv";
+import type { XXHashAPI } from "xxhash-wasm";
 
-import * as path from "path";
-import crypto from "node:crypto";
+/** Paths to always ignore. */
+const ALWAYS_IGNORE = new Set(["node_modules"]);
+const HIDDEN_FILES_TO_INCLUDE = new Set([
+  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
+]);
 
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
+    // Skip files that we never want to process.
+    if (ALWAYS_IGNORE.has(file.name)) {
+      continue;
+    }
+    // Skip hidden files (starting with .) except for some special ones
+    if (file.name.startsWith(".") && !HIDDEN_FILES_TO_INCLUDE.has(file.name)) {
+      continue;
+    }
+    // TODO: follow symlinks??
     if (file.isDirectory()) {
       yield* await getFilesInFolder(path.join(dirPath, file.name));
     } else {
@@ -17,27 +40,33 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   }
 }
 
-async function hashFileContent(filePath: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const hash = crypto.createHash("sha1");
-    const rs = createReadStream(filePath);
-    rs.on("error", reject);
-    rs.on("data", (chunk) => hash.update(chunk));
-    rs.on("end", () => resolve(hash.digest("hex")));
-  });
+/**
+ * Create a hash key for the given content using the xxhash algorithm.
+ *
+ * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key
+ * the most important thing here is to detect changes of a single file to invalidate the cache and
+ * it's impossible to serve two different files with the same name
+ */
+function hashFileContent(hasher: XXHashAPI, content: string): string {
+  return hasher.h64ToString(content).substring(0, 10);
 }
 
-async function hashFile(filePath: string): Promise<{
-  filePath: string;
-  hash: string;
-}> {
-  const extName = path.extname(filePath);
+/**
+ * Create a hashed asset key for the given asset.
+ *
+ * The key will change if the file path or content of the asset changes.
+ * The algorithm used here matches that of Wrangler 1.
+ */
+function hashAsset(
+  hasher: XXHashAPI,
+  filePath: string,
+  content: string
+): string {
+  const extName = path.extname(filePath) || "";
   const baseName = path.basename(filePath, extName);
-  const hash = await hashFileContent(filePath);
-  return {
-    filePath: `${baseName}.${hash}${extName || ""}`,
-    hash,
-  };
+  const directory = path.dirname(filePath);
+  const hash = hashFileContent(hasher, content);
+  return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
 }
 
 async function createKVNamespaceIfNotAlreadyExisting(
@@ -46,70 +75,219 @@ async function createKVNamespaceIfNotAlreadyExisting(
 ) {
   // check if it already exists
   // TODO: this is super inefficient, should be made better
-  const namespaces = await listNamespaces(accountId);
+  const namespaces = await listKVNamespaces(accountId);
   const found = namespaces.find((x) => x.title === title);
   if (found) {
     return { created: false, id: found.id };
   }
 
   // else we make the namespace
-  // TODO: use an export from ./kv
-  const json = await cfetch<{ id: string }>(
-    `/accounts/${accountId}/storage/kv/namespaces`,
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ title }),
-    }
-  );
+  const id = await createKVNamespace(accountId, title);
+  logger.log(`🌀 Created namespace for Workers Site "${title}"`);
 
   return {
     created: true,
-    id: json.id,
+    id,
   };
 }
 
+/**
+ * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
+ * the worker given by `scriptName`.
+ *
+ * @param accountId the account to upload to.
+ * @param scriptName the name of the worker whose assets we are uploading.
+ * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
+ * @param preview if true then upload to a "preview" KV namespace.
+ * @returns a promise for an object mapping the relative paths of the assets to the key of that
+ * asset in the KV namespace.
+ */
 export async function syncAssets(
   accountId: string,
   scriptName: string,
-  dirPath: string,
+  siteAssets: AssetPaths | undefined,
   preview: boolean,
-  _env?: string
-) {
-  const title = `__${scriptName}_sites_assets${preview ? "_preview" : ""}`;
+  dryRun: boolean | undefined
+): Promise<{
+  manifest: { [filePath: string]: string } | undefined;
+  namespace: string | undefined;
+}> {
+  if (siteAssets === undefined) {
+    return { manifest: undefined, namespace: undefined };
+  }
+
+  if (dryRun) {
+    logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
+    return { manifest: undefined, namespace: undefined };
+  }
+
+  const title = `__${scriptName}-workers_sites_assets${
+    preview ? "_preview" : ""
+  }`;
+
   const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
     title,
     accountId
   );
 
   // let's get all the keys in this namespace
-  const keys = new Set(
-    (await listNamespaceKeys(accountId, namespace)).map((x) => x.name)
+  const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
+  const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
+
+  const manifest: Record<string, string> = {};
+  const toUpload: KeyValue[] = [];
+
+  const include = createPatternMatcher(siteAssets.includePatterns, false);
+  const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
+  const hasher = await xxhash();
+
+  const assetDirectory = path.join(
+    siteAssets.baseDirectory,
+    siteAssets.assetDirectory
   );
+  for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
+    const assetFile = path.relative(siteAssets.baseDirectory, absAssetFile);
+    if (!include(assetFile)) {
+      continue;
+    }
+    if (exclude(assetFile)) {
+      continue;
+    }
+
+    await validateAssetSize(absAssetFile, assetFile);
+    logger.log(`Reading ${assetFile}...`);
+    const content = await readFile(absAssetFile, "base64");
+
+    const assetKey = hashAsset(hasher, assetFile, content);
+    validateAssetKey(assetKey);
 
-  const manifest = {};
-  const upload = [];
-  // TODO: this can be more efficient by parallelising
-  for await (const file of getFilesInFolder(dirPath)) {
-    // TODO: "exclude:" config
-    const { filePath } = await hashFile(file);
     // now put each of the files into kv
-    if (!keys.has(filePath)) {
-      console.log(`uploading ${file}...`);
-      const content = await readFile(file, "base64");
-      if (content.length > 25 * 1024 * 1024) {
-        throw new Error(`File ${file} is too big, it should be under 25 mb.`);
-      }
-      upload.push({
-        key: filePath,
+    if (!namespaceKeys.has(assetKey)) {
+      logger.log(`Uploading as ${assetKey}...`);
+      toUpload.push({
+        key: assetKey,
         value: content,
         base64: true,
       });
+    } else {
+      logger.log(`Skipping - already uploaded.`);
     }
-    manifest[path.relative(dirPath, file)] = filePath;
+
+    // remove the key from the set so we know what we've already uploaded
+    namespaceKeys.delete(assetKey);
+    manifest[path.relative(siteAssets.assetDirectory, absAssetFile)] = assetKey;
   }
-  await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
+
+  // keys now contains all the files we're deleting
+  for (const key of namespaceKeys) {
+    logger.log(`Deleting ${key} from the asset store...`);
+  }
+
+  await Promise.all([
+    // upload all the new assets
+    putKVBulkKeyValue(accountId, namespace, toUpload, () => {}),
+    // delete all the unused assets
+    deleteKVBulkKeyValue(
+      accountId,
+      namespace,
+      Array.from(namespaceKeys),
+      () => {}
+    ),
+  ]);
+
+  logger.log("↗️  Done syncing assets");
+
   return { manifest, namespace };
 }
+
+function createPatternMatcher(
+  patterns: string[],
+  exclude: boolean
+): (filePath: string) => boolean {
+  if (patterns.length === 0) {
+    return (_filePath) => !exclude;
+  } else {
+    const ignorer = ignore().add(patterns);
+    return (filePath) => ignorer.test(filePath).ignored;
+  }
+}
+
+async function validateAssetSize(
+  absFilePath: string,
+  relativeFilePath: string
+) {
+  const { size } = await stat(absFilePath);
+  if (size > 25 * 1024 * 1024) {
+    throw new Error(
+      `File ${relativeFilePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
+    );
+  }
+}
+
+function validateAssetKey(assetKey: string) {
+  if (assetKey.length > 512) {
+    throw new Error(
+      `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",`
+    );
+  }
+}
+
+/**
+ * Convert a filePath to be safe to use as a relative URL.
+ *
+ * Primarily this involves converting Windows backslashes to forward slashes.
+ */
+function urlSafe(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+/**
+ * Information about the assets that should be uploaded
+ */
+export interface AssetPaths {
+  /**
+   * Absolute path to the root of the project.
+   *
+   * This is the directory containing wrangler.toml or cwd if no config.
+   */
+  baseDirectory: string;
+  /**
+   * The path to the assets directory, relative to the `baseDirectory`.
+   */
+  assetDirectory: string;
+  /**
+   * An array of patterns that match files that should be uploaded.
+   */
+  includePatterns: string[];
+  /**
+   * An array of patterns that match files that should not be uploaded.
+   */
+  excludePatterns: string[];
+}
+
+/**
+ * Get an object that describes what assets to upload, if any.
+ *
+ * Uses the args (passed from the command line) if available,
+ * falling back to those defined in the config.
+ *
+ */
+export function getAssetPaths(
+  config: Config,
+  assetDirectory = config.site?.bucket,
+  includePatterns = config.site?.include ?? [],
+  excludePatterns = config.site?.exclude ?? []
+): AssetPaths | undefined {
+  const baseDirectory = path.resolve(
+    path.dirname(config.configPath ?? "wrangler.toml")
+  );
+
+  return assetDirectory
+    ? {
+        baseDirectory,
+        assetDirectory,
+        includePatterns,
+        excludePatterns,
+      }
+    : undefined;
+}

package/src/tail/filters.ts

@@ -0,0 +1,279 @@
+/**
+ * When tailing logs from a worker, oftentimes you don't want to see _every
+ * single event_. That's where filters come in. We can send a set of filters
+ * to the tail worker, and it will pre-filter any logs for us so that we
+ * only recieve the ones we care about.
+ */
+
+/**
+ * These are the filters we accept in the CLI. They
+ * were copied directly from wrangler 1 in order to
+ * maintain compatability, so they aren't actually the exact
+ * filters we need to send up to the tail worker. They generally map 1:1,
+ * but often require some transformation or
+ * renaming to match what it expects.
+ *
+ * For a full description of each filter, either check the
+ * CLI description or see the documentation for `ApiFilter`.
+ */
+export type TailCLIFilters = {
+  status?: ("ok" | "error" | "canceled")[];
+  header?: string;
+  method?: string[];
+  search?: string;
+  samplingRate?: number;
+  clientIp?: string[];
+};
+
+/**
+ * These are the filters we send to the tail worker. We
+ * actually send a list of filters (an array of objects),
+ * so rather than having a single TailAPIFilters type,
+ * each kind of filter gets its own type and we define
+ * TailAPIFilter to be the union of those types.
+ */
+export type TailAPIFilter =
+  | SamplingRateFilter
+  | OutcomeFilter
+  | MethodFilter
+  | HeaderFilter
+  | ClientIPFilter
+  | QueryFilter;
+
+/**
+ * Filters logs based on a given sampling rate.
+ * For example, a `sampling_rate` of 0.25 will let one-quarter of the
+ * logs through.
+ */
+type SamplingRateFilter = {
+  sampling_rate: number;
+};
+
+/**
+ * Filters logs based on the outcome of the worker's event handler.
+ */
+type OutcomeFilter = {
+  outcome: Outcome[];
+};
+
+/**
+ * There are five possible outcomes we can get, three of which
+ * (exception, exceededCpu, and unknown) are considered errors
+ */
+export type Outcome =
+  | "ok"
+  | "canceled"
+  | "exception"
+  | "exceededCpu"
+  | "unknown";
+
+/**
+ * Filters logs based on the HTTP method used for the request
+ * that triggered the worker.
+ */
+type MethodFilter = {
+  method: string[];
+};
+
+/**
+ * Filters logs based on an HTTP header on the request that
+ * triggered the worker.
+ */
+type HeaderFilter = {
+  header: {
+    /**
+     * Filters on the header "key", e.g. "X-CLOUDFLARE-HEADER"
+     * or "X-CUSTOM-HEADER"
+     */
+    key: string;
+
+    /**
+     * Filters on the header "value", e.g. if this is set to
+     * "filter-for-me" and the "key" is "X-SHOULD-LOG", only
+     * events triggered by requests with the header
+     * "X-SHOULD-LOG:filter-for-me" will be logged.
+     */
+    query?: string;
+  };
+};
+
+/**
+ * Filters on the IP address the request came from that triggered
+ * the worker. A value of "self" will be replaced with the IP
+ * address that is running `wrangler tail`
+ */
+type ClientIPFilter = {
+  client_ip: string[];
+};
+
+/**
+ * Filters logs by a query string. This means only logs that
+ * contain the given string will be sent to wrangler, and any
+ * that don't will be discarded by the tail worker.
+ */
+type QueryFilter = {
+  query: string;
+};
+
+/**
+ * The full message we send to the tail worker includes our
+ * filters and a debug flag.
+ */
+export type TailFilterMessage = {
+  filters: TailAPIFilter[];
+  debug: boolean;
+};
+
+/**
+ * Translate the flags passed in via a CLI invokation of wrangler
+ * into a message that we can send to the tail worker.
+ *
+ * @param cliFilters An object containing all the filters passed in from the CLI
+ * @param debug Whether or not we should be in debug mode
+ * @returns A filter message ready to be sent to the tail worker
+ */
+export function translateCLICommandToFilterMessage(
+  cliFilters: TailCLIFilters,
+  debug: boolean
+): TailFilterMessage {
+  const apiFilters: TailAPIFilter[] = [];
+
+  if (cliFilters.samplingRate) {
+    apiFilters.push(parseSamplingRate(cliFilters.samplingRate));
+  }
+
+  if (cliFilters.status) {
+    apiFilters.push(parseOutcome(cliFilters.status));
+  }
+
+  if (cliFilters.method) {
+    apiFilters.push(parseMethod(cliFilters.method));
+  }
+
+  if (cliFilters.header) {
+    apiFilters.push(parseHeader(cliFilters.header));
+  }
+
+  if (cliFilters.clientIp) {
+    apiFilters.push(parseIP(cliFilters.clientIp));
+  }
+
+  if (cliFilters.search) {
+    apiFilters.push(parseQuery(cliFilters.search));
+  }
+
+  return {
+    filters: apiFilters,
+    debug,
+  };
+}
+
+/**
+ * Parse the sampling rate passed in via command line
+ *
+ * @param sampling_rate the sampling rate passed in via CLI
+ * @throws an Error if the rate doesn't make sense
+ * @returns a SamplingRateFilter for use with the API
+ */
+function parseSamplingRate(sampling_rate: number): SamplingRateFilter {
+  if (sampling_rate <= 0 || sampling_rate >= 1) {
+    throw new Error(
+      "A sampling rate must be between 0 and 1 in order to have any effect.\nFor example, a sampling rate of 0.25 means 25% of events will be logged."
+    );
+  }
+
+  return { sampling_rate };
+}
+
+/**
+ * Translate from CLI "status"es to API "outcome"s, including
+ * broadening "error" into "exception", "exceededCpu", and "unknown".
+ *
+ * @param statuses statuses passed in via CLI
+ * @returns an OutcomeFilter for use with the API
+ */
+function parseOutcome(
+  statuses: ("ok" | "error" | "canceled")[]
+): OutcomeFilter {
+  const outcomes = new Set<Outcome>();
+
+  for (const status of statuses) {
+    switch (status) {
+      case "ok":
+        outcomes.add("ok");
+        break;
+
+      case "canceled":
+        outcomes.add("canceled");
+        break;
+
+      case "error":
+        outcomes.add("exception");
+        outcomes.add("exceededCpu");
+        outcomes.add("unknown");
+        break;
+
+      default:
+        break;
+    }
+  }
+
+  return {
+    outcome: Array.from(outcomes),
+  };
+}
+
+/**
+ * We just send silly methods through to the API anyway, since they don't
+ * cause any harm.
+ *
+ * @param method an array of HTTP request methods passed in via CLI
+ * @returns a MethodFilter for use with the API
+ */
+function parseMethod(method: string[]): MethodFilter {
+  return { method };
+}
+
+/**
+ * Header filters can contain either just a key ("X-HEADER-KEY") or both
+ * a key and a value ("X-HEADER-KEY:some-value"). This function parses
+ * a given string according to that pattern.
+ *
+ * @param header a header string, "X-HEADER-KEY" or "X-HEADER-KEY:some-value"
+ * @returns a HeaderFilter for use with the API
+ */
+function parseHeader(header: string): HeaderFilter {
+  const [headerKey, headerQuery] = header.split(":", 2);
+
+  return {
+    header: {
+      key: headerKey.trim(),
+      query: headerQuery?.trim(),
+    },
+  };
+}
+
+/**
+ * A list of IPs can be passed in to filter for messages that come from
+ * a worker triggered by a request originating from one of those IPs.
+ * You can also pass in the string "self" to filter for the IP of the
+ * machine running `wrangler tail`.
+ *
+ * @param client_ip an array of IP addresses to filter
+ * @returns a ClientIPFilter for use with the API
+ */
+function parseIP(client_ip: string[]): ClientIPFilter {
+  return { client_ip };
+}
+
+/**
+ * Users can filter for logs that contain a "search" or a "query string".
+ * For example, if `--search findme` is passed to then we will only
+ * receive logs that contain the string "findme".
+ *
+ * @param query a query string to search for
+ * @returns a QueryFilter for use with the API
+ */
+function parseQuery(query: string): QueryFilter {
+  return { query };
+}