wrangler 0.0.0-e978986 → 0.0.0-e981d0be5

package/src/sites.tsx

@@ -1,295 +0,0 @@
-import assert from "node:assert";
-import { readdir, readFile, stat } from "node:fs/promises";
-import * as path from "node:path";
-import ignore from "ignore";
-import xxhash from "xxhash-wasm";
-import {
-  createKVNamespace,
-  listKVNamespaceKeys,
-  listKVNamespaces,
-  putKVBulkKeyValue,
-  deleteKVBulkKeyValue,
-} from "./kv";
-import { logger } from "./logger";
-import type { Config } from "./config";
-import type { KeyValue } from "./kv";
-import type { XXHashAPI } from "xxhash-wasm";
-
-/** Paths to always ignore. */
-const ALWAYS_IGNORE = new Set(["node_modules"]);
-const HIDDEN_FILES_TO_INCLUDE = new Set([
-  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
-]);
-
-async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
-  const files = await readdir(dirPath, { withFileTypes: true });
-  for (const file of files) {
-    // Skip files that we never want to process.
-    if (ALWAYS_IGNORE.has(file.name)) {
-      continue;
-    }
-    // Skip hidden files (starting with .) except for some special ones
-    if (file.name.startsWith(".") && !HIDDEN_FILES_TO_INCLUDE.has(file.name)) {
-      continue;
-    }
-    // TODO: follow symlinks??
-    if (file.isDirectory()) {
-      yield* await getFilesInFolder(path.join(dirPath, file.name));
-    } else {
-      yield path.join(dirPath, file.name);
-    }
-  }
-}
-
-/**
- * Create a hash key for the given content using the xxhash algorithm.
- *
- * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key
- * the most important thing here is to detect changes of a single file to invalidate the cache and
- * it's impossible to serve two different files with the same name
- */
-function hashFileContent(hasher: XXHashAPI, content: string): string {
-  return hasher.h64ToString(content).substring(0, 10);
-}
-
-/**
- * Create a hashed asset key for the given asset.
- *
- * The key will change if the file path or content of the asset changes.
- * The algorithm used here matches that of Wrangler 1.
- */
-function hashAsset(
-  hasher: XXHashAPI,
-  filePath: string,
-  content: string
-): string {
-  const extName = path.extname(filePath) || "";
-  const baseName = path.basename(filePath, extName);
-  const directory = path.dirname(filePath);
-  const hash = hashFileContent(hasher, content);
-  return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
-}
-
-async function createKVNamespaceIfNotAlreadyExisting(
-  title: string,
-  accountId: string
-) {
-  // check if it already exists
-  // TODO: this is super inefficient, should be made better
-  const namespaces = await listKVNamespaces(accountId);
-  const found = namespaces.find((x) => x.title === title);
-  if (found) {
-    return { created: false, id: found.id };
-  }
-
-  // else we make the namespace
-  const id = await createKVNamespace(accountId, title);
-  logger.log(`🌀 Created namespace for Workers Site "${title}"`);
-
-  return {
-    created: true,
-    id,
-  };
-}
-
-/**
- * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
- * the worker given by `scriptName`.
- *
- * @param accountId the account to upload to.
- * @param scriptName the name of the worker whose assets we are uploading.
- * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
- * @param preview if true then upload to a "preview" KV namespace.
- * @returns a promise for an object mapping the relative paths of the assets to the key of that
- * asset in the KV namespace.
- */
-export async function syncAssets(
-  accountId: string | undefined,
-  scriptName: string,
-  siteAssets: AssetPaths | undefined,
-  preview: boolean,
-  dryRun: boolean | undefined
-): Promise<{
-  manifest: { [filePath: string]: string } | undefined;
-  namespace: string | undefined;
-}> {
-  if (siteAssets === undefined) {
-    return { manifest: undefined, namespace: undefined };
-  }
-
-  if (dryRun) {
-    logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
-    return { manifest: undefined, namespace: undefined };
-  }
-  assert(accountId, "Missing accountId");
-
-  const title = `__${scriptName}-workers_sites_assets${
-    preview ? "_preview" : ""
-  }`;
-
-  const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
-    title,
-    accountId
-  );
-
-  // let's get all the keys in this namespace
-  const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
-  const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
-
-  const manifest: Record<string, string> = {};
-  const toUpload: KeyValue[] = [];
-
-  const include = createPatternMatcher(siteAssets.includePatterns, false);
-  const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
-  const hasher = await xxhash();
-
-  const assetDirectory = path.join(
-    siteAssets.baseDirectory,
-    siteAssets.assetDirectory
-  );
-  for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
-    const assetFile = path.relative(siteAssets.baseDirectory, absAssetFile);
-    if (!include(assetFile)) {
-      continue;
-    }
-    if (exclude(assetFile)) {
-      continue;
-    }
-
-    await validateAssetSize(absAssetFile, assetFile);
-    logger.log(`Reading ${assetFile}...`);
-    const content = await readFile(absAssetFile, "base64");
-
-    const assetKey = hashAsset(hasher, assetFile, content);
-    validateAssetKey(assetKey);
-
-    // now put each of the files into kv
-    if (!namespaceKeys.has(assetKey)) {
-      logger.log(`Uploading as ${assetKey}...`);
-      toUpload.push({
-        key: assetKey,
-        value: content,
-        base64: true,
-      });
-    } else {
-      logger.log(`Skipping - already uploaded.`);
-    }
-
-    // remove the key from the set so we know what we've already uploaded
-    namespaceKeys.delete(assetKey);
-
-    // prevent causing different manifest keys on windows
-    const maifestKey = urlSafe(
-      path.relative(siteAssets.assetDirectory, absAssetFile)
-    );
-    manifest[maifestKey] = assetKey;
-  }
-
-  // keys now contains all the files we're deleting
-  for (const key of namespaceKeys) {
-    logger.log(`Deleting ${key} from the asset store...`);
-  }
-
-  await Promise.all([
-    // upload all the new assets
-    putKVBulkKeyValue(accountId, namespace, toUpload),
-    // delete all the unused assets
-    deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys)),
-  ]);
-
-  logger.log("↗️  Done syncing assets");
-
-  return { manifest, namespace };
-}
-
-function createPatternMatcher(
-  patterns: string[],
-  exclude: boolean
-): (filePath: string) => boolean {
-  if (patterns.length === 0) {
-    return (_filePath) => !exclude;
-  } else {
-    const ignorer = ignore().add(patterns);
-    return (filePath) => ignorer.test(filePath).ignored;
-  }
-}
-
-async function validateAssetSize(
-  absFilePath: string,
-  relativeFilePath: string
-) {
-  const { size } = await stat(absFilePath);
-  if (size > 25 * 1024 * 1024) {
-    throw new Error(
-      `File ${relativeFilePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
-    );
-  }
-}
-
-function validateAssetKey(assetKey: string) {
-  if (assetKey.length > 512) {
-    throw new Error(
-      `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",`
-    );
-  }
-}
-
-/**
- * Convert a filePath to be safe to use as a relative URL.
- *
- * Primarily this involves converting Windows backslashes to forward slashes.
- */
-function urlSafe(filePath: string): string {
-  return filePath.replace(/\\/g, "/");
-}
-
-/**
- * Information about the assets that should be uploaded
- */
-export interface AssetPaths {
-  /**
-   * Absolute path to the root of the project.
-   *
-   * This is the directory containing wrangler.toml or cwd if no config.
-   */
-  baseDirectory: string;
-  /**
-   * The path to the assets directory, relative to the `baseDirectory`.
-   */
-  assetDirectory: string;
-  /**
-   * An array of patterns that match files that should be uploaded.
-   */
-  includePatterns: string[];
-  /**
-   * An array of patterns that match files that should not be uploaded.
-   */
-  excludePatterns: string[];
-}
-
-/**
- * Get an object that describes what assets to upload, if any.
- *
- * Uses the args (passed from the command line) if available,
- * falling back to those defined in the config.
- *
- */
-export function getAssetPaths(
-  config: Config,
-  assetDirectory = config.site?.bucket,
-  includePatterns = config.site?.include ?? [],
-  excludePatterns = config.site?.exclude ?? []
-): AssetPaths | undefined {
-  const baseDirectory = path.resolve(
-    path.dirname(config.configPath ?? "wrangler.toml")
-  );
-
-  return assetDirectory
-    ? {
-        baseDirectory,
-        assetDirectory,
-        includePatterns,
-        excludePatterns,
-      }
-    : undefined;
-}

package/src/tail/filters.ts

@@ -1,279 +0,0 @@
-/**
- * When tailing logs from a worker, oftentimes you don't want to see _every
- * single event_. That's where filters come in. We can send a set of filters
- * to the tail worker, and it will pre-filter any logs for us so that we
- * only recieve the ones we care about.
- */
-
-/**
- * These are the filters we accept in the CLI. They
- * were copied directly from wrangler 1 in order to
- * maintain compatability, so they aren't actually the exact
- * filters we need to send up to the tail worker. They generally map 1:1,
- * but often require some transformation or
- * renaming to match what it expects.
- *
- * For a full description of each filter, either check the
- * CLI description or see the documentation for `ApiFilter`.
- */
-export type TailCLIFilters = {
-  status?: ("ok" | "error" | "canceled")[];
-  header?: string;
-  method?: string[];
-  search?: string;
-  samplingRate?: number;
-  clientIp?: string[];
-};
-
-/**
- * These are the filters we send to the tail worker. We
- * actually send a list of filters (an array of objects),
- * so rather than having a single TailAPIFilters type,
- * each kind of filter gets its own type and we define
- * TailAPIFilter to be the union of those types.
- */
-export type TailAPIFilter =
-  | SamplingRateFilter
-  | OutcomeFilter
-  | MethodFilter
-  | HeaderFilter
-  | ClientIPFilter
-  | QueryFilter;
-
-/**
- * Filters logs based on a given sampling rate.
- * For example, a `sampling_rate` of 0.25 will let one-quarter of the
- * logs through.
- */
-type SamplingRateFilter = {
-  sampling_rate: number;
-};
-
-/**
- * Filters logs based on the outcome of the worker's event handler.
- */
-type OutcomeFilter = {
-  outcome: Outcome[];
-};
-
-/**
- * There are five possible outcomes we can get, three of which
- * (exception, exceededCpu, and unknown) are considered errors
- */
-export type Outcome =
-  | "ok"
-  | "canceled"
-  | "exception"
-  | "exceededCpu"
-  | "unknown";
-
-/**
- * Filters logs based on the HTTP method used for the request
- * that triggered the worker.
- */
-type MethodFilter = {
-  method: string[];
-};
-
-/**
- * Filters logs based on an HTTP header on the request that
- * triggered the worker.
- */
-type HeaderFilter = {
-  header: {
-    /**
-     * Filters on the header "key", e.g. "X-CLOUDFLARE-HEADER"
-     * or "X-CUSTOM-HEADER"
-     */
-    key: string;
-
-    /**
-     * Filters on the header "value", e.g. if this is set to
-     * "filter-for-me" and the "key" is "X-SHOULD-LOG", only
-     * events triggered by requests with the header
-     * "X-SHOULD-LOG:filter-for-me" will be logged.
-     */
-    query?: string;
-  };
-};
-
-/**
- * Filters on the IP address the request came from that triggered
- * the worker. A value of "self" will be replaced with the IP
- * address that is running `wrangler tail`
- */
-type ClientIPFilter = {
-  client_ip: string[];
-};
-
-/**
- * Filters logs by a query string. This means only logs that
- * contain the given string will be sent to wrangler, and any
- * that don't will be discarded by the tail worker.
- */
-type QueryFilter = {
-  query: string;
-};
-
-/**
- * The full message we send to the tail worker includes our
- * filters and a debug flag.
- */
-export type TailFilterMessage = {
-  filters: TailAPIFilter[];
-  debug: boolean;
-};
-
-/**
- * Translate the flags passed in via a CLI invokation of wrangler
- * into a message that we can send to the tail worker.
- *
- * @param cliFilters An object containing all the filters passed in from the CLI
- * @param debug Whether or not we should be in debug mode
- * @returns A filter message ready to be sent to the tail worker
- */
-export function translateCLICommandToFilterMessage(
-  cliFilters: TailCLIFilters,
-  debug: boolean
-): TailFilterMessage {
-  const apiFilters: TailAPIFilter[] = [];
-
-  if (cliFilters.samplingRate) {
-    apiFilters.push(parseSamplingRate(cliFilters.samplingRate));
-  }
-
-  if (cliFilters.status) {
-    apiFilters.push(parseOutcome(cliFilters.status));
-  }
-
-  if (cliFilters.method) {
-    apiFilters.push(parseMethod(cliFilters.method));
-  }
-
-  if (cliFilters.header) {
-    apiFilters.push(parseHeader(cliFilters.header));
-  }
-
-  if (cliFilters.clientIp) {
-    apiFilters.push(parseIP(cliFilters.clientIp));
-  }
-
-  if (cliFilters.search) {
-    apiFilters.push(parseQuery(cliFilters.search));
-  }
-
-  return {
-    filters: apiFilters,
-    debug,
-  };
-}
-
-/**
- * Parse the sampling rate passed in via command line
- *
- * @param sampling_rate the sampling rate passed in via CLI
- * @throws an Error if the rate doesn't make sense
- * @returns a SamplingRateFilter for use with the API
- */
-function parseSamplingRate(sampling_rate: number): SamplingRateFilter {
-  if (sampling_rate <= 0 || sampling_rate >= 1) {
-    throw new Error(
-      "A sampling rate must be between 0 and 1 in order to have any effect.\nFor example, a sampling rate of 0.25 means 25% of events will be logged."
-    );
-  }
-
-  return { sampling_rate };
-}
-
-/**
- * Translate from CLI "status"es to API "outcome"s, including
- * broadening "error" into "exception", "exceededCpu", and "unknown".
- *
- * @param statuses statuses passed in via CLI
- * @returns an OutcomeFilter for use with the API
- */
-function parseOutcome(
-  statuses: ("ok" | "error" | "canceled")[]
-): OutcomeFilter {
-  const outcomes = new Set<Outcome>();
-
-  for (const status of statuses) {
-    switch (status) {
-      case "ok":
-        outcomes.add("ok");
-        break;
-
-      case "canceled":
-        outcomes.add("canceled");
-        break;
-
-      case "error":
-        outcomes.add("exception");
-        outcomes.add("exceededCpu");
-        outcomes.add("unknown");
-        break;
-
-      default:
-        break;
-    }
-  }
-
-  return {
-    outcome: Array.from(outcomes),
-  };
-}
-
-/**
- * We just send silly methods through to the API anyway, since they don't
- * cause any harm.
- *
- * @param method an array of HTTP request methods passed in via CLI
- * @returns a MethodFilter for use with the API
- */
-function parseMethod(method: string[]): MethodFilter {
-  return { method };
-}
-
-/**
- * Header filters can contain either just a key ("X-HEADER-KEY") or both
- * a key and a value ("X-HEADER-KEY:some-value"). This function parses
- * a given string according to that pattern.
- *
- * @param header a header string, "X-HEADER-KEY" or "X-HEADER-KEY:some-value"
- * @returns a HeaderFilter for use with the API
- */
-function parseHeader(header: string): HeaderFilter {
-  const [headerKey, headerQuery] = header.split(":", 2);
-
-  return {
-    header: {
-      key: headerKey.trim(),
-      query: headerQuery?.trim(),
-    },
-  };
-}
-
-/**
- * A list of IPs can be passed in to filter for messages that come from
- * a worker triggered by a request originating from one of those IPs.
- * You can also pass in the string "self" to filter for the IP of the
- * machine running `wrangler tail`.
- *
- * @param client_ip an array of IP addresses to filter
- * @returns a ClientIPFilter for use with the API
- */
-function parseIP(client_ip: string[]): ClientIPFilter {
-  return { client_ip };
-}
-
-/**
- * Users can filter for logs that contain a "search" or a "query string".
- * For example, if `--search findme` is passed to then we will only
- * receive logs that contain the string "findme".
- *
- * @param query a query string to search for
- * @returns a QueryFilter for use with the API
- */
-function parseQuery(query: string): QueryFilter {
-  return { query };
-}