wrangler 0.0.0-bfbb0be → 0.0.0-bfbe49d0

package/src/pages/upload.tsx

@@ -1,363 +0,0 @@
-import { readdir, readFile, stat } from "node:fs/promises";
-import { basename, extname, join, relative, resolve, sep } from "node:path";
-import { hash as blake3hash } from "blake3-wasm";
-import { render, Text } from "ink";
-import Spinner from "ink-spinner";
-import { getType } from "mime";
-import PQueue from "p-queue";
-import prettyBytes from "pretty-bytes";
-import React from "react";
-import { fetchResult } from "../cfetch";
-import { FatalError } from "../errors";
-import { logger } from "../logger";
-import {
-  BULK_UPLOAD_CONCURRENCY,
-  MAX_BUCKET_FILE_COUNT,
-  MAX_BUCKET_SIZE,
-  MAX_UPLOAD_ATTEMPTS,
-} from "./constants";
-import { pagesBetaWarning } from "./utils";
-import type { UploadPayloadFile } from "./types";
-import type { ArgumentsCamelCase, Argv } from "yargs";
-
-type UploadArgs = {
-  directory: string;
-};
-
-export function Options(yargs: Argv): Argv<UploadArgs> {
-  return yargs
-    .positional("directory", {
-      type: "string",
-      demandOption: true,
-      description: "The directory of static files to upload",
-    })
-    .epilogue(pagesBetaWarning);
-}
-
-export const Handler = async ({
-  directory,
-}: ArgumentsCamelCase<UploadArgs>) => {
-  if (!directory) {
-    throw new FatalError("Must specify a directory.", 1);
-  }
-
-  if (!process.env.CF_PAGES_UPLOAD_JWT) {
-    throw new FatalError("No JWT given.", 1);
-  }
-
-  await upload({
-    directory,
-    jwt: process.env.CF_PAGES_UPLOAD_JWT,
-  });
-
-  logger.log(`✨ Upload complete!`);
-};
-
-export const upload = async (
-  args:
-    | {
-        directory: string;
-        jwt: string;
-      }
-    | { directory: string; accountId: string; projectName: string }
-) => {
-  async function fetchJwt(): Promise<string> {
-    if ("jwt" in args) {
-      return args.jwt;
-    } else {
-      return (
-        await fetchResult<{ jwt: string }>(
-          `/accounts/${args.accountId}/pages/projects/${args.projectName}/upload-token`
-        )
-      ).jwt;
-    }
-  }
-
-  type FileContainer = {
-    content: string;
-    contentType: string;
-    sizeInBytes: number;
-    hash: string;
-  };
-
-  const IGNORE_LIST = [
-    "_worker.js",
-    "_redirects",
-    "_headers",
-    ".DS_Store",
-    "node_modules",
-    ".git",
-  ];
-
-  const directory = resolve(args.directory);
-
-  const walk = async (
-    dir: string,
-    fileMap: Map<string, FileContainer> = new Map(),
-    startingDir: string = dir
-  ) => {
-    const files = await readdir(dir);
-
-    await Promise.all(
-      files.map(async (file) => {
-        const filepath = join(dir, file);
-        const filestat = await stat(filepath);
-
-        if (IGNORE_LIST.includes(file)) {
-          return;
-        }
-
-        if (filestat.isSymbolicLink()) {
-          return;
-        }
-
-        if (filestat.isDirectory()) {
-          fileMap = await walk(filepath, fileMap, startingDir);
-        } else {
-          const name = relative(startingDir, filepath).split(sep).join("/");
-
-          // TODO: Move this to later so we don't hold as much in memory
-          const fileContent = await readFile(filepath);
-
-          const base64Content = fileContent.toString("base64");
-          const extension = extname(basename(name)).substring(1);
-
-          if (filestat.size > 25 * 1024 * 1024) {
-            throw new FatalError(
-              `Error: Pages only supports files up to ${prettyBytes(
-                25 * 1024 * 1024
-              )} in size\n${name} is ${prettyBytes(filestat.size)} in size`,
-              1
-            );
-          }
-
-          fileMap.set(name, {
-            content: base64Content,
-            contentType: getType(name) || "application/octet-stream",
-            sizeInBytes: filestat.size,
-            hash: blake3hash(base64Content + extension)
-              .toString("hex")
-              .slice(0, 32),
-          });
-        }
-      })
-    );
-
-    return fileMap;
-  };
-
-  const fileMap = await walk(directory);
-
-  if (fileMap.size > 20000) {
-    throw new FatalError(
-      `Error: Pages only supports up to 20,000 files in a deployment. Ensure you have specified your build output directory correctly.`,
-      1
-    );
-  }
-
-  const files = [...fileMap.values()];
-
-  let jwt = await fetchJwt();
-
-  const start = Date.now();
-
-  const missingHashes = await fetchResult<string[]>(
-    `/pages/assets/check-missing`,
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${jwt}`,
-      },
-      body: JSON.stringify({
-        hashes: files.map(({ hash }) => hash),
-      }),
-    }
-  );
-
-  const sortedFiles = files
-    .filter((file) => missingHashes.includes(file.hash))
-    .sort((a, b) => b.sizeInBytes - a.sizeInBytes);
-
-  // Start with a few buckets so small projects still get
-  // the benefit of multiple upload streams
-  const buckets: {
-    files: FileContainer[];
-    remainingSize: number;
-  }[] = new Array(BULK_UPLOAD_CONCURRENCY).fill(null).map(() => ({
-    files: [],
-    remainingSize: MAX_BUCKET_SIZE,
-  }));
-
-  let bucketOffset = 0;
-  for (const file of sortedFiles) {
-    let inserted = false;
-
-    for (let i = 0; i < buckets.length; i++) {
-      // Start at a different bucket for each new file
-      const bucket = buckets[(i + bucketOffset) % buckets.length];
-      if (
-        bucket.remainingSize >= file.sizeInBytes &&
-        bucket.files.length < MAX_BUCKET_FILE_COUNT
-      ) {
-        bucket.files.push(file);
-        bucket.remainingSize -= file.sizeInBytes;
-        inserted = true;
-        break;
-      }
-    }
-
-    if (!inserted) {
-      buckets.push({
-        files: [file],
-        remainingSize: MAX_BUCKET_SIZE - file.sizeInBytes,
-      });
-    }
-    bucketOffset++;
-  }
-
-  let counter = fileMap.size - sortedFiles.length;
-  const { rerender, unmount } = render(
-    <Progress done={counter} total={fileMap.size} />
-  );
-
-  const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
-
-  for (const bucket of buckets) {
-    // Don't upload empty buckets (can happen for tiny projects)
-    if (bucket.files.length === 0) continue;
-
-    const payload: UploadPayloadFile[] = bucket.files.map((file) => ({
-      key: file.hash,
-      value: file.content,
-      metadata: {
-        contentType: file.contentType,
-      },
-      base64: true,
-    }));
-
-    let attempts = 0;
-    const doUpload = async (): Promise<void> => {
-      try {
-        return await fetchResult(`/pages/assets/upload`, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            Authorization: `Bearer ${jwt}`,
-          },
-          body: JSON.stringify(payload),
-        });
-      } catch (e) {
-        if (attempts < MAX_UPLOAD_ATTEMPTS) {
-          // Linear backoff, 0 second first time, then 1 second etc.
-          await new Promise((resolvePromise) =>
-            setTimeout(resolvePromise, attempts++ * 1000)
-          );
-
-          if ((e as { code: number }).code === 8000013) {
-            // Looks like the JWT expired, fetch another one
-            jwt = await fetchJwt();
-          }
-          return doUpload();
-        } else {
-          throw e;
-        }
-      }
-    };
-
-    queue.add(() =>
-      doUpload().then(
-        () => {
-          counter += bucket.files.length;
-          rerender(<Progress done={counter} total={fileMap.size} />);
-        },
-        (error) => {
-          return Promise.reject(
-            new FatalError(
-              "Failed to upload files. Please try again.",
-              error.code || 1
-            )
-          );
-        }
-      )
-    );
-  }
-
-  await queue.onIdle();
-
-  unmount();
-
-  const uploadMs = Date.now() - start;
-
-  const skipped = fileMap.size - missingHashes.length;
-  const skippedMessage = skipped > 0 ? `(${skipped} already uploaded) ` : "";
-
-  logger.log(
-    `✨ Success! Uploaded ${
-      sortedFiles.length
-    } files ${skippedMessage}${formatTime(uploadMs)}\n`
-  );
-
-  const doUpsertHashes = async (): Promise<void> => {
-    try {
-      return await fetchResult(`/pages/assets/upsert-hashes`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${jwt}`,
-        },
-        body: JSON.stringify({
-          hashes: files.map(({ hash }) => hash),
-        }),
-      });
-    } catch (e) {
-      await new Promise((resolvePromise) => setTimeout(resolvePromise, 1000));
-
-      if ((e as { code: number }).code === 8000013) {
-        // Looks like the JWT expired, fetch another one
-        jwt = await fetchJwt();
-      }
-
-      return await fetchResult(`/pages/assets/upsert-hashes`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${jwt}`,
-        },
-        body: JSON.stringify({
-          hashes: files.map(({ hash }) => hash),
-        }),
-      });
-    }
-  };
-
-  try {
-    await doUpsertHashes();
-  } catch {
-    logger.warn(
-      "Failed to update file hashes. Every upload appeared to succeed for this deployment, but you might need to re-upload for future deployments. This shouldn't have any impact other than slowing the upload speed of your next deployment."
-    );
-  }
-
-  return Object.fromEntries(
-    [...fileMap.entries()].map(([fileName, file]) => [
-      `/${fileName}`,
-      file.hash,
-    ])
-  );
-};
-
-function formatTime(duration: number) {
-  return `(${(duration / 1000).toFixed(2)} sec)`;
-}
-
-function Progress({ done, total }: { done: number; total: number }) {
-  return (
-    <>
-      <Text>
-        <Spinner type="earth" />
-        {` Uploading... (${done}/${total})\n`}
-      </Text>
-    </>
-  );
-}

package/src/pages/utils.ts

@@ -1,12 +0,0 @@
-import type { BuildResult } from "esbuild";
-
-export const RUNNING_BUILDERS: BuildResult[] = [];
-
-export const CLEANUP_CALLBACKS: (() => void)[] = [];
-export const CLEANUP = () => {
-  CLEANUP_CALLBACKS.forEach((callback) => callback());
-  RUNNING_BUILDERS.forEach((builder) => builder.stop?.());
-};
-
-export const pagesBetaWarning =
-  "🚧 'wrangler pages <command>' is a beta command. Please report any issues to https://github.com/cloudflare/wrangler2/issues/new/choose";

package/src/parse.ts

@@ -1,222 +0,0 @@
-import * as fs from "node:fs";
-import { resolve } from "node:path";
-import TOML from "@iarna/toml";
-import { formatMessagesSync } from "esbuild";
-import { logger } from "./logger";
-
-export type Message = {
-  text: string;
-  location?: Location;
-  notes?: Message[];
-  kind?: "warning" | "error";
-};
-
-export type Location = File & {
-  line: number;
-  column: number;
-  length?: number;
-  lineText?: string;
-  suggestion?: string;
-};
-
-export type File = {
-  file?: string;
-  fileText?: string;
-};
-
-/**
- * Formats a `Message` using esbuild's pretty-printing algorithm.
- */
-export function formatMessage(
-  { text, notes, location, kind = "error" }: Message,
-  color = true
-): string {
-  const input = { text, notes, location };
-  delete input.location?.fileText;
-  for (const note of notes ?? []) {
-    delete note.location?.fileText;
-  }
-  const lines = formatMessagesSync([input], {
-    color,
-    kind: kind,
-    terminalWidth: logger.columns,
-  });
-  return lines.join("\n");
-}
-
-/**
- * An error that's thrown when something fails to parse.
- */
-export class ParseError extends Error implements Message {
-  readonly text: string;
-  readonly notes: Message[];
-  readonly location?: Location;
-  readonly kind: "warning" | "error";
-
-  constructor({ text, notes, location, kind }: Message) {
-    super(text);
-    this.name = this.constructor.name;
-    this.text = text;
-    this.notes = notes ?? [];
-    this.location = location;
-    this.kind = kind ?? "error";
-  }
-}
-
-const TOML_ERROR_NAME = "TomlError";
-const TOML_ERROR_SUFFIX = " at row ";
-
-type TomlError = Error & {
-  line: number;
-  col: number;
-};
-
-/**
- * A wrapper around `TOML.parse` that throws a `ParseError`.
- */
-export function parseTOML(input: string, file?: string): TOML.JsonMap | never {
-  try {
-    // Normalize CRLF to LF to avoid hitting https://github.com/iarna/iarna-toml/issues/33.
-    const normalizedInput = input.replace(/\r\n/g, "\n");
-    return TOML.parse(normalizedInput);
-  } catch (err) {
-    const { name, message, line, col } = err as TomlError;
-    if (name !== TOML_ERROR_NAME) {
-      throw err;
-    }
-    const text = message.substring(0, message.lastIndexOf(TOML_ERROR_SUFFIX));
-    const lineText = input.split("\n")[line];
-    const location = {
-      lineText,
-      line: line + 1,
-      column: col - 1,
-      file,
-      fileText: input,
-    };
-    throw new ParseError({ text, location });
-  }
-}
-
-const JSON_ERROR_SUFFIX = " in JSON at position ";
-
-/**
- * A minimal type describing a package.json file.
- */
-export type PackageJSON = {
-  devDependencies?: Record<string, unknown>;
-  dependencies?: Record<string, unknown>;
-  scripts?: Record<string, unknown>;
-};
-
-/**
- * A typed version of `parseJSON()`.
- */
-export function parsePackageJSON<T extends PackageJSON = PackageJSON>(
-  input: string,
-  file?: string
-): T {
-  return parseJSON<T>(input, file);
-}
-
-/**
- * A wrapper around `JSON.parse` that throws a `ParseError`.
- */
-export function parseJSON<T>(input: string, file?: string): T {
-  try {
-    return JSON.parse(input);
-  } catch (err) {
-    const { message } = err as Error;
-    const index = message.lastIndexOf(JSON_ERROR_SUFFIX);
-    if (index < 0) {
-      throw err;
-    }
-    const text = message.substring(0, index);
-    const position = parseInt(
-      message.substring(index + JSON_ERROR_SUFFIX.length)
-    );
-    const location = indexLocation({ file, fileText: input }, position);
-    throw new ParseError({ text, location });
-  }
-}
-
-/**
- * Reads a file into a node Buffer.
- */
-export function readFileSyncToBuffer(file: string): Buffer {
-  try {
-    return fs.readFileSync(file);
-  } catch (err) {
-    const { message } = err as Error;
-    throw new ParseError({
-      text: `Could not read file: ${file}`,
-      notes: [
-        {
-          text: message.replace(file, resolve(file)),
-        },
-      ],
-    });
-  }
-}
-
-/**
- * Reads a file and parses it based on its type.
- */
-export function readFileSync(file: string): string {
-  try {
-    return fs.readFileSync(file, { encoding: "utf-8" });
-  } catch (err) {
-    const { message } = err as Error;
-    throw new ParseError({
-      text: `Could not read file: ${file}`,
-      notes: [
-        {
-          text: message.replace(file, resolve(file)),
-        },
-      ],
-    });
-  }
-}
-
-/**
- * Calculates the line and column location from an index.
- */
-export function indexLocation(file: File, index: number): Location {
-  let lineText,
-    line = 0,
-    column = 0,
-    cursor = 0;
-  const { fileText = "" } = file;
-  for (const row of fileText.split("\n")) {
-    line++;
-    cursor += row.length + 1;
-    if (cursor >= index) {
-      lineText = row;
-      column = row.length - (cursor - index);
-      break;
-    }
-  }
-  return { lineText, line, column, ...file };
-}
-
-/**
- * Guesses the line and column location of a search query.
- */
-export function searchLocation(file: File, query: unknown): Location {
-  let lineText,
-    length,
-    line = 0,
-    column = 0;
-  const queryText = String(query);
-  const { fileText = "" } = file;
-  for (const content of fileText.split("\n")) {
-    line++;
-    const index = content.indexOf(queryText);
-    if (index >= 0) {
-      lineText = content;
-      column = index;
-      length = queryText.length;
-      break;
-    }
-  }
-  return { lineText, line, column, length, ...file };
-}

package/src/paths.ts

@@ -1,26 +0,0 @@
-import { assert } from "console";
-
-type DiscriminatedPath<Discriminator extends string> = string & {
-  _discriminator: Discriminator;
-};
-
-/**
- * A branded string that expects to be URL compatible.
- *
- * Require this type when you want callers to ensure that they have converted file-path strings into URL-safe paths.
- */
-export type UrlPath = DiscriminatedPath<"UrlPath">;
-
-/**
- * Convert a file-path string to a URL-path string.
- *
- * Use this helper to convert a `string` to a `UrlPath` when it is not clear whether the string needs normalizing.
- * Replaces all back-slashes with forward-slashes, and throws an error if the path contains a drive letter (e.g. `C:`).
- */
-export function toUrlPath(path: string): UrlPath {
-  assert(
-    !/^[a-z]:/i.test(path),
-    "Tried to convert a Windows file path with a drive to a URL path."
-  );
-  return path.replace(/\\/g, "/") as UrlPath;
-}