wrangler 0.0.0-e14e589 → 0.0.0-e2a26a0

package/src/sites.tsx

@@ -1,14 +1,36 @@
-import { readdir, readFile } from "node:fs/promises";
-import { createReadStream } from "node:fs";
-import cfetch from "./cfetch";
-import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
+import { readdir, readFile, stat } from "node:fs/promises";
+import * as path from "node:path";
+import ignore from "ignore";
+import { XXHash64 } from "xxhash-addon";
+import {
+  createNamespace,
+  listNamespaceKeys,
+  listNamespaces,
+  putBulkKeyValue,
+} from "./kv";
+import type { Config } from "./config";
 
-import * as path from "path";
-import crypto from "node:crypto";
+/** Paths to always ignore. */
+const ALWAYS_IGNORE = ["node_modules"];
+const HIDDEN_FILES_TO_INCLUDE = [
+  ".well-known", // See https://datatracker.ietf.org/doc/html/rfc8615
+];
 
 async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   const files = await readdir(dirPath, { withFileTypes: true });
   for (const file of files) {
+    // Skip files that we never want to process.
+    if (ALWAYS_IGNORE.some((p) => file.name === p)) {
+      continue;
+    }
+    // Skip hidden files (starting with .) except for some special ones
+    if (
+      file.name.startsWith(".") &&
+      !HIDDEN_FILES_TO_INCLUDE.some((p) => file.name === p)
+    ) {
+      continue;
+    }
+    // TODO: follow symlinks??
     if (file.isDirectory()) {
       yield* await getFilesInFolder(path.join(dirPath, file.name));
     } else {
@@ -17,27 +39,32 @@ async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
   }
 }
 
-async function hashFileContent(filePath: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const hash = crypto.createHash("sha1");
-    const rs = createReadStream(filePath);
-    rs.on("error", reject);
-    rs.on("data", (chunk) => hash.update(chunk));
-    rs.on("end", () => resolve(hash.digest("hex")));
-  });
+/**
+ * Create a hash key for the given content using the xxhash algorithm.
+ *
+ * Note we only return the first 10 characters, since we will also include the file name in the asset manifest key
+ * the most important thing here is to detect changes of a single file to invalidate the cache and
+ * it's impossible to serve two different files with the same name
+ */
+function hashFileContent(content: string): string {
+  const hasher = new XXHash64();
+  hasher.update(Buffer.from(content));
+  const hash = hasher.digest();
+  return hash.toString("hex").substring(0, 10);
 }
 
-async function hashFile(filePath: string): Promise<{
-  filePath: string;
-  hash: string;
-}> {
-  const extName = path.extname(filePath);
+/**
+ * Create a hashed asset key for the given asset.
+ *
+ * The key will change if the file path or content of the asset changes.
+ * The algorithm used here matches that of Wrangler 1.
+ */
+function hashAsset(filePath: string, content: string): string {
+  const extName = path.extname(filePath) || "";
   const baseName = path.basename(filePath, extName);
-  const hash = await hashFileContent(filePath);
-  return {
-    filePath: `${baseName}.${hash}${extName || ""}`,
-    hash,
-  };
+  const directory = path.dirname(filePath);
+  const hash = hashFileContent(content);
+  return urlSafe(path.join(directory, `${baseName}.${hash}${extName}`));
 }
 
 async function createKVNamespaceIfNotAlreadyExisting(
@@ -53,63 +80,161 @@ async function createKVNamespaceIfNotAlreadyExisting(
   }
 
   // else we make the namespace
-  // TODO: use an export from ./kv
-  const json = await cfetch<{ id: string }>(
-    `/accounts/${accountId}/storage/kv/namespaces`,
-    {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ title }),
-    }
-  );
+  const id = await createNamespace(accountId, title);
+  console.log(`🌀 Created namespace for Workers Site "${title}"`);
 
   return {
     created: true,
-    id: json.id,
+    id,
   };
 }
 
+/**
+ * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
+ * the worker given by `scriptName`.
+ *
+ * @param accountId the account to upload to.
+ * @param scriptName the name of the worker whose assets we are uploading.
+ * @param siteAssets an objects describing what assets to upload, or undefined if there are no assets to upload.
+ * @param preview if true then upload to a "preview" KV namespace.
+ * @param _env (not implemented).
+ * @returns a promise for an object mapping the relative paths of the assets to the key of that
+ * asset in the KV namespace.
+ */
 export async function syncAssets(
   accountId: string,
   scriptName: string,
-  dirPath: string,
+  siteAssets: AssetPaths | undefined,
   preview: boolean,
-  env?: string
-) {
-  const title = `__${scriptName}_sites_assets${preview ? "_preview" : ""}`;
+  env: string | undefined
+): Promise<{
+  manifest: { [filePath: string]: string } | undefined;
+  namespace: string | undefined;
+}> {
+  if (siteAssets === undefined) {
+    return { manifest: undefined, namespace: undefined };
+  }
+
+  const title = `__${scriptName}${env ? `-${env}` : ""}-workers_sites_assets${
+    preview ? "_preview" : ""
+  }`;
   const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
     title,
     accountId
   );
 
   // let's get all the keys in this namespace
-  const keys = new Set(
-    (await listNamespaceKeys(accountId, namespace)).map((x) => x.name)
-  );
+  const result = await listNamespaceKeys(accountId, namespace);
+  const keys = new Set(result.map((x) => x.name));
 
   const manifest = {};
-  const upload = [];
+  const upload: {
+    key: string;
+    value: string;
+    base64: boolean;
+  }[] = [];
+
+  const include = createPatternMatcher(siteAssets.includePatterns, false);
+  const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
   // TODO: this can be more efficient by parallelising
-  for await (const file of getFilesInFolder(dirPath)) {
-    // TODO: "exclude:" config
-    const { filePath } = await hashFile(file);
+  for await (const file of getFilesInFolder(siteAssets.baseDirectory)) {
+    if (!include(file)) {
+      continue;
+    }
+    if (exclude(file)) {
+      continue;
+    }
+
+    await validateAssetSize(file);
+    console.log(`reading ${file}...`);
+    const content = await readFile(file, "base64");
+
+    const assetKey = hashAsset(file, content);
+    validateAssetKey(assetKey);
+
     // now put each of the files into kv
-    if (!keys.has(filePath)) {
-      console.log(`uploading ${file}...`);
-      const content = await readFile(file, "base64");
-      if (content.length > 25 * 1024 * 1024) {
-        throw new Error(`File ${file} is too big, it should be under 25 mb.`);
-      }
+    if (!keys.has(assetKey)) {
+      console.log(`uploading as ${assetKey}...`);
       upload.push({
-        key: filePath,
+        key: assetKey,
         value: content,
         base64: true,
       });
+    } else {
+      console.log(`skipping - already uploaded`);
     }
-    manifest[path.relative(dirPath, file)] = filePath;
+    manifest[path.relative(siteAssets.baseDirectory, file)] = assetKey;
   }
   await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
   return { manifest, namespace };
 }
+
+function createPatternMatcher(
+  patterns: string[],
+  exclude: boolean
+): (filePath: string) => boolean {
+  if (patterns.length === 0) {
+    return (_filePath) => !exclude;
+  } else {
+    const ignorer = ignore().add(patterns);
+    return (filePath) => ignorer.test(filePath).ignored;
+  }
+}
+
+async function validateAssetSize(filePath: string) {
+  const { size } = await stat(filePath);
+  if (size > 25 * 1024 * 1024) {
+    throw new Error(
+      `File ${filePath} is too big, it should be under 25 MiB. See https://developers.cloudflare.com/workers/platform/limits#kv-limits`
+    );
+  }
+}
+
+function validateAssetKey(assetKey: string) {
+  if (assetKey.length > 512) {
+    throw new Error(
+      `The asset path key "${assetKey}" exceeds the maximum key size limit of 512. See https://developers.cloudflare.com/workers/platform/limits#kv-limits",`
+    );
+  }
+}
+
+/**
+ * Convert a filePath to be safe to use as a relative URL.
+ *
+ * Primarily this involves converting Windows backslashes to forward slashes.
+ */
+function urlSafe(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+/**
+ * Information about the assets that should be uploaded
+ */
+export interface AssetPaths {
+  baseDirectory: string;
+  includePatterns: string[];
+  excludePatterns: string[];
+}
+
+/**
+ * Get an object that describes what assets to upload, if any.
+ *
+ * Uses the args (passed from the command line) if available,
+ * falling back to those defined in the config.
+ *
+ * // TODO: Support for environments
+ */
+export function getAssetPaths(
+  config: Config,
+  baseDirectory = config.site?.bucket,
+  includePatterns = config.site?.include ?? [],
+  excludePatterns = config.site?.exclude ?? []
+): undefined | AssetPaths {
+  return baseDirectory
+    ? {
+        baseDirectory,
+        includePatterns,
+        excludePatterns,
+      }
+    : undefined;
+}

package/src/tail.tsx

@@ -1,6 +1,6 @@
 import WebSocket from "ws";
-import cfetch from "./cfetch";
 import { version as packageVersion } from "../package.json";
+import { fetchResult } from "./cfetch";
 
 export type TailApiResponse = {
   id: string;
@@ -27,13 +27,15 @@ async function createTailButDontConnect(
 ): Promise<TailApiResponse> {
   const createTailUrl = makeCreateTailUrl(accountId, workerName);
   /// https://api.cloudflare.com/#worker-tail-logs-start-tail
-  return await cfetch<TailApiResponse>(createTailUrl, { method: "POST" });
+  return await fetchResult<TailApiResponse>(createTailUrl, {
+    method: "POST",
+  });
 }
 
 export async function createTail(
   accountId: string,
   workerName: string,
-  filters: Filters
+  _filters: Filters
 ): Promise<{
   tail: WebSocket;
   expiration: Date;
@@ -48,7 +50,7 @@ export async function createTail(
 
   // deletes the tail
   async function deleteTail() {
-    await cfetch(deleteUrl, { method: "DELETE" });
+    await fetchResult(deleteUrl, { method: "DELETE" });
   }
 
   const tail = new WebSocket(websocketUrl, "trace-v1", {

package/src/user.tsx

@@ -205,25 +205,25 @@
    limitations under the License.
   */
 
-import React from "react";
-import { render } from "ink";
-import Table from "ink-table";
-import SelectInput from "ink-select-input";
-import fetch from "node-fetch";
+import assert from "node:assert";
 import { webcrypto as crypto } from "node:crypto";
-import { TextEncoder } from "node:util";
-import open from "open";
-import url from "node:url";
-import http from "node:http";
 import { readFile, writeFile, rm, mkdir } from "node:fs/promises";
+import http from "node:http";
+import os from "node:os";
 import path from "node:path";
 import process from "node:process";
-import os from "node:os";
+import url from "node:url";
+import { TextEncoder } from "node:util";
 import TOML from "@iarna/toml";
-import assert from "node:assert";
-import type { ParsedUrlQuery } from "node:querystring";
+import { render, Text } from "ink";
+import SelectInput from "ink-select-input";
+import Table from "ink-table";
+import open from "open";
+import React from "react";
+import { fetch } from "undici";
 import { CF_API_BASE_URL } from "./cfetch";
-import type { Response } from "node-fetch";
+import type { ParsedUrlQuery } from "node:querystring";
+import type { Response } from "undici";
 
 /**
  * An implementation of rfc6749#section-4.1 and rfc7636.
@@ -254,38 +254,37 @@ interface AccessToken {
   expiry: string;
 }
 
-type Scope =
-  | "account:read"
-  | "user:read"
-  | "workers:write"
-  | "workers_kv:write"
-  | "workers_routes:write"
-  | "workers_scripts:write"
-  | "workers_tail:read"
-  | "zone:read"
-  | "offline_access"; // this should be included by default
-
-const Scopes: Scope[] = [
-  "account:read",
-  "user:read",
-  "workers:write",
-  "workers_kv:write",
-  "workers_routes:write",
-  "workers_scripts:write",
-  "workers_tail:read",
-  "zone:read",
-];
-
-const ScopeDescriptions = [
-  "See your account info such as account details, analytics, and memberships.",
-  "See your user info such as name, email address, and account memberships.",
-  "See and change Cloudflare Workers data such as zones, KV storage, namespaces, scripts, and routes.",
-  "See and change Cloudflare Workers KV Storage data such as keys and namespaces.",
-  "See and change Cloudflare Workers data such as filters and routes.",
-  "See and change Cloudflare Workers scripts, durable objects, subdomains, triggers, and tail data.",
-  "See Cloudflare Workers tail and script data.",
-  "Grants read level access to account zone.",
-];
+const Scopes = {
+  "account:read":
+    "See your account info such as account details, analytics, and memberships.",
+  "user:read":
+    "See your user info such as name, email address, and account memberships.",
+  "workers:write":
+    "See and change Cloudflare Workers data such as zones, KV storage, namespaces, scripts, and routes.",
+  "workers_kv:write":
+    "See and change Cloudflare Workers KV Storage data such as keys and namespaces.",
+  "workers_routes:write":
+    "See and change Cloudflare Workers data such as filters and routes.",
+  "workers_scripts:write":
+    "See and change Cloudflare Workers scripts, durable objects, subdomains, triggers, and tail data.",
+  "workers_tail:read": "See Cloudflare Workers tail and script data.",
+  "zone:read": "Grants read level access to account zone.",
+} as const;
+
+/**
+ * The possible keys for a Scope.
+ *
+ * "offline_access" is automatically included.
+ */
+type Scope = keyof typeof Scopes | "offline_access";
+
+const ScopeKeys = Object.keys(Scopes) as Scope[];
+
+export function validateScopeKeys(
+  scopes: string[]
+): scopes is typeof ScopeKeys {
+  return scopes.every((scope) => Scopes[scope]);
+}
 
 const CLIENT_ID = "54d11594-84e4-41aa-b438-e81b8fa78ee7";
 const AUTH_URL = "https://dash.cloudflare.com/oauth2/auth";
@@ -298,9 +297,9 @@ let initialised = false;
 
 // we do this because we have some async stuff
 // TODO: this should just happen in the top level
-// abd we should fiure out how to do top level await
+// and we should figure out how to do top level await
 export async function initialise(): Promise<void> {
-  // get refreshtoken/accesstoken from fs if exists
+  // get refreshToken/accessToken from fs if exists
   try {
     // if CF_API_TOKEN available, use that
     if (process.env.CF_API_TOKEN) {
@@ -344,11 +343,10 @@ function throwIfNotInitialised() {
   }
 }
 
-export function getAPIToken(): string {
+export function getAPIToken(): string | undefined {
   if (process.env.CF_API_TOKEN) {
     return process.env.CF_API_TOKEN;
   }
-
   throwIfNotInitialised();
   return LocalState.accessToken?.value;
 }
@@ -548,7 +546,7 @@ function isReturningFromAuthServer(query: ParsedUrlQuery): boolean {
   return true;
 }
 
-export async function getAuthURL(scopes?: string[]): Promise<string> {
+export async function getAuthURL(scopes = ScopeKeys): Promise<string> {
   const { codeChallenge, codeVerifier } = await generatePKCECodes();
   const stateQueryParam = generateRandomState(RECOMMENDED_STATE_LENGTH);
 
@@ -558,16 +556,12 @@ export async function getAuthURL(scopes?: string[]): Promise<string> {
     stateQueryParam,
   });
 
-  // TODO: verify that the scopes passed are legit
-
   return (
     AUTH_URL +
     `?response_type=code&` +
     `client_id=${encodeURIComponent(CLIENT_ID)}&` +
     `redirect_uri=${encodeURIComponent(CALLBACK_URL)}&` +
-    `scope=${encodeURIComponent(
-      (scopes || Scopes).concat("offline_access").join(" ")
-    )}&` +
+    `scope=${encodeURIComponent(scopes.concat("offline_access").join(" "))}&` +
     `state=${stateQueryParam}&` +
     `code_challenge=${encodeURIComponent(codeChallenge)}&` +
     `code_challenge_method=S256`
@@ -578,19 +572,16 @@ export async function getAuthURL(scopes?: string[]): Promise<string> {
  * Refresh an access token from the remote service.
  */
 async function exchangeRefreshTokenForAccessToken(): Promise<AccessContext> {
-  const { refreshToken } = LocalState;
-
-  if (!refreshToken) {
+  if (!LocalState.refreshToken) {
     console.warn("No refresh token is present.");
   }
 
-  const url = TOKEN_URL;
   const body =
     `grant_type=refresh_token&` +
-    `refresh_token=${refreshToken?.value}&` +
+    `refresh_token=${LocalState.refreshToken?.value}&` +
     `client_id=${CLIENT_ID}`;
 
-  const response = await fetch(url, {
+  const response = await fetch(TOKEN_URL, {
     method: "POST",
     body,
     headers: {
@@ -617,10 +608,9 @@ async function exchangeRefreshTokenForAccessToken(): Promise<AccessContext> {
       LocalState.accessToken = accessToken;
 
       if (refresh_token) {
-        const refreshToken: RefreshToken = {
+        LocalState.refreshToken = {
           value: refresh_token,
         };
-        LocalState.refreshToken = refreshToken;
       }
 
       if (scope) {
@@ -666,7 +656,6 @@ async function exchangeAuthCodeForAccessToken(): Promise<AccessContext> {
     console.warn("No authorization grant code is being passed.");
   }
 
-  const url = TOKEN_URL;
   const body =
     `grant_type=authorization_code&` +
     `code=${encodeURIComponent(authorizationCode || "")}&` +
@@ -674,7 +663,7 @@ async function exchangeAuthCodeForAccessToken(): Promise<AccessContext> {
     `client_id=${encodeURIComponent(CLIENT_ID)}&` +
     `code_verifier=${codeVerifier}`;
 
-  const response = await fetch(url, {
+  const response = await fetch(TOKEN_URL, {
     method: "POST",
     body,
     headers: {
@@ -709,10 +698,9 @@ async function exchangeAuthCodeForAccessToken(): Promise<AccessContext> {
   LocalState.accessToken = accessToken;
 
   if (refresh_token) {
-    const refreshToken: RefreshToken = {
+    LocalState.refreshToken = {
       value: refresh_token,
     };
-    LocalState.refreshToken = refreshToken;
   }
 
   if (scope) {
@@ -798,7 +786,7 @@ expiration_time = "${tokenData.token?.expiry}"
 }
 
 type LoginProps = {
-  scopes?: string[];
+  scopes?: Scope[];
 };
 
 export async function loginOrRefreshIfRequired(): Promise<boolean> {
@@ -816,7 +804,7 @@ export async function loginOrRefreshIfRequired(): Promise<boolean> {
 
 export async function login(props?: LoginProps): Promise<boolean> {
   const urlToOpen = await getAuthURL(props?.scopes);
-  open(urlToOpen);
+  await open(urlToOpen);
   // TODO: log url only if on system where it's unreliable/unavailable
   // console.log(`💁 Opened ${urlToOpen}`);
   let server;
@@ -921,15 +909,14 @@ export async function refreshToken(): Promise<boolean> {
 
 export async function logout(): Promise<void> {
   throwIfNotInitialised();
-  const { refreshToken } = LocalState;
-  if (!refreshToken) {
+  if (!LocalState.refreshToken) {
     console.log("Not logged in, exiting...");
     return;
   }
   const body =
     `client_id=${encodeURIComponent(CLIENT_ID)}&` +
     `token_type_hint=refresh_token&` +
-    `token=${encodeURIComponent(refreshToken?.value || "")}`;
+    `token=${encodeURIComponent(LocalState.refreshToken?.value || "")}`;
 
   const response = await fetch(REVOKE_URL, {
     method: "POST",
@@ -952,9 +939,9 @@ export async function logout(): Promise<void> {
 export function listScopes(): void {
   throwIfNotInitialised();
   console.log("💁 Available scopes:");
-  const data = Scopes.map((scope, index) => ({
+  const data = ScopeKeys.map((scope) => ({
     Scope: scope,
-    Description: ScopeDescriptions[index],
+    Description: Scopes[scope],
   }));
   render(<Table data={data} />);
   // TODO: maybe a good idea to show usage here
@@ -978,14 +965,14 @@ export async function getAccountId() {
     });
   } catch (err) {
     // probably offline
+    return;
   }
-  if (!response) return;
-  let accountId: string;
-  // @ts-expect-error need to type this response
-  const responseJSON: {
+
+  let accountId: string | undefined;
+  const responseJSON = (await response.json()) as {
     success: boolean;
     result: { id: string; account: { id: string; name: string } }[];
-  } = await response.json();
+  };
 
   if (responseJSON.success === true) {
     if (responseJSON.result.length === 1) {
@@ -1017,13 +1004,16 @@ export function ChooseAccount(props: {
   onSelect: (item) => void;
 }) {
   return (
-    <SelectInput
-      items={props.accounts.map((item) => ({
-        key: item.id,
-        label: item.name,
-        value: item,
-      }))}
-      onSelect={props.onSelect}
-    />
+    <>
+      <Text bold>Select an account from below:</Text>
+      <SelectInput
+        items={props.accounts.map((item) => ({
+          key: item.id,
+          label: item.name,
+          value: item,
+        }))}
+        onSelect={props.onSelect}
+      />
+    </>
   );
 }