wrangler 0.0.0-e2f9bb2 → 0.0.0-e3278fa

package/src/sites.tsx

@@ -1,13 +1,14 @@
+import assert from "node:assert";
 import { readdir, readFile, stat } from "node:fs/promises";
 import * as path from "node:path";
 import ignore from "ignore";
 import xxhash from "xxhash-wasm";
 import {
-  createNamespace,
-  listNamespaceKeys,
-  listNamespaces,
-  putBulkKeyValue,
-  deleteBulkKeyValue,
+  createKVNamespace,
+  listKVNamespaceKeys,
+  listKVNamespaces,
+  putKVBulkKeyValue,
+  deleteKVBulkKeyValue,
 } from "./kv";
 import { logger } from "./logger";
 import type { Config } from "./config";
@@ -75,14 +76,14 @@ async function createKVNamespaceIfNotAlreadyExisting(
 ) {
   // check if it already exists
   // TODO: this is super inefficient, should be made better
-  const namespaces = await listNamespaces(accountId);
+  const namespaces = await listKVNamespaces(accountId);
   const found = namespaces.find((x) => x.title === title);
   if (found) {
     return { created: false, id: found.id };
   }
 
   // else we make the namespace
-  const id = await createNamespace(accountId, title);
+  const id = await createKVNamespace(accountId, title);
   logger.log(`🌀 Created namespace for Workers Site "${title}"`);
 
   return {
@@ -103,7 +104,7 @@ async function createKVNamespaceIfNotAlreadyExisting(
  * asset in the KV namespace.
  */
 export async function syncAssets(
-  accountId: string,
+  accountId: string | undefined,
   scriptName: string,
   siteAssets: AssetPaths | undefined,
   preview: boolean,
@@ -120,6 +121,7 @@ export async function syncAssets(
     logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
     return { manifest: undefined, namespace: undefined };
   }
+  assert(accountId, "Missing accountId");
 
   const title = `__${scriptName}-workers_sites_assets${
     preview ? "_preview" : ""
@@ -131,11 +133,17 @@ export async function syncAssets(
   );
 
   // let's get all the keys in this namespace
-  const namespaceKeysResponse = await listNamespaceKeys(accountId, namespace);
+  const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
   const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
 
   const manifest: Record<string, string> = {};
-  const toUpload: KeyValue[] = [];
+
+  // A batch of uploads where each bucket has to be less than 100mb
+  const uploadBuckets: KeyValue[][] = [];
+  // The "live" bucket that we'll keep filling until it's just below 100mb
+  let uploadBucket: KeyValue[] = [];
+  // A size counter for the live bucket
+  let uploadBucketSize = 0;
 
   const include = createPatternMatcher(siteAssets.includePatterns, false);
   const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
@@ -146,7 +154,7 @@ export async function syncAssets(
     siteAssets.assetDirectory
   );
   for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
-    const assetFile = path.relative(siteAssets.baseDirectory, absAssetFile);
+    const assetFile = path.relative(assetDirectory, absAssetFile);
     if (!include(assetFile)) {
       continue;
     }
@@ -154,17 +162,32 @@ export async function syncAssets(
       continue;
     }
 
-    await validateAssetSize(absAssetFile, assetFile);
     logger.log(`Reading ${assetFile}...`);
     const content = await readFile(absAssetFile, "base64");
-
+    await validateAssetSize(absAssetFile, assetFile);
+    // while KV accepts files that are 25 MiB **before** b64 encoding
+    // the overall bucket size must be below 100 MiB **after** b64 encoding
+    const assetSize = Buffer.from(content).length;
     const assetKey = hashAsset(hasher, assetFile, content);
     validateAssetKey(assetKey);
 
     // now put each of the files into kv
     if (!namespaceKeys.has(assetKey)) {
       logger.log(`Uploading as ${assetKey}...`);
-      toUpload.push({
+
+      // Check if adding this asset to the bucket would
+      // push it over the 100 MiB limit KV bulk API limit
+      if (uploadBucketSize + assetSize > 100 * 1024 * 1024) {
+        // If so, move the current bucket into the batch,
+        // and reset the counter/bucket
+        uploadBuckets.push(uploadBucket);
+        uploadBucketSize = 0;
+        uploadBucket = [];
+      }
+
+      // Update the bucket and the size counter
+      uploadBucketSize += assetSize;
+      uploadBucket.push({
         key: assetKey,
         value: content,
         base64: true,
@@ -173,27 +196,31 @@ export async function syncAssets(
       logger.log(`Skipping - already uploaded.`);
     }
 
-    // remove the key from the set so we know what we've already uploaded
+    // Remove the key from the set so we know what we've already uploaded
     namespaceKeys.delete(assetKey);
-    manifest[path.relative(siteAssets.assetDirectory, absAssetFile)] = assetKey;
+
+    // Prevent different manifest keys on windows
+    const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
+    manifest[manifestKey] = assetKey;
   }
 
+  // Add the last (potentially only) bucket to the batch
+  uploadBuckets.push(uploadBucket);
+
   // keys now contains all the files we're deleting
   for (const key of namespaceKeys) {
     logger.log(`Deleting ${key} from the asset store...`);
   }
 
-  await Promise.all([
-    // upload all the new assets
-    putBulkKeyValue(accountId, namespace, toUpload, () => {}),
-    // delete all the unused assets
-    deleteBulkKeyValue(
-      accountId,
-      namespace,
-      Array.from(namespaceKeys),
-      () => {}
-    ),
-  ]);
+  // upload each bucket in parallel
+  const bucketsToPut = [];
+  for (const bucket of uploadBuckets) {
+    bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
+  }
+  await Promise.all(bucketsToPut);
+
+  // then delete all the assets that aren't used anymore
+  await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
 
   logger.log("↗️  Done syncing assets");
 
@@ -212,10 +239,16 @@ function createPatternMatcher(
   }
 }
 
+/**
+ * validate that the passed-in file is below 25 MiB
+ * **PRIOR** to base64 encoding. 25 MiB is a KV limit
+ * @param absFilePath
+ * @param relativeFilePath
+ */
 async function validateAssetSize(
   absFilePath: string,
   relativeFilePath: string
-) {
+): Promise<void> {
   const { size } = await stat(absFilePath);
   if (size > 25 * 1024 * 1024) {
     throw new Error(

package/src/user.tsx

@@ -214,6 +214,7 @@ import path from "node:path";
 import url from "node:url";
 import { TextEncoder } from "node:util";
 import TOML from "@iarna/toml";
+import { HostURL } from "@webcontainer/env";
 import { render, Text } from "ink";
 import SelectInput from "ink-select-input";
 import Table from "ink-table";
@@ -232,7 +233,7 @@ import type { Response } from "undici";
 /**
  * Try to read the API token from the environment.
  */
-const getCloudflareAPITokenFromEnv = getEnvironmentVariableFactory({
+export const getCloudflareAPITokenFromEnv = getEnvironmentVariableFactory({
   variableName: "CLOUDFLARE_API_TOKEN",
   deprecatedName: "CF_API_TOKEN",
 });
@@ -339,9 +340,18 @@ export function validateScopeKeys(
 const CLIENT_ID = "54d11594-84e4-41aa-b438-e81b8fa78ee7";
 const AUTH_URL = "https://dash.cloudflare.com/oauth2/auth";
 const TOKEN_URL = "https://dash.cloudflare.com/oauth2/token";
-const CALLBACK_URL = "http://localhost:8976/oauth/callback";
 const REVOKE_URL = "https://dash.cloudflare.com/oauth2/revoke";
 
+/**
+ * To allow OAuth callbacks in environments such as WebContainer we need to
+ * create a host URL which only resolves `localhost` to a WebContainer
+ * hostname if the process is running in a WebContainer. On local this will
+ * be a no-op and it leaves the URL unmodified.
+ *
+ * @see https://www.npmjs.com/package/@webcontainer/env
+ */
+const CALLBACK_URL = HostURL.parse("http://localhost:8976/oauth/callback").href;
+
 let LocalState: State = getAuthTokens();
 
 /**
@@ -1191,3 +1201,14 @@ export async function requireAuth(config: {
 
   return accountId;
 }
+
+/**
+ * Throw an error if there is no API token available.
+ */
+export function requireApiToken(): string {
+  const authToken = getAPIToken();
+  if (!authToken) {
+    throw new Error("No API token found.");
+  }
+  return authToken;
+}

package/src/whoami.tsx

@@ -3,7 +3,7 @@ import Table from "ink-table";
 import React from "react";
 import { fetchListResult, fetchResult } from "./cfetch";
 import { logger } from "./logger";
-import { getAPIToken } from "./user";
+import { getAPIToken, getCloudflareAPITokenFromEnv } from "./user";
 
 export async function whoami() {
   logger.log("Getting User settings...");
@@ -48,10 +48,11 @@ export interface UserInfo {
 
 export async function getUserInfo(): Promise<UserInfo | undefined> {
   const apiToken = getAPIToken();
+  const apiTokenFromEnv = getCloudflareAPITokenFromEnv();
   return apiToken
     ? {
         apiToken,
-        authType: "OAuth",
+        authType: apiTokenFromEnv ? "API" : "OAuth",
         email: await getEmail(),
         accounts: await getAccounts(),
       }

package/src/worker.ts

@@ -105,6 +105,7 @@ interface CfDurableObject {
   name: string;
   class_name: string;
   script_name?: string;
+  environment?: string;
 }
 
 interface CfR2Bucket {
@@ -112,6 +113,12 @@ interface CfR2Bucket {
   bucket_name: string;
 }
 
+interface CfService {
+  binding: string;
+  service: string;
+  environment?: string;
+}
+
 interface CfUnsafeBinding {
   name: string;
   type: string;
@@ -157,6 +164,7 @@ export interface CfWorkerInit {
     data_blobs: CfDataBlobBindings | undefined;
     durable_objects: { bindings: CfDurableObject[] } | undefined;
     r2_buckets: CfR2Bucket[] | undefined;
+    services: CfService[] | undefined;
     unsafe: CfUnsafeBinding[] | undefined;
   };
   migrations: CfDurableObjectMigrations | undefined;
@@ -168,5 +176,6 @@ export interface CfWorkerInit {
 export interface CfWorkerContext {
   env: string | undefined;
   legacyEnv: boolean | undefined;
-  zone: { id: string; host: string } | undefined;
+  zone: string | undefined;
+  host: string | undefined;
 }

package/src/zones.ts

@@ -0,0 +1,73 @@
+import { fetchListResult } from "./cfetch";
+import type { Route } from "./config/environment";
+
+/**
+ * An object holding information about a zone for publishing.
+ */
+export interface Zone {
+  id: string;
+  host: string;
+}
+
+/**
+ * Try to compute the a zone ID and host name for one or more routes.
+ *
+ * When we're given a route, we do 2 things:
+ * - We try to extract a host from it
+ * - We try to get a zone id from the host
+ */
+export async function getZoneForRoute(route: Route): Promise<Zone | undefined> {
+  const host =
+    typeof route === "string"
+      ? getHostFromUrl(route)
+      : typeof route === "object"
+      ? "zone_name" in route
+        ? getHostFromUrl(route.zone_name)
+        : getHostFromUrl(route.pattern)
+      : undefined;
+  const id =
+    typeof route === "object" && "zone_id" in route
+      ? route.zone_id
+      : host
+      ? await getZoneIdFromHost(host)
+      : undefined;
+  return id && host ? { id, host } : undefined;
+}
+
+/**
+ * Given something that resembles a URL, try to extract a host from it.
+ */
+function getHostFromUrl(urlLike: string): string | undefined {
+  // strip leading * / *.
+  urlLike = urlLike.replace(/^\*(\.)?/g, "");
+
+  if (!(urlLike.startsWith("http://") || urlLike.startsWith("https://"))) {
+    urlLike = "http://" + urlLike;
+  }
+  return new URL(urlLike).host;
+}
+
+/**
+ * Given something that resembles a host, try to infer a zone id from it.
+ *
+ * It's hard to get a 'valid' domain from a string, so we don't even try to validate TLDs, etc.
+ * For each domain-like part of the host (e.g. w.x.y.z) try to get a zone id for it by
+ * lopping off subdomains until we get a hit from the API.
+ */
+export async function getZoneIdFromHost(host: string): Promise<string> {
+  const hostPieces = host.split(".");
+
+  while (hostPieces.length > 1) {
+    const zones = await fetchListResult<{ id: string }>(
+      `/zones`,
+      {},
+      new URLSearchParams({ name: hostPieces.join(".") })
+    );
+    if (zones.length > 0) {
+      return zones[0].id;
+    }
+    hostPieces.shift();
+  }
+
+  throw new Error(`Could not find zone for ${host}`);
+}

package/templates/new-worker-scheduled.js

@@ -0,0 +1,17 @@
+/**
+ * Welcome to Cloudflare Workers! This is your first scheduled worker.
+ *
+ * - Run `wrangler dev --local` in your terminal to start a development server
+ * - Run `curl "http://localhost:8787/cdn-cgi/mf/scheduled"` to trigger the scheduled event
+ * - Go back to the console to see what your worker has logged
+ * - Update the Cron trigger in wrangler.toml (see https://developers.cloudflare.com/workers/wrangler/configuration/#triggers)
+ * - Run `wrangler publish --name my-worker` to publish your worker
+ *
+ * Learn more at https://developers.cloudflare.com/workers/runtime-apis/scheduled-event/
+ */
+
+export default {
+  async scheduled(controller, env, ctx) {
+    console.log(`Hello World!`);
+  },
+};

package/templates/new-worker-scheduled.ts

@@ -0,0 +1,32 @@
+/**
+ * Welcome to Cloudflare Workers! This is your first scheduled worker.
+ *
+ * - Run `wrangler dev --local` in your terminal to start a development server
+ * - Run `curl "http://localhost:8787/cdn-cgi/mf/scheduled"` to trigger the scheduled event
+ * - Go back to the console to see what your worker has logged
+ * - Update the Cron trigger in wrangler.toml (see https://developers.cloudflare.com/workers/wrangler/configuration/#triggers)
+ * - Run `wrangler publish --name my-worker` to publish your worker
+ *
+ * Learn more at https://developers.cloudflare.com/workers/runtime-apis/scheduled-event/
+ */
+
+export interface Env {
+  // Example binding to KV. Learn more at https://developers.cloudflare.com/workers/runtime-apis/kv/
+  // MY_KV_NAMESPACE: KVNamespace;
+  //
+  // Example binding to Durable Object. Learn more at https://developers.cloudflare.com/workers/runtime-apis/durable-objects/
+  // MY_DURABLE_OBJECT: DurableObjectNamespace;
+  //
+  // Example binding to R2. Learn more at https://developers.cloudflare.com/workers/runtime-apis/r2/
+  // MY_BUCKET: R2Bucket;
+}
+
+export default {
+  async scheduled(
+    controller: ScheduledController,
+    env: Env,
+    ctx: ExecutionContext
+  ): Promise<void> {
+    console.log(`Hello World!`);
+  },
+};

package/templates/new-worker.ts

@@ -8,8 +8,23 @@
  * Learn more at https://developers.cloudflare.com/workers/
  */
 
+export interface Env {
+  // Example binding to KV. Learn more at https://developers.cloudflare.com/workers/runtime-apis/kv/
+  // MY_KV_NAMESPACE: KVNamespace;
+  //
+  // Example binding to Durable Object. Learn more at https://developers.cloudflare.com/workers/runtime-apis/durable-objects/
+  // MY_DURABLE_OBJECT: DurableObjectNamespace;
+  //
+  // Example binding to R2. Learn more at https://developers.cloudflare.com/workers/runtime-apis/r2/
+  // MY_BUCKET: R2Bucket;
+}
+
 export default {
-  async fetch(request: Request): Promise<Response> {
+  async fetch(
+    request: Request,
+    env: Env,
+    ctx: ExecutionContext
+  ): Promise<Response> {
     return new Response("Hello World!");
   },
 };

package/templates/static-asset-facade.js

@@ -4,10 +4,6 @@ import { getAssetFromKV } from "@cloudflare/kv-asset-handler";
 import manifest from "__STATIC_CONTENT_MANIFEST";
 const ASSET_MANIFEST = JSON.parse(manifest);
 
-// TODO: remove this
-globalThis.__STATIC_CONTENT = undefined;
-globalThis.__STATIC_CONTENT_MANIFEST = undefined;
-
 export default {
   async fetch(request, env, ctx) {
     let options = {