wrangler 0.0.2 → 0.0.6

package/src/proxy.ts

@@ -0,0 +1,104 @@
+import { connect } from "node:http2";
+import { createServer } from "node:http";
+import type {
+  Server,
+  IncomingHttpHeaders,
+  OutgoingHttpHeaders,
+  RequestListener,
+} from "node:http";
+import WebSocket from "faye-websocket";
+import serveStatic from "serve-static";
+
+export interface HttpProxyInit {
+  host: string;
+  assetPath?: string | null;
+  onRequest?: (headers: IncomingHttpHeaders) => void;
+  onResponse?: (headers: OutgoingHttpHeaders) => void;
+}
+
+/**
+ * Creates a HTTP/1 proxy that sends requests over HTTP/2.
+ */
+export function createHttpProxy(init: HttpProxyInit): Server {
+  const { host, assetPath, onRequest = () => {}, onResponse = () => {} } = init;
+  const remote = connect(`https://${host}`);
+  const local = createServer();
+  // HTTP/2 -> HTTP/2
+  local.on("stream", (stream, headers: IncomingHttpHeaders) => {
+    onRequest(headers);
+    headers[":authority"] = host;
+    const request = stream.pipe(remote.request(headers));
+    request.on("response", (headers: OutgoingHttpHeaders) => {
+      onResponse(headers);
+      stream.respond(headers);
+      request.pipe(stream, { end: true });
+    });
+  });
+  // HTTP/1 -> HTTP/2
+  const handleRequest: RequestListener = (message, response) => {
+    const { httpVersionMajor, headers, method, url } = message;
+    if (httpVersionMajor >= 2) {
+      return; // Already handled by the "stream" event.
+    }
+    onRequest(headers);
+    headers[":method"] = method;
+    headers[":path"] = url;
+    headers[":authority"] = host;
+    headers[":scheme"] = "https";
+    for (const name of Object.keys(headers)) {
+      if (HTTP1_HEADERS.has(name.toLowerCase())) {
+        delete headers[name];
+      }
+    }
+    const request = message.pipe(remote.request(headers));
+    request.on("response", (headers) => {
+      const status = headers[":status"];
+      onResponse(headers);
+      for (const name of Object.keys(headers)) {
+        if (name.startsWith(":")) {
+          delete headers[name];
+        }
+      }
+      response.writeHead(status, headers);
+      request.pipe(response, { end: true });
+    });
+  };
+  // If an asset path is defined, check the file system
+  // for a file first and serve if it exists.
+  if (assetPath) {
+    const handleAsset = serveStatic(assetPath, {
+      cacheControl: false,
+    });
+    local.on("request", (request, response) => {
+      handleAsset(request, response, () => {
+        handleRequest(request, response);
+      });
+    });
+  } else {
+    local.on("request", handleRequest);
+  }
+  // HTTP/1 -> WebSocket (over HTTP/1)
+  local.on("upgrade", (message, socket, body) => {
+    const { headers, url } = message;
+    onRequest(headers);
+    headers["host"] = host;
+    const local = new WebSocket(message, socket, body);
+    // TODO(soon): Custom WebSocket protocol is not working?
+    const remote = new WebSocket.Client(`wss://${host}${url}`, [], { headers });
+    local.pipe(remote).pipe(local);
+  });
+  remote.on("close", () => {
+    local.close();
+  });
+  return local;
+}
+
+const HTTP1_HEADERS = new Set([
+  "host",
+  "connection",
+  "upgrade",
+  "keep-alive",
+  "proxy-connection",
+  "transfer-encoding",
+  "http2-settings",
+]);

package/src/publish.ts

@@ -0,0 +1,358 @@
+import type { CfWorkerInit } from "./api/worker";
+import { toFormData } from "./api/form_data";
+import esbuild from "esbuild";
+import tmp from "tmp-promise";
+import type { Config } from "./config";
+import path from "path";
+import { readFile } from "fs/promises";
+import cfetch from "./cfetch";
+import assert from "node:assert";
+import { syncAssets } from "./sites";
+import makeModuleCollector from "./module-collection";
+import { execa } from "execa";
+
+type CfScriptFormat = void | "modules" | "service-worker";
+
+type Props = {
+  config: Config;
+  format?: CfScriptFormat;
+  script?: string;
+  name?: string;
+  env?: string;
+  public?: string;
+  site?: string;
+  triggers?: (string | number)[];
+  routes?: (string | number)[];
+  legacyEnv?: boolean;
+  jsxFactory: void | string;
+  jsxFragment: void | string;
+};
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export default async function publish(props: Props): Promise<void> {
+  if (props.public && props.format === "service-worker") {
+    // TODO: check config too
+    throw new Error(
+      "You cannot use the service worker format with a public directory."
+    );
+  }
+  // TODO: warn if git/hg has uncommitted changes
+  const { config } = props;
+  const {
+    account_id: accountId,
+    build,
+    // @ts-expect-error hidden
+    __path__,
+  } = config;
+
+  const triggers = props.triggers || config.triggers?.crons;
+  const routes = props.routes || config.routes;
+
+  const jsxFactory = props.jsxFactory || config.jsx_factory;
+  const jsxFragment = props.jsxFragment || config.jsx_fragment;
+
+  assert(config.account_id, "missing account id");
+
+  let scriptName = props.name || config.name;
+  assert(
+    scriptName,
+    'You need to provide a name when publishing a worker. Either pass it as a cli arg with `--name <name>` or in your config file as `name = "<name>"`'
+  );
+
+  let file: string;
+  if (props.script) {
+    file = props.script;
+  } else {
+    assert(build?.upload?.main, "missing main file");
+    file = path.join(path.dirname(__path__), build.upload.main);
+  }
+
+  if (props.legacyEnv) {
+    scriptName += props.env ? `-${props.env}` : "";
+  }
+  const envName = props.env ?? "production";
+
+  const destination = await tmp.dir({ unsafeCleanup: true });
+
+  if (props.config.build?.command) {
+    // TODO: add a deprecation message here?
+    console.log("running:", props.config.build.command);
+    const buildCommandPieces = props.config.build.command.split(" ");
+    await execa(buildCommandPieces[0], buildCommandPieces.slice(1), {
+      stdout: "inherit",
+      stderr: "inherit",
+      ...(props.config.build?.cwd && { cwd: props.config.build.cwd }),
+    });
+  }
+
+  const moduleCollector = makeModuleCollector();
+  const result = await esbuild.build({
+    ...(props.public
+      ? {
+          stdin: {
+            contents: (
+              await readFile(
+                path.join(__dirname, "../static-asset-facade.js"),
+                "utf8"
+              )
+            ).replace("__ENTRY_POINT__", path.join(process.cwd(), file)),
+            sourcefile: "static-asset-facade.js",
+            resolveDir: path.dirname(file),
+          },
+        }
+      : { entryPoints: [file] }),
+    bundle: true,
+    nodePaths: props.public ? [path.join(__dirname, "../vendor")] : undefined,
+    outdir: destination.path,
+    external: ["__STATIC_CONTENT_MANIFEST"],
+    format: "esm",
+    sourcemap: true,
+    metafile: true,
+    conditions: ["worker", "browser"],
+    loader: {
+      ".js": "jsx",
+    },
+    plugins: [moduleCollector.plugin],
+    ...(jsxFactory && { jsxFactory }),
+    ...(jsxFragment && { jsxFragment }),
+  });
+
+  const chunks = Object.entries(result.metafile.outputs).find(
+    ([_path, { entryPoint }]) =>
+      entryPoint ===
+      (props.public
+        ? path.join(path.dirname(file), "static-asset-facade.js")
+        : file)
+  );
+
+  const { format } = props;
+  const bundle = {
+    type: chunks[1].exports.length > 0 ? "esm" : "commonjs",
+    exports: chunks[1].exports,
+  };
+
+  // TODO: instead of bundling the facade with the worker, we should just bundle the worker and expose it as a module.
+  // That way we'll be able to accurately tell if this is a service worker or not.
+
+  if (format === "modules" && bundle.type === "commonjs") {
+    console.error("⎔ Cannot use modules with a commonjs bundle.");
+    // TODO: a much better error message here, with what to do next
+    return;
+  }
+  if (format === "service-worker" && bundle.type !== "esm") {
+    console.error("⎔ Cannot use service-worker with a esm bundle.");
+    // TODO: a much better error message here, with what to do next
+    return;
+  }
+
+  const content = await readFile(chunks[0], { encoding: "utf-8" });
+  await destination.cleanup();
+
+  // if config.migrations
+  // get current migration tag
+  let migrations;
+  if ("migrations" in config) {
+    const scripts = await cfetch<{ id: string; migration_tag: string }[]>(
+      `/accounts/${accountId}/workers/scripts`
+    );
+    const script = scripts.find((script) => script.id === scriptName);
+    if (script?.migration_tag) {
+      // was already published once
+      const foundIndex = config.migrations.findIndex(
+        (migration) => migration.tag === script.migration_tag
+      );
+      if (foundIndex === -1) {
+        console.warn(
+          `The published script ${scriptName} has a migration tag "${script.migration_tag}, which was not found in wrangler.toml. You may have already delated it. Applying all available migrations to the script...`
+        );
+        migrations = {
+          old_tag: script.migration_tag,
+          new_tag: config.migrations[config.migrations.length - 1].tag,
+          steps: config.migrations.map(({ tag: _tag, ...rest }) => rest),
+        };
+      } else {
+        migrations = {
+          old_tag: script.migration_tag,
+          new_tag: config.migrations[config.migrations.length - 1].tag,
+          steps: config.migrations
+            .slice(foundIndex + 1)
+            .map(({ tag: _tag, ...rest }) => rest),
+        };
+      }
+    } else {
+      migrations = {
+        new_tag: config.migrations[config.migrations.length - 1].tag,
+        steps: config.migrations.map(({ tag: _tag, ...rest }) => rest),
+      };
+    }
+  }
+
+  const assets =
+    props.public || props.site || props.config.site?.bucket // TODO: allow both
+      ? await syncAssets(
+          accountId,
+          scriptName,
+          props.public || props.site || props.config.site?.bucket,
+          false
+        )
+      : { manifest: undefined, namespace: undefined };
+
+  const envRootObj = props.env ? config.env[props.env] || {} : config;
+
+  const worker: CfWorkerInit = {
+    name: scriptName,
+    main: {
+      name: path.basename(chunks[0]),
+      content: content,
+      type: bundle.type === "esm" ? "esm" : "commonjs",
+    },
+    variables: {
+      ...(envRootObj?.vars || {}),
+      ...(envRootObj?.kv_namespaces || []).reduce(
+        (obj, { binding, preview_id: _preview_id, id }) => {
+          return { ...obj, [binding]: { namespaceId: id } };
+        },
+        {}
+      ),
+      ...(envRootObj?.durable_objects?.bindings || []).reduce(
+        (obj, { name, class_name, script_name }) => {
+          return {
+            ...obj,
+            [name]: { class_name, ...(script_name && { script_name }) },
+          };
+        },
+        {}
+      ),
+      ...(assets.namespace
+        ? { __STATIC_CONTENT: { namespaceId: assets.namespace } }
+        : {}),
+    },
+    ...(migrations && { migrations }),
+    modules: assets.manifest
+      ? moduleCollector.modules.concat({
+          name: "__STATIC_CONTENT_MANIFEST",
+          content: JSON.stringify(assets.manifest),
+          type: "text",
+        })
+      : moduleCollector.modules,
+    compatibility_date: config.compatibility_date,
+    compatibility_flags: config.compatibility_flags,
+    usage_model: config.usage_model,
+  };
+
+  const start = Date.now();
+  function formatTime(duration: number) {
+    return `(${(duration / 1000).toFixed(2)} sec)`;
+  }
+
+  const notProd = !props.legacyEnv && props.env;
+  const workerName = notProd ? `${scriptName} (${envName})` : scriptName;
+  const workerUrl = notProd
+    ? `/accounts/${accountId}/workers/services/${scriptName}/environments/${envName}`
+    : `/accounts/${accountId}/workers/scripts/${scriptName}`;
+
+  // Upload the script so it has time to propogate.
+  const { available_on_subdomain } = await cfetch(
+    `${workerUrl}?available_on_subdomain=true`,
+    {
+      method: "PUT",
+      // @ts-expect-error: TODO: fix this type error!
+      body: toFormData(worker),
+    }
+  );
+
+  const uploadMs = Date.now() - start;
+  console.log("Uploaded", workerName, formatTime(uploadMs));
+  const deployments: Promise<string[]>[] = [];
+
+  const userSubdomain = (
+    await cfetch<{ subdomain: string }>(
+      `/accounts/${accountId}/workers/subdomain`
+    )
+  ).subdomain;
+
+  const scriptURL =
+    props.legacyEnv || !props.env
+      ? `${scriptName}.${userSubdomain}.workers.dev`
+      : `${envName}.${scriptName}.${userSubdomain}.workers.dev`;
+
+  // Enable the `workers.dev` subdomain.
+  // TODO: Make this configurable.
+  if (!available_on_subdomain) {
+    deployments.push(
+      cfetch(`${workerUrl}/subdomain`, {
+        method: "POST",
+        body: JSON.stringify({ enabled: true }),
+        headers: {
+          "Content-Type": "application/json",
+        },
+      })
+        .then(() => [scriptURL])
+        // Add a delay when the subdomain is first created.
+        // This is to prevent an issue where a negative cache-hit
+        // causes the subdomain to be unavailable for 30 seconds.
+        // This is a temporary measure until we fix this on the edge.
+        .then(async (url) => {
+          await sleep(3000);
+          return url;
+        })
+    );
+  } else {
+    deployments.push(Promise.resolve([scriptURL]));
+  }
+
+  // Update routing table for the script.
+  if (routes && routes.length) {
+    deployments.push(
+      cfetch(`${workerUrl}/routes`, {
+        // TODO: PATCH will not delete previous routes on this script,
+        // whereas PUT will. We need to decide on the default behaviour
+        // and how to configure it.
+        method: "PUT",
+        body: JSON.stringify(routes.map((pattern) => ({ pattern }))),
+        headers: {
+          "Content-Type": "application/json",
+        },
+      }).then(() => {
+        if (routes.length > 10) {
+          return routes
+            .slice(0, 9)
+            .map(String)
+            .concat([`...and ${routes.length - 10} more routes`]);
+        }
+        return routes.map(String);
+      })
+    );
+  }
+
+  // Configure any schedules for the script.
+  // TODO: rename this to `schedules`?
+  if (triggers && triggers.length) {
+    deployments.push(
+      cfetch(`${workerUrl}/schedules`, {
+        // TODO: Unlike routes, this endpoint does not support PATCH.
+        // So technically, this will override any previous schedules.
+        // We should change the endpoint to support PATCH.
+        method: "PUT",
+        body: JSON.stringify(triggers.map((cron) => ({ cron }))),
+        headers: {
+          "Content-Type": "application/json",
+        },
+      }).then(() => triggers.map(String))
+    );
+  }
+
+  if (!deployments.length) {
+    return;
+  }
+
+  const targets = await Promise.all(deployments);
+  const deployMs = Date.now() - start - uploadMs;
+  console.log("Deployed", workerName, formatTime(deployMs));
+  for (const target of targets.flat()) {
+    console.log(" ", target);
+  }
+}

package/src/sites.tsx

@@ -0,0 +1,115 @@
+import { readdir, readFile } from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import cfetch from "./cfetch";
+import { listNamespaceKeys, listNamespaces, putBulkKeyValue } from "./kv";
+
+import * as path from "path";
+import crypto from "node:crypto";
+
+async function* getFilesInFolder(dirPath: string): AsyncIterable<string> {
+  const files = await readdir(dirPath, { withFileTypes: true });
+  for (const file of files) {
+    if (file.isDirectory()) {
+      yield* await getFilesInFolder(path.join(dirPath, file.name));
+    } else {
+      yield path.join(dirPath, file.name);
+    }
+  }
+}
+
+async function hashFileContent(filePath: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash("sha1");
+    const rs = createReadStream(filePath);
+    rs.on("error", reject);
+    rs.on("data", (chunk) => hash.update(chunk));
+    rs.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+async function hashFile(filePath: string): Promise<{
+  filePath: string;
+  hash: string;
+}> {
+  const extName = path.extname(filePath);
+  const baseName = path.basename(filePath, extName);
+  const hash = await hashFileContent(filePath);
+  return {
+    filePath: `${baseName}.${hash}${extName || ""}`,
+    hash,
+  };
+}
+
+async function createKVNamespaceIfNotAlreadyExisting(
+  title: string,
+  accountId: string
+) {
+  // check if it already exists
+  // TODO: this is super inefficient, should be made better
+  const namespaces = await listNamespaces(accountId);
+  const found = namespaces.find((x) => x.title === title);
+  if (found) {
+    return { created: false, id: found.id };
+  }
+
+  // else we make the namespace
+  // TODO: use an export from ./kv
+  const json = await cfetch<{ id: string }>(
+    `/accounts/${accountId}/storage/kv/namespaces`,
+    {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({ title }),
+    }
+  );
+
+  return {
+    created: true,
+    id: json.id,
+  };
+}
+
+export async function syncAssets(
+  accountId: string,
+  scriptName: string,
+  dirPath: string,
+  preview: boolean,
+  _env?: string
+) {
+  const title = `__${scriptName}_sites_assets${preview ? "_preview" : ""}`;
+  const { id: namespace } = await createKVNamespaceIfNotAlreadyExisting(
+    title,
+    accountId
+  );
+
+  // let's get all the keys in this namespace
+  const keys = new Set(
+    (await listNamespaceKeys(accountId, namespace)).map((x) => x.name)
+  );
+
+  const manifest = {};
+  const upload = [];
+  // TODO: this can be more efficient by parallelising
+  for await (const file of getFilesInFolder(dirPath)) {
+    // TODO: "exclude:" config
+    const { filePath } = await hashFile(file);
+    // now put each of the files into kv
+    if (!keys.has(filePath)) {
+      console.log(`uploading ${file}...`);
+      const content = await readFile(file, "base64");
+      if (content.length > 25 * 1024 * 1024) {
+        throw new Error(`File ${file} is too big, it should be under 25 mb.`);
+      }
+      upload.push({
+        key: filePath,
+        value: content,
+        base64: true,
+      });
+    }
+    manifest[path.relative(dirPath, file)] = filePath;
+  }
+  await putBulkKeyValue(accountId, namespace, JSON.stringify(upload));
+  return { manifest, namespace };
+}

package/src/tail.tsx

@@ -0,0 +1,71 @@
+import WebSocket from "ws";
+import cfetch from "./cfetch";
+import { version as packageVersion } from "../package.json";
+
+export type TailApiResponse = {
+  id: string;
+  url: string;
+  expires_at: Date;
+};
+
+function makeCreateTailUrl(accountId: string, workerName: string): string {
+  return `/accounts/${accountId}/workers/scripts/${workerName}/tails`;
+}
+
+function makeDeleteTailUrl(
+  accountId: string,
+  workerName: string,
+  tailId: string
+): string {
+  return `/accounts/${accountId}/workers/scripts/${workerName}/tails/${tailId}`;
+}
+
+/// Creates a tail, but doesn't connect to it.
+async function createTailButDontConnect(
+  accountId: string,
+  workerName: string
+): Promise<TailApiResponse> {
+  const createTailUrl = makeCreateTailUrl(accountId, workerName);
+  /// https://api.cloudflare.com/#worker-tail-logs-start-tail
+  return await cfetch<TailApiResponse>(createTailUrl, { method: "POST" });
+}
+
+export async function createTail(
+  accountId: string,
+  workerName: string,
+  _filters: Filters
+): Promise<{
+  tail: WebSocket;
+  expiration: Date;
+  deleteTail: () => Promise<void>;
+}> {
+  const {
+    id: tailId,
+    url: websocketUrl,
+    expires_at: expiration,
+  } = await createTailButDontConnect(accountId, workerName);
+  const deleteUrl = makeDeleteTailUrl(accountId, workerName, tailId);
+
+  // deletes the tail
+  async function deleteTail() {
+    await cfetch(deleteUrl, { method: "DELETE" });
+  }
+
+  const tail = new WebSocket(websocketUrl, "trace-v1", {
+    headers: {
+      "Sec-WebSocket-Protocol": "trace-v1", // needs to be `trace-v1` to be accepted
+      "User-Agent": `wrangler-js/${packageVersion}`,
+    },
+  });
+
+  // TODO: send filters as well
+  return { tail, expiration, deleteTail };
+}
+
+export type Filters = {
+  status?: "ok" | "error" | "canceled";
+  header?: string;
+  method?: string;
+  "sampling-rate"?: number;
+  search?: string;
+};