staticalize 0.2.2

package/src/deps/jsr.io/@std/path/1.1.4/windows/to_file_url.ts

@@ -0,0 +1,38 @@
+// Copyright 2018-2025 the Deno authors. MIT license.
+// This module is browser compatible.
+
+import { encodeWhitespace } from "../_common/to_file_url.js";
+import { isAbsolute } from "./is_absolute.js";
+
+/**
+ * Converts a path string to a file URL.
+ *
+ * @example Usage
+ * ```ts
+ * import { toFileUrl } from "@std/path/windows/to-file-url";
+ * import { assertEquals } from "@std/assert";
+ *
+ * assertEquals(toFileUrl("\\home\\foo"), new URL("file:///home/foo"));
+ * assertEquals(toFileUrl("C:\\Users\\foo"), new URL("file:///C:/Users/foo"));
+ * assertEquals(toFileUrl("\\\\127.0.0.1\\home\\foo"), new URL("file://127.0.0.1/home/foo"));
+ * ```
+ * @param path The path to convert.
+ * @returns The file URL.
+ */
+export function toFileUrl(path: string): URL {
+  if (!isAbsolute(path)) {
+    throw new TypeError(`Path must be absolute: received "${path}"`);
+  }
+  const [, hostname, pathname] = path.match(
+    /^(?:[/\\]{2}([^/\\]+)(?=[/\\](?:[^/\\]|$)))?(.*)/,
+  )!;
+  const url = new URL("file:///");
+  url.pathname = encodeWhitespace(pathname!.replace(/%/g, "%25"));
+  if (hostname !== undefined && hostname !== "localhost") {
+    url.hostname = hostname;
+    if (!url.hostname) {
+      throw new TypeError(`Invalid hostname: "${url.hostname}"`);
+    }
+  }
+  return url;
+}

package/src/deps/jsr.io/@std/path/1.1.4/windows/to_namespaced_path.ts

@@ -0,0 +1,60 @@
+// Copyright 2018-2025 the Deno authors. MIT license.
+// This module is browser compatible.
+
+import {
+  CHAR_BACKWARD_SLASH,
+  CHAR_COLON,
+  CHAR_DOT,
+  CHAR_QUESTION_MARK,
+} from "../_common/constants.js";
+import { isWindowsDeviceRoot } from "./_util.js";
+import { resolve } from "./resolve.js";
+
+/**
+ * Resolves path to a namespace path
+ *
+ * @example Usage
+ * ```ts
+ * import { toNamespacedPath } from "@std/path/windows/to-namespaced-path";
+ * import { assertEquals } from "@std/assert";
+ *
+ * const namespaced = toNamespacedPath("C:\\foo\\bar");
+ * assertEquals(namespaced, "\\\\?\\C:\\foo\\bar");
+ * ```
+ *
+ * @param path The path to resolve to namespaced path
+ * @returns The resolved namespaced path
+ */
+export function toNamespacedPath(path: string): string {
+  // Note: this will *probably* throw somewhere.
+  if (typeof path !== "string") return path;
+  if (path.length === 0) return "";
+
+  const resolvedPath = resolve(path);
+
+  if (resolvedPath.length >= 3) {
+    if (resolvedPath.charCodeAt(0) === CHAR_BACKWARD_SLASH) {
+      // Possible UNC root
+
+      if (resolvedPath.charCodeAt(1) === CHAR_BACKWARD_SLASH) {
+        const code = resolvedPath.charCodeAt(2);
+        if (code !== CHAR_QUESTION_MARK && code !== CHAR_DOT) {
+          // Matched non-long UNC root, convert the path to a long UNC path
+          return `\\\\?\\UNC\\${resolvedPath.slice(2)}`;
+        }
+      }
+    } else if (isWindowsDeviceRoot(resolvedPath.charCodeAt(0))) {
+      // Possible device root
+
+      if (
+        resolvedPath.charCodeAt(1) === CHAR_COLON &&
+        resolvedPath.charCodeAt(2) === CHAR_BACKWARD_SLASH
+      ) {
+        // Matched device root, convert the path to a long UNC path
+        return `\\\\?\\${resolvedPath}`;
+      }
+    }
+  }
+
+  return path;
+}

package/src/main.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+import "./_dnt.polyfills.js";
+import * as dntShim from "./_dnt.shims.js";
+import { main } from "effection";
+import { staticalize } from "./staticalize.js";
+import { config } from "./config.js";
+
+
+await main(function* (args) {
+  let parser = config.createParser({
+    envs: [{ name: "ENV", value: dntShim.Deno.env.toObject() }],
+    args,
+  })
+
+  switch (parser.type) {
+    case "help":
+    case "version":
+      console.log(parser.print());
+      break;
+    case "main": {
+      let result = parser.parse();
+      if (result.ok) {
+	let { base, site, output } = result.value;
+        yield* staticalize({
+	  base: new URL(base),
+	  host: new URL(site),
+	  dir: output,	  
+	});
+      } else {
+	console.error(result.error.message);
+	console.log(`\n\`staticalize --help\` for available options`)	
+        
+      }
+      break;
+    }
+  }
+});

package/src/mod.ts

@@ -0,0 +1,2 @@
+import "./_dnt.polyfills.js";
+export * from "./staticalize.js";

package/src/staticalize.ts

@@ -0,0 +1,216 @@
+import * as dntShim from "./_dnt.shims.js";
+import {
+  call,
+  type Operation,
+  resource,
+  spawn,
+  until,
+  useAbortSignal,
+} from "effection";
+import { dirname, join, normalize } from "./deps/jsr.io/@std/path/1.1.4/mod.js";
+import { ensureDir } from "./deps/jsr.io/@std/fs/1.0.22/ensure_dir.js";
+import { stringify } from "./deps/jsr.io/@libs/xml/6.0.8/stringify.js";
+import { fromHtml } from "hast-util-from-html";
+import { toHtml } from "hast-util-to-html";
+import { selectAll } from "hast-util-select";
+import { parse } from "./deps/jsr.io/@libs/xml/6.0.8/parse.js";
+import { useTaskBuffer } from "./task-buffer.js";
+
+export interface StaticalizeOptions {
+  host: URL;
+  base: URL;
+  dir: string;
+}
+
+export interface StaticalizeSummary {
+  durationMS: number;
+}
+
+export function* staticalize(options: StaticalizeOptions): Operation<void> {
+  let { host, base, dir } = options;
+
+  let signal = yield* useAbortSignal();
+
+  let urls: SitemapURL[] = yield* call(async () => {
+    let url = new URL("/sitemap.xml", host);
+    let response = await fetch(url, { signal });
+    if (!response.ok) {
+      let error = new Error(
+        `GET ${url} ${response.status} ${response.statusText}`,
+      );
+      error.name = `SitemapError`;
+      throw error;
+    }
+    let text = await response.text();
+    let xml = parse(text, {
+      flatten: { attributes: false, empty: false, text: true },
+    }) as unknown as SitemapXML;
+
+    let urls = xml.urlset.url;
+
+    return Array.isArray(urls) ? urls : [urls];
+  });
+
+  let downloader = yield* useDownloader({ host, base, outdir: dir });
+
+  yield* call(() => ensureDir(dir));
+
+  for (let url of urls) {
+    yield* downloader.download(url.loc);
+  }
+
+  let sitemap = yield* spawn(function* () {
+    let xml = stringify({
+      urlset: {
+        "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9",
+        "urls": urls.map((url) => {
+          let loc = new URL(url.loc);
+          loc.host = base.host;
+          loc.port = base.port;
+          loc.protocol = base.protocol;
+          return { loc: { "#text": loc } };
+        }),
+      },
+    });
+    yield* call(() =>
+      dntShim.Deno.writeFile(
+        join(dir, "sitemap.xml"),
+        new TextEncoder().encode(xml),
+      )
+    );
+  });
+
+  yield* sitemap;
+  yield* downloader;
+}
+
+interface Downloader extends Operation<void> {
+  download(spec: string, context?: URL): Operation<void>;
+}
+
+interface DownloaderOptions {
+  host: URL;
+  base: URL;
+  outdir: string;
+}
+
+function useDownloader(opts: DownloaderOptions): Operation<Downloader> {
+  let seen = new Map<string, boolean>();
+  return resource(function* (provide) {
+    let { host, base, outdir } = opts;
+
+    let buffer = yield* useTaskBuffer(75);
+
+    let signal = yield* useAbortSignal();
+
+    let downloader: Downloader = {
+      *download(loc, context = host) {
+        if (seen.get(loc)) {
+          return;
+        }
+        seen.set(loc, true);
+        if (loc.startsWith("//")) {
+          return;
+        }
+        let source = loc.match(/^\w+:/) ? new URL(loc) : new URL(loc, context);
+        if (source.host !== host.host) {
+          return;
+        }
+        let path = normalize(join(outdir, source.pathname));
+
+        yield* buffer.spawn(function* () {
+          let response = yield* call(() =>
+            fetch(source.toString(), { signal })
+          );
+          if (response.ok) {
+            if (response.headers.get("Content-Type")?.includes("html")) {
+              let destpath = join(path, "index.html");
+              let content = yield* until(response.text());
+              let html = fromHtml(content);
+
+              let links = selectAll("link[href]", html);
+
+              for (let link of links) {
+                let href = link.properties.href as string;
+                yield* downloader.download(href, source);
+
+                // replace self-referencing absolute urls with the destination site
+                if (href.startsWith(host.origin)) {
+                  let url = new URL(href);
+                  url.host = base.host;
+                  url.port = base.port;
+                  url.protocol = base.protocol;
+                  link.properties.href = url.href;
+                }
+              }
+
+              let assets = selectAll("[src]", html);
+
+              for (let element of assets) {
+                let src = element.properties.src as string;
+                yield* downloader.download(src, source);
+
+                // replace self-referencing absolute urls with the destination sie
+                if (src.startsWith(host.origin)) {
+                  let url = new URL(src);
+                  url.host = base.host;
+                  url.port = base.port;
+                  url.protocol = base.protocol;
+                  element.properties.src = url.href;
+                }
+              }
+
+              let withContents = selectAll("[content]", html);
+              for (let element of withContents) {
+                let attr = String(element.properties.content);
+                if (attr.startsWith(host.origin)) {
+                  yield* downloader.download(attr, source);
+                  let url = new URL(attr);
+                  url.host = base.host;
+                  url.port = base.port;
+                  url.protocol = base.protocol;
+                  element.properties.content = url.href;
+                }
+              }
+
+              yield* call(async () => {
+                let destdir = dirname(destpath);
+                await ensureDir(destdir);
+                await dntShim.Deno.writeTextFile(destpath, toHtml(html));
+              });
+            } else {
+              yield* call(async () => {
+                let destpath = path;
+                let destdir = dirname(destpath);
+                await ensureDir(destdir);
+                await dntShim.Deno.writeFile(destpath, response.body!);
+              });
+            }
+          } else {
+            throw new Error(
+              `GET ${source} ${response.status} ${response.statusText}`,
+            );
+          }
+        });
+      },
+      *[Symbol.iterator]() {
+        yield* buffer;
+      },
+    };
+
+    yield* provide(downloader);
+  });
+}
+
+interface SitemapURL {
+  loc: string;
+  lastmod?: string;
+  changefreq?: string;
+  priority?: string;
+}
+
+interface SitemapXML {
+  urlset: {
+    url: SitemapURL | SitemapURL[];
+  };
+}

package/src/task-buffer.ts

@@ -0,0 +1,87 @@
+import {
+  createChannel,
+  Err,
+  Ok,
+  type Operation,
+  type Resolve,
+  resource,
+  type Result,
+  spawn,
+  type Stream,
+  type Task,
+  useScope,
+  withResolvers,
+} from "effection";
+
+export interface TaskBuffer extends Operation<void> {
+  spawn<T>(op: () => Operation<T>): Operation<Operation<Task<T>>>;
+}
+
+export function useTaskBuffer(max: number): Operation<TaskBuffer> {
+  return resource(function* (provide) {
+    let input = createChannel<void, never>();
+
+    let output = createChannel<Result<unknown>, never>();
+
+    let buffer = new Set<Task<unknown>>();
+
+    let scope = yield* useScope();
+
+    let requests: SpawnRequest<unknown>[] = [];
+
+    yield* spawn(function* () {
+      while (true) {
+        if (requests.length === 0) {
+          yield* next(input);
+        } else if (buffer.size < max) {
+          let request = requests.pop()!;
+          let task = yield* scope.spawn(request.operation);
+          buffer.add(task);
+          yield* spawn(function* () {
+            try {
+              let result = Ok(yield* task);
+              buffer.delete(task);
+              yield* output.send(result);
+            } catch (error) {
+              buffer.delete(task);
+              yield* output.send(Err(error as Error));
+            }
+          });
+          request.resolve(task);
+        } else {
+          yield* next(output);
+        }
+      }
+    });
+
+    yield* provide({
+      *[Symbol.iterator]() {
+        let outputs = yield* output;
+        while (buffer.size > 0 || requests.length > 0) {
+          yield* outputs.next();
+        }
+      },
+      *spawn<T>(fn: () => Operation<T>) {
+        let { operation, resolve } = withResolvers<Task<T>>();
+        requests.unshift({
+          operation: fn,
+          resolve: resolve as Resolve<unknown>,
+        });
+        yield* input.send();
+        return operation;
+      },
+    });
+  });
+}
+
+interface SpawnRequest<T> {
+  operation(): Operation<T>;
+  resolve: Resolve<Task<T>>;
+}
+
+function* next<T, TClose>(
+  stream: Stream<T, TClose>,
+): Operation<IteratorResult<T, TClose>> {
+  let subscription = yield* stream;
+  return yield* subscription.next();
+}