staticalize 0.2.2

package/esm/staticalize.js

@@ -0,0 +1,144 @@
+import * as dntShim from "./_dnt.shims.js";
+import { call, resource, spawn, until, useAbortSignal, } from "effection";
+import { dirname, join, normalize } from "./deps/jsr.io/@std/path/1.1.4/mod.js";
+import { ensureDir } from "./deps/jsr.io/@std/fs/1.0.22/ensure_dir.js";
+import { stringify } from "./deps/jsr.io/@libs/xml/6.0.8/stringify.js";
+import { fromHtml } from "hast-util-from-html";
+import { toHtml } from "hast-util-to-html";
+import { selectAll } from "hast-util-select";
+import { parse } from "./deps/jsr.io/@libs/xml/6.0.8/parse.js";
+import { useTaskBuffer } from "./task-buffer.js";
+export function* staticalize(options) {
+    let { host, base, dir } = options;
+    let signal = yield* useAbortSignal();
+    let urls = yield* call(async () => {
+        let url = new URL("/sitemap.xml", host);
+        let response = await fetch(url, { signal });
+        if (!response.ok) {
+            let error = new Error(`GET ${url} ${response.status} ${response.statusText}`);
+            error.name = `SitemapError`;
+            throw error;
+        }
+        let text = await response.text();
+        let xml = parse(text, {
+            flatten: { attributes: false, empty: false, text: true },
+        });
+        let urls = xml.urlset.url;
+        return Array.isArray(urls) ? urls : [urls];
+    });
+    let downloader = yield* useDownloader({ host, base, outdir: dir });
+    yield* call(() => ensureDir(dir));
+    for (let url of urls) {
+        yield* downloader.download(url.loc);
+    }
+    let sitemap = yield* spawn(function* () {
+        let xml = stringify({
+            urlset: {
+                "@xmlns": "http://www.sitemaps.org/schemas/sitemap/0.9",
+                "urls": urls.map((url) => {
+                    let loc = new URL(url.loc);
+                    loc.host = base.host;
+                    loc.port = base.port;
+                    loc.protocol = base.protocol;
+                    return { loc: { "#text": loc } };
+                }),
+            },
+        });
+        yield* call(() => dntShim.Deno.writeFile(join(dir, "sitemap.xml"), new TextEncoder().encode(xml)));
+    });
+    yield* sitemap;
+    yield* downloader;
+}
+function useDownloader(opts) {
+    let seen = new Map();
+    return resource(function* (provide) {
+        let { host, base, outdir } = opts;
+        let buffer = yield* useTaskBuffer(75);
+        let signal = yield* useAbortSignal();
+        let downloader = {
+            *download(loc, context = host) {
+                if (seen.get(loc)) {
+                    return;
+                }
+                seen.set(loc, true);
+                if (loc.startsWith("//")) {
+                    return;
+                }
+                let source = loc.match(/^\w+:/) ? new URL(loc) : new URL(loc, context);
+                if (source.host !== host.host) {
+                    return;
+                }
+                let path = normalize(join(outdir, source.pathname));
+                yield* buffer.spawn(function* () {
+                    let response = yield* call(() => fetch(source.toString(), { signal }));
+                    if (response.ok) {
+                        if (response.headers.get("Content-Type")?.includes("html")) {
+                            let destpath = join(path, "index.html");
+                            let content = yield* until(response.text());
+                            let html = fromHtml(content);
+                            let links = selectAll("link[href]", html);
+                            for (let link of links) {
+                                let href = link.properties.href;
+                                yield* downloader.download(href, source);
+                                // replace self-referencing absolute urls with the destination site
+                                if (href.startsWith(host.origin)) {
+                                    let url = new URL(href);
+                                    url.host = base.host;
+                                    url.port = base.port;
+                                    url.protocol = base.protocol;
+                                    link.properties.href = url.href;
+                                }
+                            }
+                            let assets = selectAll("[src]", html);
+                            for (let element of assets) {
+                                let src = element.properties.src;
+                                yield* downloader.download(src, source);
+                                // replace self-referencing absolute urls with the destination sie
+                                if (src.startsWith(host.origin)) {
+                                    let url = new URL(src);
+                                    url.host = base.host;
+                                    url.port = base.port;
+                                    url.protocol = base.protocol;
+                                    element.properties.src = url.href;
+                                }
+                            }
+                            let withContents = selectAll("[content]", html);
+                            for (let element of withContents) {
+                                let attr = String(element.properties.content);
+                                if (attr.startsWith(host.origin)) {
+                                    yield* downloader.download(attr, source);
+                                    let url = new URL(attr);
+                                    url.host = base.host;
+                                    url.port = base.port;
+                                    url.protocol = base.protocol;
+                                    element.properties.content = url.href;
+                                }
+                            }
+                            yield* call(async () => {
+                                let destdir = dirname(destpath);
+                                await ensureDir(destdir);
+                                await dntShim.Deno.writeTextFile(destpath, toHtml(html));
+                            });
+                        }
+                        else {
+                            yield* call(async () => {
+                                let destpath = path;
+                                let destdir = dirname(destpath);
+                                await ensureDir(destdir);
+                                await dntShim.Deno.writeFile(destpath, response.body);
+                            });
+                        }
+                    }
+                    else {
+                        throw new Error(`GET ${source} ${response.status} ${response.statusText}`);
+                    }
+                });
+            },
+            *[Symbol.iterator]() {
+                yield* buffer;
+            },
+        };
+        yield* provide(downloader);
+    });
+}
+//# sourceMappingURL=staticalize.js.map

package/esm/staticalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"staticalize.js","sourceRoot":"","sources":["../src/staticalize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EACL,IAAI,EAEJ,QAAQ,EACR,KAAK,EACL,KAAK,EACL,cAAc,GACf,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,4CAA4C,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,4CAA4C,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,wCAAwC,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAYjD,MAAM,SAAS,CAAC,CAAC,WAAW,CAAC,OAA2B;IACtD,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAElC,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,cAAc,EAAE,CAAC;IAErC,IAAI,IAAI,GAAiB,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;QAC9C,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,IAAI,KAAK,GAAG,IAAI,KAAK,CACnB,OAAO,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CACvD,CAAC;YACF,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;YAC5B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE;YACpB,OAAO,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE;SACzD,CAA0B,CAAC;QAE5B,IAAI,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;QAE1B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAG,KAAK,CAAC,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAEnE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAElC,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,KAAK,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAClC,IAAI,GAAG,GAAG,SAAS,CAAC;YAClB,MAAM,EAAE;gBACN,QAAQ,EAAE,6CAA6C;gBACvD,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;oBACvB,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC3B,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oBACrB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oBACrB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oBAC7B,OAAO,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;gBACnC,CAAC,CAAC;aACH;SACF,CAAC,CAAC;QACH,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CACf,OAAO,CAAC,IAAI,CAAC,SAAS,CACpB,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EACxB,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAC9B,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,CAAC,OAAO,CAAC;IACf,KAAK,CAAC,CAAC,UAAU,CAAC;AACpB,CAAC;AAYD,SAAS,aAAa,CAAC,IAAuB;IAC5C,IAAI,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IACtC,OAAO,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO;QAChC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAElC,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAEtC,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,cAAc,EAAE,CAAC;QAErC,IAAI,UAAU,GAAe;YAC3B,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI;gBAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClB,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACpB,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBACzB,OAAO;gBACT,CAAC;gBACD,IAAI,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACvE,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC9B,OAAO;gBACT,CAAC;gBACD,IAAI,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAEpD,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;oBAC3B,IAAI,QAAQ,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAC9B,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CACrC,CAAC;oBACF,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;wBAChB,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;4BAC3D,IAAI,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;4BACxC,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;4BAC5C,IAAI,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;4BAE7B,IAAI,KAAK,GAAG,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;4BAE1C,KAAK,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gCACvB,IAAI,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,IAAc,CAAC;gCAC1C,KAAK,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gCAEzC,mEAAmE;gCACnE,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;oCACjC,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;oCACxB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oCACrB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oCACrB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oCAC7B,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;gCAClC,CAAC;4BACH,CAAC;4BAED,IAAI,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;4BAEtC,KAAK,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;gCAC3B,IAAI,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,GAAa,CAAC;gCAC3C,KAAK,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gCAExC,kEAAkE;gCAClE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;oCAChC,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oCACvB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oCACrB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oCACrB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oCAC7B,OAAO,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;gCACpC,CAAC;4BACH,CAAC;4BAED,IAAI,YAAY,GAAG,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;4BAChD,KAAK,IAAI,OAAO,IAAI,YAAY,EAAE,CAAC;gCACjC,IAAI,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gCAC9C,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;oCACjC,KAAK,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;oCACzC,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;oCACxB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oCACrB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;oCACrB,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;oCAC7B,OAAO,CAAC,UAAU,CAAC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC;gCACxC,CAAC;4BACH,CAAC;4BAED,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gCACrB,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;gCAChC,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;gCACzB,MAAM,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC3D,CAAC,CAAC,CAAC;wBACL,CAAC;6BAAM,CAAC;4BACN,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gCACrB,IAAI,QAAQ,GAAG,IAAI,CAAC;gCACpB,IAAI,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;gCAChC,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;gCACzB,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAK,CAAC,CAAC;4BACzD,CAAC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,MAAM,IAAI,KAAK,CACb,OAAO,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC1D,CAAC;oBACJ,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YACD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChB,KAAK,CAAC,CAAC,MAAM,CAAC;YAChB,CAAC;SACF,CAAC;QAEF,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC"}

package/esm/task-buffer.d.ts

@@ -0,0 +1,6 @@
+import { type Operation, type Task } from "effection";
+export interface TaskBuffer extends Operation<void> {
+    spawn<T>(op: () => Operation<T>): Operation<Operation<Task<T>>>;
+}
+export declare function useTaskBuffer(max: number): Operation<TaskBuffer>;
+//# sourceMappingURL=task-buffer.d.ts.map

package/esm/task-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-buffer.d.ts","sourceRoot":"","sources":["../src/task-buffer.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,SAAS,EAMd,KAAK,IAAI,EAGV,MAAM,WAAW,CAAC;AAEnB,MAAM,WAAW,UAAW,SAAQ,SAAS,CAAC,IAAI,CAAC;IACjD,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CACjE;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC,UAAU,CAAC,CAuDhE"}

package/esm/task-buffer.js

@@ -0,0 +1,59 @@
+import { createChannel, Err, Ok, resource, spawn, useScope, withResolvers, } from "effection";
+export function useTaskBuffer(max) {
+    return resource(function* (provide) {
+        let input = createChannel();
+        let output = createChannel();
+        let buffer = new Set();
+        let scope = yield* useScope();
+        let requests = [];
+        yield* spawn(function* () {
+            while (true) {
+                if (requests.length === 0) {
+                    yield* next(input);
+                }
+                else if (buffer.size < max) {
+                    let request = requests.pop();
+                    let task = yield* scope.spawn(request.operation);
+                    buffer.add(task);
+                    yield* spawn(function* () {
+                        try {
+                            let result = Ok(yield* task);
+                            buffer.delete(task);
+                            yield* output.send(result);
+                        }
+                        catch (error) {
+                            buffer.delete(task);
+                            yield* output.send(Err(error));
+                        }
+                    });
+                    request.resolve(task);
+                }
+                else {
+                    yield* next(output);
+                }
+            }
+        });
+        yield* provide({
+            *[Symbol.iterator]() {
+                let outputs = yield* output;
+                while (buffer.size > 0 || requests.length > 0) {
+                    yield* outputs.next();
+                }
+            },
+            *spawn(fn) {
+                let { operation, resolve } = withResolvers();
+                requests.unshift({
+                    operation: fn,
+                    resolve: resolve,
+                });
+                yield* input.send();
+                return operation;
+            },
+        });
+    });
+}
+function* next(stream) {
+    let subscription = yield* stream;
+    return yield* subscription.next();
+}
+//# sourceMappingURL=task-buffer.js.map

package/esm/task-buffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-buffer.js","sourceRoot":"","sources":["../src/task-buffer.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,EACb,GAAG,EACH,EAAE,EAGF,QAAQ,EAER,KAAK,EAGL,QAAQ,EACR,aAAa,GACd,MAAM,WAAW,CAAC;AAMnB,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,OAAO,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO;QAChC,IAAI,KAAK,GAAG,aAAa,EAAe,CAAC;QAEzC,IAAI,MAAM,GAAG,aAAa,EAA0B,CAAC;QAErD,IAAI,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;QAEtC,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;QAE9B,IAAI,QAAQ,GAA4B,EAAE,CAAC;QAE3C,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;YACpB,OAAO,IAAI,EAAE,CAAC;gBACZ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;qBAAM,IAAI,MAAM,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC;oBAC7B,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,EAAG,CAAC;oBAC9B,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;oBACjD,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjB,KAAK,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;wBACpB,IAAI,CAAC;4BACH,IAAI,MAAM,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;4BAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;4BACpB,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC7B,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;4BACpB,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAc,CAAC,CAAC,CAAC;wBAC1C,CAAC;oBACH,CAAC,CAAC,CAAC;oBACH,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,CAAC,OAAO,CAAC;YACb,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChB,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC;gBAC5B,OAAO,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9C,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,CAAC,KAAK,CAAI,EAAsB;gBAC9B,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,aAAa,EAAW,CAAC;gBACtD,QAAQ,CAAC,OAAO,CAAC;oBACf,SAAS,EAAE,EAAE;oBACb,OAAO,EAAE,OAA2B;iBACrC,CAAC,CAAC;gBACH,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBACpB,OAAO,SAAS,CAAC;YACnB,CAAC;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAOD,QAAQ,CAAC,CAAC,IAAI,CACZ,MAAyB;IAEzB,IAAI,YAAY,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC;IACjC,OAAO,KAAK,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;AACpC,CAAC"}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "staticalize",
+  "version": "0.2.2",
+  "description": "Download a dynamic website and turn it into a static site",
+  "author": "engineering@frontside.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/thefrontside/statical.git"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/thefrontside/statical/issues"
+  },
+  "module": "./esm/mod.js",
+  "exports": {
+    ".": {
+      "import": "./esm/mod.js"
+    }
+  },
+  "scripts": {},
+  "bin": {
+    "staticalize": "./esm/main.js"
+  },
+  "engines": {
+    "node": ">= 18"
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "configliere": "^0.2.2",
+    "effection": "4.0.0-beta.2",
+    "hast-util-from-html": "^2.0.3",
+    "hast-util-select": "^6.0.4",
+    "hast-util-to-html": "^9.0.5",
+    "zod": "^4.1.11",
+    "@deno/shim-deno": "~0.18.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.9.0"
+  },
+  "_generatedBy": "dnt@dev"
+}

package/src/_dnt.polyfills.ts

@@ -0,0 +1,283 @@
+declare global {
+  // https://github.com/microsoft/TypeScript/blob/main/lib/lib.es2021.string.d.ts
+  interface String {
+    /**
+     * Replace all instances of a substring in a string, using a regular expression or search string.
+     * @param searchValue A string to search for.
+     * @param replaceValue A string containing the text to replace for every successful match of searchValue in this string.
+     */
+    replaceAll(searchValue: string | RegExp, replaceValue: string): string;
+
+    /**
+     * Replace all instances of a substring in a string, using a regular expression or search string.
+     * @param searchValue A string to search for.
+     * @param replacer A function that returns the replacement text.
+     */
+    replaceAll(
+      searchValue: string | RegExp,
+      replacer: (substring: string, ...args: any[]) => string,
+    ): string;
+  }
+}
+
+/**
+ * String.prototype.replaceAll() polyfill
+ * https://gomakethings.com/how-to-replace-a-section-of-a-string-with-another-one-with-vanilla-js/
+ * @author Chris Ferdinandi
+ * @license MIT
+ */
+if (!String.prototype.replaceAll) {
+  String.prototype.replaceAll = function (str: any, newStr: any) {
+    // If a regex pattern
+    if (
+      Object.prototype.toString.call(str).toLowerCase() === "[object regexp]"
+    ) {
+      return this.replace(str, newStr);
+    }
+
+    // If a string
+    return this.replace(new RegExp(str, "g"), newStr);
+  };
+}
+
+export {};
+/**
+ * Based on [import-meta-ponyfill](https://github.com/gaubee/import-meta-ponyfill),
+ * but instead of using npm to install additional dependencies,
+ * this approach manually consolidates cjs/mjs/d.ts into a single file.
+ *
+ * Note that this code might be imported multiple times
+ * (for example, both dnt.test.polyfills.ts and dnt.polyfills.ts contain this code;
+ *  or Node.js might dynamically clear the cache and then force a require).
+ * Therefore, it's important to avoid redundant writes to global objects.
+ * Additionally, consider that commonjs is used alongside esm,
+ * so the two ponyfill functions are stored independently in two separate global objects.
+ */
+//@ts-ignore
+import { createRequire } from "node:module";
+//@ts-ignore
+import { fileURLToPath, pathToFileURL, type URL } from "node:url";
+//@ts-ignore
+import { dirname } from "node:path";
+declare global {
+  interface ImportMeta {
+    /** A string representation of the fully qualified module URL. When the
+     * module is loaded locally, the value will be a file URL (e.g.
+     * `file:///path/module.ts`).
+     *
+     * You can also parse the string as a URL to determine more information about
+     * how the current module was loaded. For example to determine if a module was
+     * local or not:
+     *
+     * ```ts
+     * const url = new URL(import.meta.url);
+     * if (url.protocol === "file:") {
+     *   console.log("this module was loaded locally");
+     * }
+     * ```
+     */
+    url: string;
+    /**
+     * A function that returns resolved specifier as if it would be imported
+     * using `import(specifier)`.
+     *
+     * ```ts
+     * console.log(import.meta.resolve("./foo.js"));
+     * // file:///dev/foo.js
+     * ```
+     *
+     * @param specifier The module specifier to resolve relative to `parent`.
+     * @param parent The absolute parent module URL to resolve from.
+     * @returns The absolute (`file:`) URL string for the resolved module.
+     */
+    resolve(specifier: string, parent?: string | URL | undefined): string;
+    /** A flag that indicates if the current module is the main module that was
+     * called when starting the program under Deno.
+     *
+     * ```ts
+     * if (import.meta.main) {
+     *   // this was loaded as the main module, maybe do some bootstrapping
+     * }
+     * ```
+     */
+    main: boolean;
+
+    /** The absolute path of the current module.
+     *
+     * This property is only provided for local modules (ie. using `file://` URLs).
+     *
+     * Example:
+     * ```
+     * // Unix
+     * console.log(import.meta.filename); // /home/alice/my_module.ts
+     *
+     * // Windows
+     * console.log(import.meta.filename); // C:\alice\my_module.ts
+     * ```
+     */
+    filename: string;
+
+    /** The absolute path of the directory containing the current module.
+     *
+     * This property is only provided for local modules (ie. using `file://` URLs).
+     *
+     * * Example:
+     * ```
+     * // Unix
+     * console.log(import.meta.dirname); // /home/alice
+     *
+     * // Windows
+     * console.log(import.meta.dirname); // C:\alice
+     * ```
+     */
+    dirname: string;
+  }
+}
+
+type NodeRequest = ReturnType<typeof createRequire>;
+type NodeModule = NonNullable<NodeRequest["main"]>;
+interface ImportMetaPonyfillCommonjs {
+  (require: NodeRequest, module: NodeModule): ImportMeta;
+}
+interface ImportMetaPonyfillEsmodule {
+  (importMeta: ImportMeta): ImportMeta;
+}
+interface ImportMetaPonyfill
+  extends ImportMetaPonyfillCommonjs, ImportMetaPonyfillEsmodule {
+}
+
+const defineGlobalPonyfill = (symbolFor: string, fn: Function) => {
+  if (!Reflect.has(globalThis, Symbol.for(symbolFor))) {
+    Object.defineProperty(
+      globalThis,
+      Symbol.for(symbolFor),
+      {
+        configurable: true,
+        get() {
+          return fn;
+        },
+      },
+    );
+  }
+};
+
+export let import_meta_ponyfill_commonjs = (
+  Reflect.get(globalThis, Symbol.for("import-meta-ponyfill-commonjs")) ??
+    (() => {
+      const moduleImportMetaWM = new WeakMap<NodeModule, ImportMeta>();
+      return (require, module) => {
+        let importMetaCache = moduleImportMetaWM.get(module);
+        if (importMetaCache == null) {
+          const importMeta = Object.assign(Object.create(null), {
+            url: pathToFileURL(module.filename).href,
+            main: require.main == module,
+            resolve: (specifier: string, parentURL = importMeta.url) => {
+              return pathToFileURL(
+                (importMeta.url === parentURL
+                  ? require
+                  : createRequire(parentURL))
+                  .resolve(specifier),
+              ).href;
+            },
+            filename: module.filename,
+            dirname: module.path,
+          });
+          moduleImportMetaWM.set(module, importMeta);
+          importMetaCache = importMeta;
+        }
+        return importMetaCache;
+      };
+    })()
+) as ImportMetaPonyfillCommonjs;
+defineGlobalPonyfill(
+  "import-meta-ponyfill-commonjs",
+  import_meta_ponyfill_commonjs,
+);
+
+export let import_meta_ponyfill_esmodule = (
+  Reflect.get(globalThis, Symbol.for("import-meta-ponyfill-esmodule")) ??
+    ((importMeta: ImportMeta) => {
+      const resolveFunStr = String(importMeta.resolve);
+      const shimWs = new WeakSet();
+      //@ts-ignore
+      const mainUrl = ("file:///" + process.argv[1].replace(/\\/g, "/"))
+        .replace(
+          /\/{3,}/,
+          "///",
+        );
+      const commonShim = (importMeta: ImportMeta) => {
+        if (typeof importMeta.main !== "boolean") {
+          importMeta.main = importMeta.url === mainUrl;
+        }
+        if (typeof importMeta.filename !== "string") {
+          importMeta.filename = fileURLToPath(importMeta.url);
+          importMeta.dirname = dirname(importMeta.filename);
+        }
+      };
+      if (
+        // v16.2.0+, v14.18.0+: Add support for WHATWG URL object to parentURL parameter.
+        resolveFunStr === "undefined" ||
+        // v20.0.0+, v18.19.0+"" This API now returns a string synchronously instead of a Promise.
+        resolveFunStr.startsWith("async")
+        // enable by --experimental-import-meta-resolve flag
+      ) {
+        import_meta_ponyfill_esmodule = (importMeta: ImportMeta) => {
+          if (!shimWs.has(importMeta)) {
+            shimWs.add(importMeta);
+            const importMetaUrlRequire = {
+              url: importMeta.url,
+              require: createRequire(importMeta.url),
+            };
+            importMeta.resolve = function resolve(
+              specifier: string,
+              parentURL = importMeta.url,
+            ) {
+              return pathToFileURL(
+                (importMetaUrlRequire.url === parentURL
+                  ? importMetaUrlRequire.require
+                  : createRequire(parentURL)).resolve(specifier),
+              ).href;
+            };
+            commonShim(importMeta);
+          }
+          return importMeta;
+        };
+      } else {
+        /// native support
+        import_meta_ponyfill_esmodule = (importMeta: ImportMeta) => {
+          if (!shimWs.has(importMeta)) {
+            shimWs.add(importMeta);
+            commonShim(importMeta);
+          }
+          return importMeta;
+        };
+      }
+      return import_meta_ponyfill_esmodule(importMeta);
+    })
+) as ImportMetaPonyfillEsmodule;
+defineGlobalPonyfill(
+  "import-meta-ponyfill-esmodule",
+  import_meta_ponyfill_esmodule,
+);
+
+export let import_meta_ponyfill = (
+  (...args: any[]) => {
+    const _MODULE = (() => {
+      if (typeof require === "function" && typeof module === "object") {
+        return "commonjs";
+      } else {
+        // eval("typeof import.meta");
+        return "esmodule";
+      }
+    })();
+    if (_MODULE === "commonjs") {
+      //@ts-ignore
+      import_meta_ponyfill = (r, m) => import_meta_ponyfill_commonjs(r, m);
+    } else {
+      //@ts-ignore
+      import_meta_ponyfill = (im) => import_meta_ponyfill_esmodule(im);
+    }
+    //@ts-ignore
+    return import_meta_ponyfill(...args);
+  }
+) as ImportMetaPonyfill;

package/src/_dnt.shims.ts

@@ -0,0 +1,64 @@
+import { Deno } from "@deno/shim-deno";
+export { Deno } from "@deno/shim-deno";
+
+const dntGlobals = {
+  Deno,
+};
+export const dntGlobalThis = createMergeProxy(globalThis, dntGlobals);
+
+function createMergeProxy<T extends object, U extends object>(
+  baseObj: T,
+  extObj: U,
+): Omit<T, keyof U> & U {
+  return new Proxy(baseObj, {
+    get(_target, prop, _receiver) {
+      if (prop in extObj) {
+        return (extObj as any)[prop];
+      } else {
+        return (baseObj as any)[prop];
+      }
+    },
+    set(_target, prop, value) {
+      if (prop in extObj) {
+        delete (extObj as any)[prop];
+      }
+      (baseObj as any)[prop] = value;
+      return true;
+    },
+    deleteProperty(_target, prop) {
+      let success = false;
+      if (prop in extObj) {
+        delete (extObj as any)[prop];
+        success = true;
+      }
+      if (prop in baseObj) {
+        delete (baseObj as any)[prop];
+        success = true;
+      }
+      return success;
+    },
+    ownKeys(_target) {
+      const baseKeys = Reflect.ownKeys(baseObj);
+      const extKeys = Reflect.ownKeys(extObj);
+      const extKeysSet = new Set(extKeys);
+      return [...baseKeys.filter((k) => !extKeysSet.has(k)), ...extKeys];
+    },
+    defineProperty(_target, prop, desc) {
+      if (prop in extObj) {
+        delete (extObj as any)[prop];
+      }
+      Reflect.defineProperty(baseObj, prop, desc);
+      return true;
+    },
+    getOwnPropertyDescriptor(_target, prop) {
+      if (prop in extObj) {
+        return Reflect.getOwnPropertyDescriptor(extObj, prop);
+      } else {
+        return Reflect.getOwnPropertyDescriptor(baseObj, prop);
+      }
+    },
+    has(_target, prop) {
+      return prop in extObj || prop in baseObj;
+    },
+  }) as any;
+}

package/src/config.ts

@@ -0,0 +1,28 @@
+import { cli, field, object, program } from "configliere";
+import z from "zod";
+import denoJSON from "./deno.js";
+
+const url = () =>
+  z.string().refine((str) => str.match(/^http/), {
+    message: `must be a hypertext (http) url`,
+  });
+
+export const config = program({
+  name: "staticalize",
+  version: denoJSON.version,
+  config: object({
+    site: {
+      description:
+        "URL of the website to staticalize. E.g. http://localhost:8000",
+      ...field(url(), cli.argument()),
+    },
+    output: {
+      description: "Directory to place the downloaded site",
+      ...field(z.string(), field.default("dist")),
+    },
+    base: {
+      description: "Base URL of the public website. E.g. http://frontside.com",
+      ...field(url()),
+    },
+  }),
+});