sliftutils 0.1.1

package/bundler/bundleEntry.ts

@@ -0,0 +1,32 @@
+import path from "path";
+import { bundle } from "./bundler";
+import fs from "fs";
+
+function getCommandLineArg(name: string) {
+    let index = process.argv.indexOf(name);
+    if (index === -1) return null;
+    return process.argv[index + 1];
+}
+async function main() {
+    let entryPoint = getCommandLineArg("--entryPoint");
+    if (!entryPoint) {
+        throw new Error("No entry point provided");
+    }
+    require("../" + entryPoint);
+
+    let name = path.basename(entryPoint);
+    if (name.endsWith(".ts") || name.endsWith(".tsx")) {
+        name = name.split(".").slice(0, -1).join(".");
+    }
+    name += ".js";
+
+    let modules = Object.values(require.cache).filter(x => x?.id !== module.id);
+
+    let bundled = await bundle({
+        modules,
+        rootPath: path.resolve("."),
+        entryPoints: [entryPoint],
+    });
+    await fs.promises.writeFile(`./build-extension/${name}`, bundled.bundle);
+}
+main().catch(console.error).finally(() => process.exit());

package/bundler/bundleEntryCaller.ts

@@ -0,0 +1,8 @@
+import child_process from "child_process";
+import { runPromise } from "socket-function/src/runPromise";
+
+
+export async function bundleEntryCaller(entryPoint: string, args = "") {
+    let command = "yarn typenode ./bundler/bundleEntry.ts --entryPoint " + entryPoint + " " + args;
+    await runPromise(command);
+}

package/bundler/bundleRequire.ts

@@ -0,0 +1,244 @@
+import { SerializedModule } from "./bundleWrapper";
+
+// Sets globalThis.require = ..., utilizing registeredModules (from bundleWrapper.ts) when require is called
+export interface BundleRequireConfig {
+    rootPath: string;
+}
+export function bundleRequire(config: BundleRequireConfig) {
+    globalThis.process = globalThis.process || {
+        env: {
+            NODE_ENV: "production",
+        },
+        versions: {},
+        on: () => { },
+    };
+    (globalThis as any).window = (globalThis as any).window || globalThis;
+    (globalThis as any).global = (globalThis as any).global || globalThis;
+    (globalThis as any).setImmediate = (globalThis as any).setImmediate || globalThis.setTimeout;
+
+    // Set document, so isNode checks return false.
+    (globalThis as any).document = (globalThis as any).document || {};
+    (globalThis as any).BOOTED_EDGE_NODE = undefined;
+
+    const builtInModuleExports = {
+        worker_threads: {
+            isMainThread: true,
+        },
+        util: {
+            // https://nodejs.org/api/util.html#util_util_inherits_constructor_superconstructor
+            inherits(constructor: any, superConstructor: any) {
+                Object.setPrototypeOf(constructor.prototype, superConstructor.prototype);
+            },
+            TextDecoder: TextDecoder,
+            TextEncoder: TextEncoder,
+        },
+        buffer: { Buffer },
+        stream: {
+            // HACK: Needed to get SAX JS to work correctly.
+            Stream: function () { },
+            Transform: function () { },
+
+            Writable: function () { },
+        },
+        timers: {
+            // TODO: Add all members of timers
+            setImmediate: globalThis.setImmediate,
+        },
+        child_process: {},
+        events: class EventEmitter { },
+    };
+
+    // Just path.resolve (but needs to be reimplemented, because we can't use imports)
+    function pathResolve(...paths: string[]): string {
+        // Start with empty path segments array
+        let segments: string[] = [];
+        let isWindowsPath = false;
+
+        paths = paths.map(x => x.replace(/\\/g, "/"));
+
+        // Process each path argument
+        for (const path of paths) {
+            // Check for Windows drive letter (e.g., C:/)
+            if (/^[A-Za-z]:/.test(path)) {
+                isWindowsPath = true;
+                // Remove drive letter for processing
+                const withoutDrive = path.slice(2);
+                if (withoutDrive.startsWith("/")) {
+                    segments = [path.slice(0, 2)]; // Keep drive letter and reset segments
+                } else {
+                    // If no leading slash, keep current segments (relative to current drive path)
+                    if (segments.length === 0 || !segments[0].match(/^[A-Za-z]:/)) {
+                        segments = [path.slice(0, 2)];
+                    }
+                }
+                // Add the rest of the path parts
+                segments.push(...withoutDrive.split("/").filter(x => x));
+                continue;
+            }
+
+            // If absolute path, reset segments but keep drive letter if present
+            if (path.startsWith("/")) {
+                if (isWindowsPath && segments.length > 0 && segments[0].match(/^[A-Za-z]:/)) {
+                    const drive = segments[0];
+                    segments = [drive];
+                } else {
+                    segments = [];
+                }
+            }
+
+            // Split path into segments and process each
+            const pathParts = path.split("/").filter(x => x);
+            for (const part of pathParts) {
+                if (part === "..") {
+                    // Don't pop off the drive letter
+                    if (segments.length > (isWindowsPath ? 1 : 0)) {
+                        segments.pop();
+                    }
+                } else if (part !== ".") {
+                    // Add segment if not current directory marker
+                    segments.push(part);
+                }
+            }
+        }
+
+        // Combine segments into final path
+        let result = segments.join("/");
+        if (!isWindowsPath) {
+            result = "/" + result;
+        }
+        return result;
+    }
+    function dirname(path: string): string {
+        return path.split("/").slice(0, -1).join("/");
+    }
+
+    const requireCache: { [id: string]: NodeJS.Module } = {};
+
+    let rootModule = createModule({
+        parentModule: undefined,
+        resolveAbsolutePath: config.rootPath + "/rootPlaceholder",
+    });
+    globalThis.require = rootModule.require as any;
+
+    function createModule(config: {
+        parentModule: NodeJS.Module | undefined;
+        resolveAbsolutePath: string;
+    }): NodeJS.Module {
+        const { parentModule, resolveAbsolutePath } = config;
+        let cached = requireCache[resolveAbsolutePath];
+        if (cached) {
+            return cached;
+        }
+        let serialized = globalThis.registeredModules?.[resolveAbsolutePath];
+
+        let newModule: NodeJS.Module = {
+            id: resolveAbsolutePath,
+            filename: resolveAbsolutePath,
+            exports: {},
+            parent: parentModule,
+            children: [],
+            isPreloading: false,
+            loaded: false,
+            path: dirname(resolveAbsolutePath),
+            paths: serialized?.paths || [],
+            require,
+            load,
+        } as any;
+        newModule.exports.default = newModule.exports;
+        if (parentModule) {
+            parentModule.children.push(newModule);
+        }
+        for (let [key, value] of Object.entries(serialized?.moduleFields || {})) {
+            if (key in newModule) continue;
+            (newModule as any)[key] = value;
+        }
+        resolve.paths = (request: string) => [];
+
+        requireCache[newModule.id] = newModule;
+        require.resolve = resolve;
+        require.cache = requireCache;
+        require.main = newModule;
+        require.extensions = "extension not implemented yet" as any;
+
+        // Resolves file extensions
+        function innerResolve(path: string): string {
+            let candidates = [
+                path,
+                path + ".js",
+                path + ".ts",
+                path + ".tsx",
+            ];
+            for (let candidate of candidates) {
+                let registered = globalThis.registeredModules?.[candidate];
+                if (registered) {
+                    return registered.id;
+                }
+            }
+            return path;
+        }
+
+        function resolve(path: string): string {
+            path = path.replace(/\\/g, "/");
+            if (path.startsWith(".")) {
+                return innerResolve(pathResolve(newModule.path, path));
+            }
+            // We need to search all paths
+            for (let searchRoot of serialized?.paths || []) {
+                let candidate = innerResolve(pathResolve(searchRoot, path));
+                let registered = globalThis.registeredModules?.[candidate];
+                if (registered) {
+                    return registered.id;
+                }
+            }
+            // It is probably "fs" or something like that
+            return path;
+            // debugger;
+            // throw new Error(`Module ${path} not found`);
+        }
+
+        function require(path: string) {
+            if (path in builtInModuleExports) {
+                return builtInModuleExports[path as keyof typeof builtInModuleExports];
+            }
+            let resolved = resolve(path);
+            let subModule = createModule({
+                parentModule: newModule,
+                resolveAbsolutePath: resolved,
+            }) as any;
+            subModule.load(newModule.filename);
+            return subModule.exports;
+        }
+
+        function load() {
+            if (newModule.loaded) return;
+            // NOTE: Set loaded immediately, in case we have a circular dependency
+            newModule.loaded = true;
+
+            if (serialized) {
+                serialized.moduleFnc(newModule.exports, require, newModule, newModule.filename, newModule.path);
+            } else {
+                // If we are being imported by the root module, we need to throw an error
+                if (!config.parentModule?.parent) {
+                    debugger;
+                    throw new Error(`Could not find required module ${JSON.stringify(config.resolveAbsolutePath)}, have ${JSON.stringify(Object.keys(globalThis.registeredModules || {}))}`);
+                }
+                newModule.exports = new Proxy(
+                    {},
+                    {
+                        get(target, property) {
+                            if (property === "__esModule") return undefined;
+                            if (property === "default") return newModule.exports;
+
+                            console.warn(
+                                `Module ${newModule.filename} is not available (are you missing "module.allowclient = true"?). Tried to access ${String(property)}.`
+                            );
+                            return undefined;
+                        },
+                    }
+                );
+            }
+        }
+
+        return newModule;
+    }
+}

package/bundler/bundleWrapper.ts

@@ -0,0 +1,115 @@
+import { cache } from "socket-function/src/caching";
+import path from "path";
+import fs from "fs";
+
+const getPackageJsonPath = cache((directory: string): string | undefined => {
+    if (!directory.includes("/") && !directory.includes("\\")) {
+        return undefined;
+    }
+    let packageJsonPath = path.resolve(directory, "package.json");
+    if (fs.existsSync(packageJsonPath)) {
+        return packageJsonPath;
+    }
+    return getPackageJsonPath(path.dirname(directory));
+});
+const getMainPath = cache((directory: string): string | undefined => {
+    let packageJsonPath = getPackageJsonPath(directory);
+    if (!packageJsonPath) return undefined;
+    let packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
+    let dir = path.dirname(packageJsonPath);
+    let mainName = packageJson.main;
+    if (!mainName) {
+        if (fs.existsSync(path.resolve(dir, "index.js"))) {
+            mainName = "index.js";
+        } else if (fs.existsSync(path.resolve(dir, "index.ts"))) {
+            mainName = "index.ts";
+        } else if (fs.existsSync(path.resolve(dir, "index.tsx"))) {
+            mainName = "index.tsx";
+        } else {
+            mainName = "index.js";
+        }
+    }
+    let mainPath = path.resolve(dir, mainName);
+    return mainPath;
+});
+
+
+// Wraps the module so it registers itself when the returned code is evaluated
+//  - See https://nodejs.org/api/modules.html#the-module-wrapper
+export function wrapModule(module: NodeJS.Module): string {
+    let contents = (module as any).moduleContents || "/* No contents */";
+
+    // NOTE: debugName only matters during module evaluation. After that the sourcemap should work.
+    let debugName = module.filename
+        .replace(/\\/g, "/")
+        .split("/")
+        .slice(-1)[0]
+        .replace(/\./g, "_")
+        .replace(/[^a-zA-Z_]/g, "");
+
+    let wrapped = `(function ${debugName}(exports, require, module, __filename, __dirname) { ${contents}
+    })`;
+
+    let moduleFields: { [flag: string]: unknown; } = {};
+    for (let [key, value] of Object.entries(module)) {
+        if (typeof value === "function") continue;
+        if (typeof value === "boolean") {
+            moduleFields[key] = value;
+        } else if (typeof value === "string" && value.length < 150) {
+            moduleFields[key] = value;
+        } else if (typeof value === "number") {
+            moduleFields[key] = value;
+        }
+    }
+
+    let isModuleMain: string | undefined;
+    let dirname = path.dirname(module.filename);
+    let packageJsonPath = getPackageJsonPath(dirname);
+    if (packageJsonPath) {
+        let mainPath = getMainPath(dirname);
+        if (mainPath?.replaceAll("\\", "/") === module.filename.replaceAll("\\", "/")) {
+            // Then we are the main of the module
+            isModuleMain = path.dirname(packageJsonPath).replaceAll("\\", "/");
+        }
+    }
+
+    // NOTE: We can't have new lines, or they break source maps
+    let objWrapped = `{`
+        + ` id: ${JSON.stringify(module.id.replaceAll("\\", "/"))},`
+        + ` filename: ${JSON.stringify(module.filename.replaceAll("\\", "/"))},`
+        + ` isModuleMain: ${JSON.stringify(isModuleMain)},`
+        + ` paths: ${JSON.stringify(module.paths.map(p => p.replaceAll("\\", "/")))},`
+        + ` moduleFields: ${JSON.stringify(moduleFields)},`
+        + ` moduleFnc: ${wrapped}`
+        + ` }`;
+
+    function initModule(serialized: SerializedModule) {
+        globalThis.registeredModules = globalThis.registeredModules || {};
+        globalThis.registeredModules[serialized.id] = serialized;
+        if (serialized.isModuleMain) {
+            globalThis.registeredModules[serialized.isModuleMain] = serialized;
+        }
+    }
+
+    return `;(${initModule.toString().replaceAll("\n", " ")})(${objWrapped});`;
+}
+
+declare global {
+    var registeredModules: { [id: string]: SerializedModule; } | undefined;
+}
+
+export interface SerializedModule {
+    id: string;
+    filename: string;
+
+    // The main main we represent. Ex, "/ai3/node_modules/typesafecss"
+    isModuleMain?: string;
+
+    // Paths which the require function searches for non-relative imports
+    paths: string[];
+
+    // Fields to be set on the module, when it is created
+    moduleFields: { [flag: string]: unknown; };
+
+    moduleFnc: (exports: any, require: any, module: any, __filename: string, __dirname: string) => unknown;
+}

package/bundler/bundler.ts

@@ -0,0 +1,72 @@
+import { sha256 } from "js-sha256";
+import { bundleRequire, BundleRequireConfig } from "./bundleRequire";
+import { wrapModule } from "./bundleWrapper";
+import { addToInProgressSourceMap, encodeSourceMapLineComment, finalizeInProgressSourceMap, getInProgressSourceMap, InProgressSourceMap, removeSourceMap } from "./sourceMaps";
+import fs from "fs";
+
+export async function bundle(config: {
+    modules: (typeof require.cache[""])[];
+    rootPath: string;
+    entryPoints: string[];
+}): Promise<{
+    bundle: string;
+}> {
+    const { modules, rootPath, entryPoints } = config;
+
+    // NOTE: We COULD use an "index source map", which contains other sourcemaps
+    //  and gives offsets for them. However... tooling support will is better
+    //  for regular sourcemaps, and it's more flexible.
+
+    let inProgressSourceMap: InProgressSourceMap = {
+        sources: [],
+        mappings: [],
+    };
+
+    let code = "";
+    let curLineCount = 0;
+    for (let module of modules) {
+        if (!module) continue;
+
+        let newCode = wrapModule(module);
+
+        let { sourceMap, code: newCode2 } = removeSourceMap(newCode);
+        newCode = newCode2;
+        if (sourceMap) {
+            let inProgress = getInProgressSourceMap(sourceMap);
+            for (let mapping of inProgress.mappings) {
+                mapping.generatedLine += curLineCount;
+            }
+            addToInProgressSourceMap(inProgressSourceMap, inProgress);
+        }
+
+        code += newCode + "\n";
+        curLineCount += (newCode.match(/\n/g) || []).length + 1;
+    }
+    code += "\n/* Inlined buffer implementation: */\n";
+    code += `\n;\n${fs.readFileSync(__dirname + "/buffer.js").toString()}\n;\n`;
+    code += `\n;globalThis.__BUNDLE_HASH__ = ${JSON.stringify(sha256(code))};`;
+    let bundleConfig: BundleRequireConfig = {
+        rootPath,
+    };
+    code += `;(${bundleRequire.toString()})(${JSON.stringify(bundleConfig)});`;
+    // Delay the initial requires, so our extension can boot and we can debug startup errors
+    code += "\n;setTimeout(() => {";
+    for (let entryPoint of entryPoints) {
+        code += `\n;globalThis.require(${JSON.stringify(entryPoint)});`;
+    }
+    code += "\n;});";
+    code += "\n" + encodeSourceMapLineComment(finalizeInProgressSourceMap(inProgressSourceMap));
+    return {
+        bundle: code,
+    };
+}
+
+declare global {
+    var __BUNDLE_HASH__: string | undefined;
+}
+
+export function extractBundleHash(code: string) {
+    let match = code.match(/;globalThis.__BUNDLE_HASH__ = "([^"]+)";/);
+    if (!match) return undefined;
+    return match[1];
+}