wasm-idle 0.0.1

package/README.md

@@ -0,0 +1,9 @@
+## wasm-idle
+
+![wasm-idle](static/image.jpeg)
+
+Executes C++, Python Code with working stdio.
+
+Refer to src/lib/clang.
+
+Powered by [wasm-clang](https://github.com/binji/wasm-clang) and Pyodide.

package/dist/clang/app.d.ts

@@ -0,0 +1,24 @@
+import { Memory, type MemFS } from "./memory";
+export default class App {
+    ready: Promise<void>;
+    mem: Memory;
+    memfs: MemFS;
+    instance: WebAssembly.Instance;
+    exports: any;
+    argv: string[];
+    environ: {
+        [key: string]: string;
+    };
+    handles: Map<number, any>;
+    nextHandle: number;
+    constructor(module: WebAssembly.Module, memfs: MemFS, name: string, ...args: string[]);
+    run(): Promise<boolean | undefined>;
+    proc_exit(code: number): void;
+    environ_sizes_get(environ_count_out: number, environ_buf_size_out: number): number;
+    environ_get(environ_ptrs: number, environ_buf: number): number;
+    args_sizes_get(argc_out: number, argv_buf_size_out: number): number;
+    args_get(argv_ptrs: number, argv_buf: number): number;
+    random_get(buf: number, buf_len: number): void;
+    clock_time_get(): void;
+    poll_oneoff(): void;
+}

package/dist/clang/app.js

@@ -0,0 +1,119 @@
+import { bindNew } from "./apply";
+import { Memory } from "./memory";
+import { getInstance } from "./wasm";
+import { NotImplemented, ProcExit } from "./error";
+const ESUCCESS = 0;
+const RAF_PROC_EXIT_CODE = 0xC0C0A;
+export default class App {
+    ready;
+    mem = null;
+    memfs;
+    instance = null;
+    exports;
+    argv;
+    environ;
+    handles = new Map();
+    nextHandle = 0;
+    constructor(module, memfs, name, ...args) {
+        this.argv = [name, ...args];
+        this.environ = { USER: 'jungol' };
+        this.memfs = memfs;
+        const env = bindNew(this);
+        const wasi_unstable = {
+            ...bindNew(this, 'proc_exit', 'environ_sizes_get', 'environ_get', 'args_sizes_get', 'args_get', 'random_get', 'clock_time_get', 'poll_oneoff'), ...this.memfs.exports
+        };
+        this.ready = getInstance(module, { wasi_unstable, env }).then(instance => {
+            this.instance = instance;
+            this.exports = this.instance.exports;
+            this.mem = new Memory(this.exports.memory);
+            this.memfs.hostMem = this.mem;
+        });
+    }
+    async run() {
+        await this.ready;
+        try {
+            this.exports._start();
+        }
+        catch (exn) {
+            let writeStack = true;
+            if (exn instanceof ProcExit) {
+                if (exn.code === RAF_PROC_EXIT_CODE) {
+                    console.log('Allowing rAF after exit.');
+                    return true;
+                }
+                // Don't allow rAF unless you return the right code.
+                console.log(`Disallowing rAF since exit code is ${exn.code}.`);
+                if (exn.code == 0)
+                    return false;
+                writeStack = false;
+            }
+            // Write error message.
+            let msg = `\x1b[91mError: ${exn.message}`;
+            if (writeStack)
+                msg = msg + `\n${exn.stack}`;
+            msg += '\x1b[0m\n';
+            this.memfs.stdout(msg);
+            // Propagate error.
+            throw exn;
+        }
+    }
+    proc_exit(code) {
+        throw new ProcExit(code);
+    }
+    environ_sizes_get(environ_count_out, environ_buf_size_out) {
+        this.mem.check();
+        let size = 0;
+        const names = Object.getOwnPropertyNames(this.environ);
+        for (const name of names) {
+            const value = this.environ[name];
+            // +2 to account for = and \0 in "name=value\0".
+            size += name.length + value.length + 2;
+        }
+        this.mem.write64(environ_count_out, names.length);
+        this.mem.write64(environ_buf_size_out, size);
+        return ESUCCESS;
+    }
+    environ_get(environ_ptrs, environ_buf) {
+        this.mem.check();
+        const names = Object.getOwnPropertyNames(this.environ);
+        for (const name of names) {
+            this.mem.write32(environ_ptrs, environ_buf);
+            environ_ptrs += 4;
+            environ_buf +=
+                this.mem.writeStr(environ_buf, `${name}=${this.environ[name]}`);
+        }
+        this.mem.write32(environ_ptrs, 0);
+        return ESUCCESS;
+    }
+    args_sizes_get(argc_out, argv_buf_size_out) {
+        this.mem.check();
+        let size = 0;
+        for (let arg of this.argv) {
+            size += arg.length + 1; // "arg\0".
+        }
+        this.mem.write64(argc_out, this.argv.length);
+        this.mem.write64(argv_buf_size_out, size);
+        return ESUCCESS;
+    }
+    args_get(argv_ptrs, argv_buf) {
+        this.mem.check();
+        for (let arg of this.argv) {
+            this.mem.write32(argv_ptrs, argv_buf);
+            argv_ptrs += 4;
+            argv_buf += this.mem.writeStr(argv_buf, arg);
+        }
+        this.mem.write32(argv_ptrs, 0);
+        return ESUCCESS;
+    }
+    random_get(buf, buf_len) {
+        const data = new Uint8Array(this.mem.buffer, buf, buf_len);
+        for (let i = 0; i < buf_len; ++i)
+            data[i] = (Math.random() * 256) | 0;
+    }
+    clock_time_get() {
+        throw new NotImplemented('wasi_unstable', 'clock_time_get');
+    }
+    poll_oneoff() {
+        throw new NotImplemented('wasi_unstable', 'poll_oneoff');
+    }
+}

package/dist/clang/apply.d.ts

@@ -0,0 +1,2 @@
+export declare function bindNew(obj: any, ...names: string[]): any;
+export declare function bind(obj: any, ...names: string[]): any;

package/dist/clang/apply.js

@@ -0,0 +1,11 @@
+export function bindNew(obj, ...names) {
+    const result = {};
+    for (let name of names)
+        result[name] = (obj[name] || (() => 0)).bind(obj);
+    return result;
+}
+export function bind(obj, ...names) {
+    for (let name of names)
+        obj[name] = obj[name].bind(obj);
+    return obj;
+}

package/dist/clang/color.d.ts

@@ -0,0 +1,3 @@
+export declare const green = "\u001B[92m";
+export declare const normal = "\u001B[0m";
+export declare const yellow = "\u001B[1;93m";

package/dist/clang/color.js

@@ -0,0 +1,3 @@
+export const green = '\x1b[92m';
+export const normal = '\x1b[0m';
+export const yellow = '\x1b[1;93m';

package/dist/clang/encode.d.ts

@@ -0,0 +1,3 @@
+export declare function readStr(u8: Uint8Array, o: number, len?: number): string;
+export declare function readOct(u8: Uint8Array, o: number, len: number): number;
+export declare function readUint8(u8: Uint8Array, o: number, len: number): Uint8Array;

package/dist/clang/encode.js

@@ -0,0 +1,13 @@
+export function readStr(u8, o, len = -1) {
+    const end = len === -1 ? u8.length : o + len;
+    let str = '';
+    for (let i = o; i < end && u8[i]; ++i)
+        str += String.fromCharCode(u8[i]);
+    return str;
+}
+export function readOct(u8, o, len) {
+    return parseInt(readStr(u8, o, len), 8);
+}
+export function readUint8(u8, o, len) {
+    return new Uint8Array(u8.buffer, o, len);
+}

package/dist/clang/error.d.ts

@@ -0,0 +1,14 @@
+export declare class ProcExit extends Error {
+    code: number;
+    constructor(code: number);
+}
+export declare class NotImplemented extends Error {
+    constructor(modname: any, fieldname: any);
+}
+export declare class AbortError extends Error {
+    constructor(msg?: string);
+}
+export declare class AssertError extends Error {
+    constructor(msg: string);
+}
+export declare function assert(cond: any): void;

package/dist/clang/error.js

@@ -0,0 +1,27 @@
+export class ProcExit extends Error {
+    code;
+    constructor(code) {
+        super(`process exited with code ${code}.`);
+        this.code = code;
+    }
+}
+export class NotImplemented extends Error {
+    constructor(modname, fieldname) {
+        super(`${modname}.${fieldname} not implemented.`);
+    }
+}
+export class AbortError extends Error {
+    constructor(msg = 'abort') {
+        super(msg);
+    }
+}
+export class AssertError extends Error {
+    constructor(msg) {
+        super(msg);
+    }
+}
+export function assert(cond) {
+    if (!cond) {
+        throw new AssertError('assertion failed.');
+    }
+}

package/dist/clang/index.d.ts

@@ -0,0 +1,30 @@
+import App from "./app";
+import { MemFS } from "./memory";
+interface APIOption {
+    stdin: () => string;
+    stdout: (str: string) => void;
+    log?: boolean;
+    showTiming?: boolean;
+    path: string;
+}
+export default class Clang {
+    ready: Promise<void>;
+    memfs: MemFS;
+    stdout: (str: string) => void;
+    moduleCache: {
+        [key: string]: WebAssembly.Module;
+    };
+    showTiming: boolean;
+    log: boolean;
+    lastCode: string;
+    path: string;
+    constructor(options: APIOption);
+    hostLog(message: string): void;
+    hostLogAsync(message: string, promise: Promise<any>): Promise<any>;
+    getModule(name: string): Promise<any>;
+    compile(options: any): Promise<App | null>;
+    link(obj: string, wasm: string): Promise<App | null>;
+    run(module: WebAssembly.Module, out: boolean, ...args: string[]): Promise<App | null>;
+    compileLinkRun(code: string): Promise<App | null>;
+}
+export {};

package/dist/clang/index.js

@@ -0,0 +1,107 @@
+import App from "./app";
+import { MemFS, untar } from "./memory";
+import { green, yellow, normal } from "./color";
+import { compile, readBuffer } from "./wasm";
+import { clangUrl, lldUrl, rootUrl } from "./url";
+const clangCommonArgs = [
+    '-disable-free',
+    '-isysroot', '/',
+    '-internal-isystem', '/include/c++/v1',
+    '-internal-isystem', '/include',
+    '-internal-isystem', '/lib/clang/8.0.1/include',
+    '-ferror-limit', '19',
+    '-fmessage-length', '80',
+    '-fcolor-diagnostics',
+];
+export default class Clang {
+    ready;
+    memfs;
+    stdout;
+    moduleCache;
+    showTiming;
+    log;
+    lastCode = '';
+    path;
+    constructor(options) {
+        this.moduleCache = {};
+        this.stdout = options.stdout;
+        this.showTiming = options.showTiming || false;
+        this.log = options.log || false;
+        this.path = options.path;
+        this.memfs = new MemFS({
+            stdout: this.stdout,
+            stdin: options.stdin,
+            path: this.path
+        });
+        this.ready = this.memfs.ready.then(() => this.hostLogAsync(`Untarring ${rootUrl(this.path)}`, readBuffer(rootUrl(this.path)).then(buffer => untar(buffer, this.memfs))));
+    }
+    hostLog(message) {
+        if (!this.log)
+            return;
+        const yellowArrow = `${yellow}>${normal} `;
+        this.stdout(`${yellowArrow}${message}`);
+    }
+    async hostLogAsync(message, promise) {
+        const start = +new Date();
+        this.hostLog(`${message}...`);
+        const result = await promise;
+        const end = +new Date();
+        if (this.log)
+            this.stdout(' done.');
+        if (this.showTiming)
+            this.stdout(` ${green}(${end - start}ms)${normal}\n`);
+        if (this.log)
+            this.stdout('\n');
+        return result;
+    }
+    async getModule(name) {
+        if (this.moduleCache[name])
+            return this.moduleCache[name];
+        const module = await this.hostLogAsync(`Fetching and compiling ${name}`, compile(name));
+        this.moduleCache[name] = module;
+        return module;
+    }
+    async compile(options) {
+        const input = options.input;
+        const code = options.code;
+        const obj = options.obj;
+        const opt = options.opt || '2';
+        await this.ready;
+        this.memfs.addFile(input, code);
+        const clang = await this.getModule(clangUrl(this.path));
+        return await this.run(clang, this.log, 'clang', '-cc1', '-emit-obj', ...clangCommonArgs, '-O' + opt, '-o', obj, '-x', 'c++', input);
+    }
+    async link(obj, wasm) {
+        const stackSize = 1024 * 1024;
+        const libdir = 'lib/wasm32-wasi';
+        const crt1 = `${libdir}/crt1.o`;
+        await this.ready;
+        const lld = await this.getModule(lldUrl(this.path));
+        return await this.run(lld, this.log, 'wasm-ld', '--no-threads', '--export-dynamic', // TODO required?
+        '-z', `stack-size=${stackSize}`, `-L${libdir}`, crt1, obj, '-lc', '-lc++', '-lc++abi', '-lcanvas', '-o', wasm);
+    }
+    async run(module, out, ...args) {
+        this.memfs.out = out;
+        this.hostLog(`${args.join(' ')}\n`);
+        const start = +new Date();
+        const app = new App(module, this.memfs, args[0], ...args.slice(1));
+        const instantiate = +new Date();
+        const stillRunning = await app.run();
+        const end = +new Date();
+        if (this.log)
+            this.stdout('\n');
+        if (this.showTiming)
+            this.stdout(`${green}(${start - instantiate}ms/${end - instantiate}ms)${normal}\n`);
+        return stillRunning ? app : null;
+    }
+    async compileLinkRun(code) {
+        const input = `test.cc`, obj = `test.o`, wasm = `test.wasm`;
+        if (this.lastCode !== code) {
+            await this.compile({ input, code, obj });
+            await this.link(obj, wasm);
+        }
+        this.lastCode = code;
+        const testMod = await this.hostLogAsync(`Compiling ${wasm}`, WebAssembly.compile(this.memfs.getFileContents(wasm)));
+        return await this.run(testMod, true, wasm);
+    }
+}

package/dist/clang/memory/index.d.ts

@@ -0,0 +1,4 @@
+import MemFS from "./memFs";
+import Memory from "./memory";
+import untar from "./tar";
+export { MemFS, Memory, untar };

package/dist/clang/memory/index.js

@@ -0,0 +1,4 @@
+import MemFS from "./memFs";
+import Memory from "./memory";
+import untar from "./tar";
+export { MemFS, Memory, untar };

package/dist/clang/memory/memFs.d.ts

@@ -0,0 +1,31 @@
+import { Memory } from './';
+interface MemFsOptions {
+    stdin: () => string;
+    stdout: (str: string) => void;
+    stdinStr?: string;
+    path: string;
+}
+export default class MemFS {
+    ready: Promise<void>;
+    mem: Memory;
+    hostMem_: Memory;
+    stdinStr: string;
+    stdin: () => string;
+    stdout: (str: string) => void;
+    instance: WebAssembly.Instance;
+    exports: any;
+    out: boolean;
+    constructor(options: MemFsOptions);
+    set hostMem(mem: Memory);
+    setStdinStr(str: string): void;
+    addDirectory(path: string): void;
+    addFile(path: string, contents: string | ArrayBuffer | Uint8Array): void;
+    getFileContents(path: string): Uint8Array;
+    abort(): void;
+    host_write(fd: number, iovs: number, iovs_len: number, nwritten_out: number): number;
+    host_read(fd: number, iovs: number, iovs_len: number, nread: number): number;
+    memfs_log(buf: number, len: number): void;
+    copy_out(clang_dst: number, memfs_src: number, size: number): void;
+    copy_in(memfs_dst: number, clang_src: number, size: number): void;
+}
+export {};

package/dist/clang/memory/memFs.js

@@ -0,0 +1,121 @@
+import { Memory } from './';
+import { bindNew } from "../apply";
+import { AbortError, assert } from "../error";
+import { compile } from "../wasm";
+import { memfsUrl } from "../url";
+const ESUCCESS = 0;
+export default class MemFS {
+    ready;
+    mem = null;
+    hostMem_ = null;
+    stdinStr;
+    stdin;
+    stdout;
+    instance = null;
+    exports;
+    out = true;
+    constructor(options) {
+        this.stdin = options.stdin;
+        this.stdout = options.stdout;
+        this.stdinStr = options.stdinStr || "";
+        const env = bindNew(this, 'abort', 'host_write', 'host_read', 'memfs_log', 'copy_in', 'copy_out');
+        this.ready = compile(memfsUrl(options.path))
+            .then(module => WebAssembly.instantiate(module, { env }))
+            .then(instance => {
+            this.instance = instance;
+            this.exports = instance.exports;
+            this.mem = new Memory(this.exports.memory);
+            this.exports.init();
+        });
+    }
+    set hostMem(mem) {
+        this.hostMem_ = mem;
+    }
+    setStdinStr(str) {
+        this.stdinStr = str;
+    }
+    addDirectory(path) {
+        this.mem.check();
+        this.mem.write(this.exports.GetPathBuf(), path);
+        this.exports.AddDirectoryNode(path.length);
+    }
+    addFile(path, contents) {
+        const length = contents instanceof ArrayBuffer ? contents.byteLength : contents.length;
+        this.mem.check();
+        this.mem.write(this.exports.GetPathBuf(), path);
+        const inode = this.exports.AddFileNode(path.length, length);
+        const addr = this.exports.GetFileNodeAddress(inode);
+        this.mem.check();
+        this.mem.write(addr, contents);
+    }
+    getFileContents(path) {
+        this.mem.check();
+        this.mem.write(this.exports.GetPathBuf(), path);
+        const inode = this.exports.FindNode(path.length);
+        const addr = this.exports.GetFileNodeAddress(inode);
+        const size = this.exports.GetFileNodeSize(inode);
+        return new Uint8Array(this.mem.buffer, addr, size);
+    }
+    abort() {
+        throw new AbortError();
+    }
+    host_write(fd, iovs, iovs_len, nwritten_out) {
+        this.hostMem_.check();
+        assert(fd <= 2);
+        let size = 0;
+        let str = '';
+        for (let i = 0; i < iovs_len; ++i) {
+            const buf = this.hostMem_.read32(iovs);
+            iovs += 4;
+            const len = this.hostMem_.read32(iovs);
+            iovs += 4;
+            str += this.hostMem_.readStr(buf, len);
+            size += len;
+        }
+        this.hostMem_.write32(nwritten_out, size);
+        if (this.out)
+            this.stdout(str);
+        return ESUCCESS;
+    }
+    host_read(fd, iovs, iovs_len, nread) {
+        this.hostMem_.check();
+        assert(fd === 0);
+        let size = 0;
+        for (let i = 0; i < iovs_len; ++i) {
+            const buf = this.hostMem_.read32(iovs);
+            iovs += 4;
+            const len = this.hostMem_.read32(iovs);
+            iovs += 4;
+            if (!this.stdinStr.length)
+                this.stdinStr = this.stdin();
+            const lenToWrite = Math.min(len, this.stdinStr.length);
+            if (lenToWrite === 0)
+                break;
+            this.hostMem_.write(buf, this.stdinStr.substring(0, lenToWrite));
+            this.stdinStr = this.stdinStr.substring(lenToWrite);
+            size += lenToWrite;
+            if (lenToWrite !== len)
+                break;
+        }
+        this.hostMem_.write32(nread, size);
+        return ESUCCESS;
+    }
+    memfs_log(buf, len) {
+        this.mem.check();
+        console.log(this.mem.readStr(buf, len));
+    }
+    copy_out(clang_dst, memfs_src, size) {
+        this.hostMem_.check();
+        const dst = new Uint8Array(this.hostMem_.buffer, clang_dst, size);
+        this.mem.check();
+        const src = new Uint8Array(this.mem.buffer, memfs_src, size);
+        dst.set(src);
+    }
+    copy_in(memfs_dst, clang_src, size) {
+        this.mem.check();
+        const dst = new Uint8Array(this.mem.buffer, memfs_dst, size);
+        this.hostMem_.check();
+        const src = new Uint8Array(this.hostMem_.buffer, clang_src, size);
+        dst.set(src);
+    }
+}

package/dist/clang/memory/memory.d.ts

@@ -0,0 +1,17 @@
+export default class Memory {
+    memory: DataView;
+    buffer: ArrayBuffer;
+    u8: Uint8Array;
+    u32: Uint32Array;
+    constructor(memory: DataView);
+    check(): void;
+    read8(o: number): number;
+    read32(o: number): number;
+    readStr(o: number, len: number): string;
+    write8(o: number, v: number): void;
+    write32(o: number, v: number): void;
+    write64(o: number, vlo: number, vhi?: number): void;
+    writeStr(o: number, str: string): number;
+    writeUint8(o: number, arr: ArrayLike<number>): number;
+    write(o: number, buf: ArrayBuffer | string | Uint8Array): number;
+}

package/dist/clang/memory/memory.js

@@ -0,0 +1,56 @@
+import { readStr } from "../encode";
+export default class Memory {
+    memory;
+    buffer;
+    u8;
+    u32;
+    constructor(memory) {
+        this.memory = memory;
+        this.buffer = memory.buffer;
+        this.u8 = new Uint8Array(this.buffer);
+        this.u32 = new Uint32Array(this.buffer);
+    }
+    check() {
+        if (this.buffer.byteLength === 0) {
+            this.buffer = this.memory.buffer;
+            this.u8 = new Uint8Array(this.buffer);
+            this.u32 = new Uint32Array(this.buffer);
+        }
+    }
+    read8(o) {
+        return this.u8[o];
+    }
+    read32(o) {
+        return this.u32[o >> 2];
+    }
+    readStr(o, len) {
+        return readStr(this.u8, o, len);
+    }
+    write8(o, v) {
+        this.u8[o] = v;
+    }
+    write32(o, v) {
+        this.u32[o >> 2] = v;
+    }
+    write64(o, vlo, vhi = 0) {
+        this.write32(o, vlo);
+        this.write32(o + 4, vhi);
+    }
+    writeStr(o, str) {
+        o += this.write(o, str);
+        this.write8(o, 0);
+        return str.length + 1;
+    }
+    writeUint8(o, arr) {
+        new Uint8Array(this.buffer, o, arr.length).set(arr);
+        return arr.length;
+    }
+    write(o, buf) {
+        if (buf instanceof ArrayBuffer)
+            return this.writeUint8(o, new Uint8Array(buf));
+        else if (typeof buf === 'string')
+            return this.writeUint8(o, buf.split('').map(x => x.charCodeAt(0)));
+        else
+            return this.writeUint8(o, buf);
+    }
+}

package/dist/clang/memory/tar.d.ts

@@ -0,0 +1,2 @@
+import type { MemFS } from "./";
+export default function untar(buffer: ArrayBuffer, memfs: MemFS): void;