toiljs 0.0.16 → 0.0.19

package/test/io.test.ts

@@ -1,69 +1,93 @@
 import { describe, expect, it } from 'vitest';
 
-import { BinaryReader } from '../src/io/BinaryReader';
-import { BinaryWriter } from '../src/io/BinaryWriter';
+import { DataReader, DataWriter } from '../src/io/codec';
 import { FastMap } from '../src/io/FastMap';
 import { FastSet } from '../src/io/FastSet';
 
-describe('BinaryWriter / BinaryReader', () => {
+describe('DataWriter / DataReader', () => {
     it('round-trips fixed-width integers', () => {
-        const w = new BinaryWriter();
-        w.writeU8(255);
-        w.writeU16(65535);
-        w.writeU32(4294967295);
-        w.writeU64(18446744073709551615n);
-        w.writeI8(-128);
-        w.writeI32(-2147483648);
+        const w = new DataWriter();
+        w.writeU8(255).writeU16(65535).writeU32(4294967295).writeU64(18446744073709551615n);
+        w.writeI8(-128).writeI16(-32768).writeI32(-2147483648).writeI64(-9223372036854775808n);
 
-        const r = new BinaryReader(w.getBuffer());
+        const r = new DataReader(w.toBytes());
         expect(r.readU8()).toBe(255);
         expect(r.readU16()).toBe(65535);
         expect(r.readU32()).toBe(4294967295);
         expect(r.readU64()).toBe(18446744073709551615n);
         expect(r.readI8()).toBe(-128);
+        expect(r.readI16()).toBe(-32768);
         expect(r.readI32()).toBe(-2147483648);
+        expect(r.readI64()).toBe(-9223372036854775808n);
+        expect(r.ok).toBe(true);
+        expect(r.remaining()).toBe(0);
     });
 
-    it('round-trips u256 and strings', () => {
-        const big = 123456789012345678901234567890n;
-        const w = new BinaryWriter();
-        w.writeU256(big);
-        w.writeStringWithLength('hello toil 🛠');
-        w.writeBoolean(true);
+    it('round-trips floats, bool, bytes and strings', () => {
+        const w = new DataWriter();
+        w.writeF32(0.5).writeF64(3.141592653589793).writeBool(true).writeBool(false);
+        w.writeBytes(new Uint8Array([1, 2, 3, 0, 255]));
+        w.writeString('hello toil 🛠');
 
-        const r = new BinaryReader(w.getBuffer());
-        expect(r.readU256()).toBe(big);
-        expect(r.readStringWithLength()).toBe('hello toil 🛠');
-        expect(r.readBoolean()).toBe(true);
+        const r = new DataReader(w.toBytes());
+        expect(r.readF32()).toBe(0.5);
+        expect(r.readF64()).toBe(3.141592653589793);
+        expect(r.readBool()).toBe(true);
+        expect(r.readBool()).toBe(false);
+        expect([...r.readBytes()]).toEqual([1, 2, 3, 0, 255]);
+        expect(r.readString()).toBe('hello toil 🛠');
+        expect(r.ok).toBe(true);
     });
 
-    it('round-trips arrays', () => {
-        const w = new BinaryWriter();
-        w.writeU32Array([1, 2, 3]);
-        w.writeStringArray(['a', 'bb', 'ccc']);
+    it('round-trips u128 / i128 / u256 / i256', () => {
+        const u = 123456789012345678901234567890n;
+        const big256 = (2n ** 256n) - 1n;
+        const w = new DataWriter();
+        w.writeU128(u).writeI128(-1234567890123456789n).writeU256(big256).writeI256(-(2n ** 200n));
 
-        const r = new BinaryReader(w.getBuffer());
-        expect(r.readU32Array()).toEqual([1, 2, 3]);
-        expect(r.readStringArray()).toEqual(['a', 'bb', 'ccc']);
+        const r = new DataReader(w.toBytes());
+        expect(r.readU128()).toBe(u);
+        expect(r.readI128()).toBe(-1234567890123456789n);
+        expect(r.readU256()).toBe(big256);
+        expect(r.readI256()).toBe(-(2n ** 200n));
     });
 
-    it('rejects out-of-range values', () => {
-        const w = new BinaryWriter();
-        expect(() => w.writeU8(256)).toThrow();
-        expect(() => w.writeI8(128)).toThrow();
+    it('grows past the initial capacity without corruption (regression)', () => {
+        // The default buffer is 64 bytes; this forces several reallocations.
+        const w = new DataWriter();
+        const text = 'z'.repeat(500);
+        for (let i = 0; i < 40; i++) w.writeU64(BigInt(i)); // 320 bytes
+        w.writeString(text);
+
+        const r = new DataReader(w.toBytes());
+        for (let i = 0; i < 40; i++) expect(r.readU64()).toBe(BigInt(i));
+        expect(r.readString()).toBe(text);
+        expect(r.ok).toBe(true);
+    });
+
+    it('round-trips big-endian when be is set, and be flips byte order', () => {
+        expect([...new DataWriter().writeU32(0x01020304).toBytes()]).toEqual([4, 3, 2, 1]);
+        expect([...new DataWriter().writeU32(0x01020304, true).toBytes()]).toEqual([1, 2, 3, 4]);
+
+        const w = new DataWriter();
+        w.writeU16(0xabcd, true).writeI32(-2, true).writeU64(0xdeadbeefn, true).writeU128(123456789012345n, true);
+        const r = new DataReader(w.toBytes());
+        expect(r.readU16(true)).toBe(0xabcd);
+        expect(r.readI32(true)).toBe(-2);
+        expect(r.readU64(true)).toBe(0xdeadbeefn);
+        expect(r.readU128(true)).toBe(123456789012345n);
     });
 
-    it('rejects negative / overflowing u256 and u128', () => {
-        const w = new BinaryWriter();
-        expect(() => w.writeU256(-1n)).toThrow();
-        expect(() => w.writeU256(2n ** 256n)).toThrow();
-        expect(() => w.writeU128(-1n)).toThrow();
-        expect(() => w.writeU128(2n ** 128n)).toThrow();
+    it('is little-endian and masks instead of throwing', () => {
+        // u32 1 → 01 00 00 00 (LE); writeU8 masks to a byte, no throw.
+        const bytes = new DataWriter().writeU32(1).writeU8(256).toBytes();
+        expect([...bytes]).toEqual([1, 0, 0, 0, 0]);
     });
 
-    it('throws when reading past the end', () => {
-        const r = new BinaryReader(new Uint8Array(2));
-        expect(() => r.readU32()).toThrow();
+    it('reports ok=false when reading past the end (no throw)', () => {
+        const r = new DataReader(new Uint8Array(2));
+        expect(r.readU32()).toBe(0);
+        expect(r.ok).toBe(false);
     });
 });
 

package/test/prettier-plugin.test.ts

@@ -0,0 +1,46 @@
+import prettier from 'prettier';
+import { describe, expect, it } from 'vitest';
+
+import * as plugin from '../presets/prettier-plugin.js';
+
+function fmt(src: string): Promise<string> {
+    return prettier.format(src, {
+        parser: 'typescript',
+        plugins: [plugin],
+        tabWidth: 4,
+        singleQuote: true,
+        semi: true,
+    });
+}
+
+// The plugin lets prettier format toilscript server code, whose native decorators on free
+// functions (@main, @remote function) are not valid JS/TS grammar (so stock prettier throws).
+describe('toiljs prettier-plugin', () => {
+    it('formats a free @remote function and keeps the decorator', async () => {
+        const out = await fmt('@remote\nfunction ping(n:i32):i32{return n+1}\n');
+        expect(out).toContain('@remote');
+        expect(out).toContain('function ping(n: i32): i32 {');
+        expect(out).not.toContain('toil-decorator'); // marker fully restored
+    });
+
+    it('formats @main', async () => {
+        const out = await fmt('@main\nfunction run():i32{return 42}\n');
+        expect(out).toMatch(/@main\nfunction run\(\): i32 \{/);
+    });
+
+    it('handles @remote export function', async () => {
+        const out = await fmt('@remote\nexport function pong():void{}\n');
+        expect(out).toContain('@remote\nexport function pong(): void {}');
+    });
+
+    it('leaves class/method decorators untouched', async () => {
+        const out = await fmt('@data\nclass A{ x:i32=0 }\n');
+        expect(out).toContain('@data');
+        expect(out).toContain('class A {');
+    });
+
+    it('is idempotent', async () => {
+        const once = await fmt('@remote\nfunction f():void{}\n@main\nfunction g():void{}\n');
+        expect(await fmt(once)).toBe(once);
+    });
+});

package/test/rpc.test.ts

@@ -0,0 +1,50 @@
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { Server } from '../src/client/rpc';
+
+// `Server` is the runtime behind the generated typed surface. Until the transport
+// is wired, it is a recursive proxy that throws on call.
+describe('Server RPC stub', () => {
+    it('throws on a direct call, naming the path', () => {
+        const s = Server as { ping: () => unknown };
+        expect(() => s.ping()).toThrow(/Server\.ping\(\)/);
+    });
+
+    it('throws on a nested service.method call', () => {
+        const s = Server as { accounts: { getUser: () => unknown } };
+        expect(() => s.accounts.getUser()).toThrow(/Server\.accounts\.getUser\(\)/);
+        expect(() => s.accounts.getUser()).toThrow(/not available yet/);
+    });
+
+    it('is not thenable (so it is not mistaken for a promise)', () => {
+        const s = Server as Record<string, unknown>;
+        expect(s.then).toBeUndefined();
+    });
+
+    it('ignores symbol probes without throwing', () => {
+        const s = Server as Record<PropertyKey, unknown>;
+        expect(s[Symbol.iterator]).toBeUndefined();
+    });
+});
+
+// `Server.REST` surfaces the working fetch client that the generated `shared/server.ts`
+// attaches to `globalThis.__toilRest` on import.
+describe('Server.REST surface', () => {
+    afterEach(() => {
+        delete (globalThis as { __toilRest?: unknown }).__toilRest;
+    });
+
+    it('returns the attached REST client when shared/server has loaded', () => {
+        const fake = { todos: { getTodo: async () => 'ok' } };
+        (globalThis as { __toilRest?: unknown }).__toilRest = fake;
+        const s = Server as { REST: typeof fake };
+        expect(s.REST).toBe(fake);
+        expect(s.REST.todos.getTodo).toBeTypeOf('function');
+    });
+
+    it('throws a helpful "not loaded" error when the REST client is absent', () => {
+        const s = Server as { REST: { todos: { getTodo: () => unknown } } };
+        expect(() => s.REST.todos.getTodo()).toThrow(/Server\.REST\.todos\.getTodo\(\)/);
+        expect(() => s.REST.todos.getTodo()).toThrow(/has not loaded/);
+    });
+});

package/tests/data-parity/generated-parity.ts

@@ -0,0 +1,99 @@
+// Proves the fork's GENERATED @data class (from --rpcModule) matches the AS @data
+// wire byte-for-byte. Compiles spec.ts (which has `@data class Foo`) with the
+// ToilScript fork, emitting both the wasm and the generated server.ts, then checks
+// the generated Foo class against the AS Foo.encode() bytes, both directions.
+// Run with: node tests/data-parity/generated-parity.ts
+import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { spawnSync } from "node:child_process";
+const here = dirname(fileURLToPath(import.meta.url));
+// The generated module imports DataWriter/DataReader from this specifier.
+const codec = join(here, "..", "..", "src", "io", "codec.ts");
+const fork = "/root/toil-stuff/toilscript";
+const spec = join(here, "spec.ts");
+const tmp = mkdtempSync(join(tmpdir(), "gen-parity-"));
+const wasmPath = join(tmp, "spec.wasm");
+const modPath = join(tmp, "server.ts");
+writeFileSync(join(tmp, "package.json"), '{ "type": "module" }\n');
+
+const compile = spawnSync(
+    "node",
+    [join(fork, "bin", "toilscript.js"), spec, "-o", wasmPath, "--runtime", "stub", "--initialMemory", "32", "--rpcModule", modPath, "--rpcRuntime", codec],
+    { stdio: "inherit" },
+);
+if (compile.status !== 0) {
+    console.error("generated parity: COMPILE FAILED");
+    rmSync(tmp, { recursive: true, force: true });
+    process.exit(1);
+}
+
+function fail(msg: string): never {
+    console.error("generated parity: FAIL,", msg);
+    rmSync(tmp, { recursive: true, force: true });
+    process.exit(1);
+}
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+    return true;
+}
+const hex = (b: Uint8Array): string => Buffer.from(b).toString("hex");
+
+// the known sample (must match spec.ts `sample()`)
+const ID = 0xcafebabedeadbeefn;
+const COUNT = -42;
+const FLAG = true;
+const BIG = 123456789n;
+const NAME = "cross-lang";
+
+// AS side: instantiate and read the sample bytes out of linear memory.
+const { instance } = await WebAssembly.instantiate(readFileSync(wasmPath), {
+    env: { abort: (_m: number, _f: number, line: number) => { throw new Error("wasm abort @ line " + line); } },
+});
+const x = instance.exports as Record<string, CallableFunction> & { memory: WebAssembly.Memory };
+const SCRATCH = 0x100000;
+const len = x.encodeSampleTo(SCRATCH) as number;
+const wasmBytes = new Uint8Array(x.memory.buffer, SCRATCH, len).slice();
+
+// Generated side: build the same value with the generated Foo class and encode.
+const gen = (await import(pathToFileURL(modPath).href)) as {
+    Foo: new () => {
+        id: bigint; count: number; flag: boolean; big: bigint; name: string;
+        encode(): Uint8Array;
+    } & Record<string, unknown>;
+};
+const FooClass = gen.Foo as unknown as {
+    new (): { id: bigint; count: number; flag: boolean; big: bigint; name: string; encode(): Uint8Array };
+    decode(buf: Uint8Array): { id: bigint; count: number; flag: boolean; big: bigint; name: string };
+    dataId(): number;
+};
+
+const foo = new FooClass();
+foo.id = ID;
+foo.count = COUNT;
+foo.flag = FLAG;
+foo.big = BIG;
+foo.name = NAME;
+const genBytes = foo.encode();
+
+// 1) byte-for-byte identical to the AS @data encoding.
+if (!bytesEqual(genBytes, wasmBytes)) {
+    fail(`byte mismatch\n  AS:        ${hex(wasmBytes)}\n  generated: ${hex(genBytes)}`);
+}
+
+// 2) the generated decode round-trips the AS bytes back to the sample.
+const back = FooClass.decode(wasmBytes);
+if (back.id !== ID) fail("id (decode)");
+if (back.count !== COUNT) fail("count (decode)");
+if (back.flag !== FLAG) fail("flag (decode)");
+if (back.big !== BIG) fail("big (decode)");
+if (back.name !== NAME) fail("name (decode)");
+
+// 3) the AS side accepts the generated bytes.
+new Uint8Array(x.memory.buffer, SCRATCH, genBytes.length).set(genBytes);
+if ((x.checkBytes(SCRATCH, genBytes.length) as number) !== 1) fail("AS rejected generated bytes");
+
+console.log(`@data generated-class parity: PASS (generated <-> AS both ways, byte-for-byte, ${len} bytes)`);
+rmSync(tmp, { recursive: true, force: true });

package/tests/data-parity/parity.ts

@@ -0,0 +1,80 @@
+// Cross-language @data byte-parity proof. Compiles spec.ts with the ToilScript
+// fork (which has @data), then checks the TS codec (src/io/codec.ts) against it
+// both directions, including byte-for-byte. Run with: node tests/data-parity/parity.ts
+import { readFileSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+import { DataWriter, DataReader } from "../../src/io/codec.ts";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const fork = "/root/toil-stuff/toilscript";
+const spec = join(here, "spec.ts");
+const tmp = mkdtempSync(join(tmpdir(), "parity-"));
+const wasmPath = join(tmp, "spec.wasm");
+
+const compile = spawnSync(
+    "node",
+    [join(fork, "bin", "toilscript.js"), spec, "-o", wasmPath, "--runtime", "stub", "--initialMemory", "32"],
+    { stdio: "inherit" },
+);
+if (compile.status !== 0) {
+    console.error("@data parity: COMPILE FAILED");
+    rmSync(tmp, { recursive: true, force: true });
+    process.exit(1);
+}
+
+function fail(msg: string): never {
+    console.error("@data parity: FAIL,", msg);
+    rmSync(tmp, { recursive: true, force: true });
+    process.exit(1);
+}
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+    return true;
+}
+const hex = (b: Uint8Array): string => Buffer.from(b).toString("hex");
+
+const { instance } = await WebAssembly.instantiate(readFileSync(wasmPath), {
+    env: { abort: (_m: number, _f: number, line: number) => { throw new Error("wasm abort @ line " + line); } },
+});
+const x = instance.exports as Record<string, CallableFunction> & { memory: WebAssembly.Memory };
+const mem = x.memory;
+const SCRATCH = 0x100000; // 1 MiB, above the low heap
+
+// the known sample (must match spec.ts)
+const ID = 0xcafebabedeadbeefn;
+const COUNT = -42;
+const FLAG = true;
+const BIG = 123456789n;
+const NAME = "cross-lang";
+const fooId = (x.fooId() as number) >>> 0;
+
+// 1) ToilScript encodes the sample; read its bytes out of linear memory.
+const len = x.encodeSampleTo(SCRATCH) as number;
+const wasmBytes = new Uint8Array(mem.buffer, SCRATCH, len).slice();
+
+// 2) TS decodes the ToilScript bytes.
+const r = new DataReader(wasmBytes);
+if ((r.readU32() >>> 0) !== fooId) fail("typeId mismatch");
+if (r.readU64() !== ID) fail("id (ToilScript -> TS)");
+if (r.readI32() !== COUNT) fail("count (ToilScript -> TS)");
+if (r.readBool() !== FLAG) fail("flag (ToilScript -> TS)");
+if (r.readU128() !== BIG) fail("big (ToilScript -> TS)");
+if (r.readString() !== NAME) fail("name (ToilScript -> TS)");
+if (!r.ok || r.remaining() !== 0) fail("trailing/ok (ToilScript -> TS)");
+
+// 3) TS encodes the same value; it must be byte-for-byte identical.
+const w = new DataWriter();
+w.writeU32(fooId).writeU64(ID).writeI32(COUNT).writeBool(FLAG).writeU128(BIG).writeString(NAME);
+const tsBytes = w.toBytes();
+if (!bytesEqual(tsBytes, wasmBytes)) fail(`byte mismatch\n  ToilScript: ${hex(wasmBytes)}\n  TS:         ${hex(tsBytes)}`);
+
+// 4) ToilScript decodes the TS bytes and confirms the value.
+new Uint8Array(mem.buffer, SCRATCH, tsBytes.length).set(tsBytes);
+if ((x.checkBytes(SCRATCH, tsBytes.length) as number) !== 1) fail("ToilScript rejected TS bytes (TS -> ToilScript)");
+
+console.log(`@data parity: PASS (ToilScript<->TS both ways, byte-for-byte, ${len} bytes)`);
+rmSync(tmp, { recursive: true, force: true });

package/tests/data-parity/spec.ts

@@ -0,0 +1,46 @@
+// ToilScript side of the cross-language @data parity proof. Compiled by the
+// ToilScript fork (which has @data + std/assembly/data.ts). Exposes a known sample so the
+// TS codec can be checked against it byte-for-byte, both directions.
+//
+// A fixed scratch region (high in linear memory, away from the low heap) is used
+// to move bytes across the JS boundary without the loader.
+
+@data
+class Foo {
+  id: u64 = 0;
+  count: i32 = 0;
+  flag: bool = false;
+  big: u128 = u128.Zero;
+  name: string = "";
+}
+
+function sample(): Foo {
+  const f = new Foo();
+  f.id = 0xCAFEBABEDEADBEEF;
+  f.count = -42;
+  f.flag = true;
+  f.big = u128.fromU64(123456789);
+  f.name = "cross-lang";
+  return f;
+}
+
+export function fooId(): u32 {
+  return Foo.dataId();
+}
+
+/** Encode the sample, copy it to `out`, return the byte length. */
+export function encodeSampleTo(out: usize): i32 {
+  const bytes = sample().encode();
+  memory.copy(out, bytes.dataStart, <usize>bytes.length);
+  return bytes.length;
+}
+
+/** Decode `len` bytes at `inp` and return 1 if they equal the sample, else 0. */
+export function checkBytes(inp: usize, len: i32): i32 {
+  const bytes = new Uint8Array(len);
+  memory.copy(bytes.dataStart, inp, <usize>len);
+  const f = Foo.decode(bytes);
+  const s = sample();
+  const ok = f.id == s.id && f.count == s.count && f.flag == s.flag && f.big == s.big && f.name == s.name;
+  return ok ? 1 : 0;
+}

package/tsconfig.json

@@ -17,6 +17,6 @@
     "exclude": [
         "node_modules",
         "build",
-        "src/server"
+        "server"
     ]
 }

package/tsconfig.server.json

@@ -5,6 +5,6 @@
         "noEmit": true
     },
     "include": [
-        "src/server/**/*.ts"
+        "server/**/*.ts"
     ]
 }

package/build/io/BinaryReader.d.ts

@@ -1,44 +0,0 @@
-import type { BufferLike, i16, i32, i64, i8, Selector, u16, u32, u8 } from './types.js';
-export declare class BinaryReader {
-    private buffer;
-    private currentOffset;
-    constructor(bytes: BufferLike);
-    get byteLength(): number;
-    static stringCompare(a: string, b: string): number;
-    static bigintCompare(a: bigint, b: bigint): number;
-    static numberCompare(a: number, b: number): number;
-    setBuffer(bytes: BufferLike): void;
-    length(): number;
-    bytesLeft(): number;
-    readI8(): i8;
-    readI16(be?: boolean): i16;
-    readI32(be?: boolean): i32;
-    readI64(be?: boolean): i64;
-    readU8(): u8;
-    readU16(be?: boolean): u16;
-    readU32(be?: boolean): u32;
-    readU64(be?: boolean): bigint;
-    readU128(be?: boolean): bigint;
-    readU256(be?: boolean): bigint;
-    readI128(be?: boolean): bigint;
-    readBoolean(): boolean;
-    readSelector(): Selector;
-    readBytes(length: u32, zeroStop?: boolean): Uint8Array;
-    readString(length: u32): string;
-    readStringWithLength(be?: boolean): string;
-    readBytesWithLength(maxLength?: number, be?: boolean): Uint8Array;
-    readArrayOfBuffer(be?: boolean): Uint8Array[];
-    readU256Array(be?: boolean): bigint[];
-    readU128Array(be?: boolean): bigint[];
-    readU64Array(be?: boolean): bigint[];
-    readU32Array(be?: boolean): u32[];
-    readU16Array(be?: boolean): u16[];
-    readU8Array(): u8[];
-    readStringArray(be?: boolean): string[];
-    readBytesArray(be?: boolean): Uint8Array[];
-    getOffset(): u16;
-    setOffset(offset: u16): void;
-    verifyEnd(size: number): void;
-    private reverseBytes;
-    private toHexString;
-}