porffor 0.16.0-a2e115b05 → 0.16.0-a8928b26c

package/CONTRIBUTING.md

@@ -26,7 +26,7 @@ You can also swap out `node` in the alias to use another runtime like Deno (`den
 
 ### Precompile
 
-**If you update any file inside `compiler/builtins` you will need to do this for it to update inside Porffor otherwise your changes will have no effect.** Run `node compiler/precompile.js` to precompile. It may error during this, if so, you might have an error in your code or there could be a compiler error with Porffor (feel free to ask for help as soon as you encounter any errors with it).
+**If you update any file inside `compiler/builtins` you will need to do this for it to update inside Porffor otherwise your changes will have no effect.** Run `./porf precompile` to precompile. It may error during this, if so, you might have an error in your code or there could be a compiler error with Porffor (feel free to ask for help as soon as you encounter any errors with it).
 
 <br>
 

package/README.md

@@ -29,7 +29,7 @@ Expect nothing to work! Only very limited JS is currently supported. See files i
 > [!WARNING]
 > Compiling to native binaries uses [2c](#2c), Porffor's own Wasm -> C compiler, which is experimental.
 
-**`porf native path/to/script.js out(.exe)`**. You can specify the compiler with `--compiler=clang/zig/gcc`, and which opt level to use with `--cO=O3` (`Ofast` by default). Output binaries are also stripped by default.
+**`porf native path/to/script.js out(.exe)`**. You can specify the compiler with `--compiler=clang|gcc|zig` (`clang` by default), and which optimization level to use with `--cO=Ofast|O3|O2|O1|O0` (`Ofast` by default). Output binaries are also stripped by default.
 
 ### Compiling to C
 > [!WARNING]
@@ -49,7 +49,7 @@ Expect nothing to work! Only very limited JS is currently supported. See files i
 
 **`porf debug path/to/script.js`**
 
-### Profiling the generated Wasm of a JS file
+### Debugging the compiled Wasm of a JS file
 > [!WARNING]
 > Very experimental WIP feature!
 
@@ -63,26 +63,14 @@ Expect nothing to work! Only very limited JS is currently supported. See files i
 - `--valtype=i32|i64|f64` (default: `f64`) to set valtype
 - `-O0` to disable opt
 - `-O1` (default) to enable basic opt (simplify insts, treeshake wasm imports)
-- `-O2` to enable advanced opt (inlining). unstable
-- `-O3` to enable advanceder opt (precompute const math). unstable
-- `--no-run` to not run wasm output, just compile
-- `--opt-log` to log some opts
-- `--code-log` to log some codegen (you probably want `-funcs`)
-- `--regex-log` to log some regex
-- `--funcs` to log funcs
-- `--ast-log` to log AST
-- `--opt-funcs` to log funcs after opt
-- `--sections` to log sections as hex
-- `--opt-no-inline` to not inline any funcs
-- `--tail-call` to enable tail calls (experimental + not widely implemented)
-- `--compile-hints` to enable V8 compilation hints (experimental + doesn't seem to do much?)
+- `-O2` to enable advanced opt (inlining). unstable!
+- `-O3` to enable advanceder opt (precompute const math). unstable!
 
 ## Limitations
 - No full object support yet
 - Little built-ins/prototype
 - No async/promise/await
 - No variables between scopes (except args and globals)
-- Literal callees only in calls (eg `print()` works, `a = print; a()` does not)
 - No `eval()` etc (since it is AOT)
 
 ## Sub-engines
@@ -112,7 +100,7 @@ These include some early (stage 1/0) and/or dead (last commit years ago) proposa
 
 - Number literals
 - Declaring functions
-- Calling functions *literal callees only*
+- Calling functions
 - `return`
 - `let`/`const`/`var` basic declarations
 - Some basic integer operators (`+-/*%`)

package/compiler/2c.js

@@ -2,10 +2,11 @@ import { read_ieee754_binary64, read_signedLEB128, read_unsignedLEB128 } from '.
 import { Blocktype, Opcodes, Valtype } from './wasmSpec.js';
 import { operatorOpcode } from './expression.js';
 import { log } from './log.js';
+import Prefs from './prefs.js';
 
 const CValtype = {
-  i8: 'i8',
-  i16: 'i16',
+  i8: 'u8',
+  i16: 'u16',
   i32: 'i32',
   u32: 'u32',
   i64: 'i64',
@@ -17,8 +18,8 @@ const CValtype = {
   undefined: 'void'
 };
 
-const alwaysPreface = `typedef uint8_t i8;
-typedef uint16_t i16;
+const alwaysPreface = `typedef uint8_t u8;
+typedef uint16_t u16;
 typedef int32_t i32;
 typedef uint32_t u32;
 typedef int64_t i64;
@@ -33,11 +34,11 @@ struct ReturnValue {
   i32 type;
 };\n\n`;
 
-// todo: is memcpy/etc safe with host endianness?
+// todo: review whether 2cMemcpy should be default or not
 
 // all:
 // immediates: ['align', 'offset']
-const CMemFuncs = {
+const CMemFuncs = Prefs['2cMemcpy'] ? {
   [Opcodes.i32_store]: {
     c: `memcpy(_memory + offset + pointer, &value, sizeof(value));`,
     args: ['pointer', 'value'],
@@ -47,13 +48,13 @@ const CMemFuncs = {
   [Opcodes.i32_store16]: {
     c: `memcpy(_memory + offset + pointer, &value, sizeof(value));`,
     args: ['pointer', 'value'],
-    argTypes: ['i32', 'i16'],
+    argTypes: ['i32', 'u16'],
     returns: false
   },
   [Opcodes.i32_store8]: {
     c: `memcpy(_memory + offset + pointer, &value, sizeof(value));`,
     args: ['pointer', 'value'],
-    argTypes: ['i32', 'i8'],
+    argTypes: ['i32', 'u8'],
     returns: false
   },
 
@@ -66,7 +67,7 @@ return out;`,
     returns: 'i32'
   },
   [Opcodes.i32_load16_u]: {
-    c: `i16 out;
+    c: `u16 out;
 memcpy(&out, _memory + offset + pointer, sizeof(out));
 return out;`,
     args: ['pointer'],
@@ -74,7 +75,7 @@ return out;`,
     returns: 'i32'
   },
   [Opcodes.i32_load8_u]: {
-    c: `i8 out;
+    c: `u8 out;
 memcpy(&out, _memory + offset + pointer, sizeof(out));
 return out;`,
     args: ['pointer'],
@@ -96,6 +97,57 @@ return out;`,
     argTypes: ['i32'],
     returns: 'f64'
   },
+} : {
+  [Opcodes.i32_store]: {
+    c: `*((i32*)(_memory + offset + pointer)) = value;`,
+    args: ['pointer', 'value'],
+    argTypes: ['i32', 'i32'],
+    returns: false
+  },
+  [Opcodes.i32_store16]: {
+    c: `*((u16*)(_memory + offset + pointer)) = value;`,
+    args: ['pointer', 'value'],
+    argTypes: ['i32', 'u16'],
+    returns: false
+  },
+  [Opcodes.i32_store8]: {
+    c: `*((u8*)(_memory + offset + pointer)) = value;`,
+    args: ['pointer', 'value'],
+    argTypes: ['i32', 'u8'],
+    returns: false
+  },
+
+  [Opcodes.i32_load]: {
+    c: `return *((i32*)(_memory + offset + pointer));`,
+    args: ['pointer'],
+    argTypes: ['i32'],
+    returns: 'i32'
+  },
+  [Opcodes.i32_load16_u]: {
+    c: `return *((u16*)(_memory + offset + pointer));`,
+    args: ['pointer'],
+    argTypes: ['i32'],
+    returns: 'i32'
+  },
+  [Opcodes.i32_load8_u]: {
+    c: `return *((u8*)(_memory + offset + pointer));`,
+    args: ['pointer'],
+    argTypes: ['i32'],
+    returns: 'i32'
+  },
+
+  [Opcodes.f64_store]: {
+    c: `*((f64*)(_memory + offset + pointer)) = value;`,
+    args: ['pointer', 'value'],
+    argTypes: ['i32', 'f64'],
+    returns: false
+  },
+  [Opcodes.f64_load]: {
+    c: `return *((f64*)(_memory + offset + pointer));`,
+    args: ['pointer'],
+    argTypes: ['i32'],
+    returns: 'f64'
+  },
 };
 
 const inv = (obj, keyMap = x => x) => Object.keys(obj).reduce((acc, x) => { acc[keyMap(obj[x])] = x; return acc; }, {});
@@ -152,11 +204,16 @@ export default ({ funcs, globals, tags, data, exceptions, pages }) => {
 
   if (pages.size > 0) {
     prepend.set('_memory', `char _memory[${pages.size * pageSize}];\n`);
-    includes.set('string.h', true);
+    if (Prefs['2cMemcpy']) includes.set('string.h', true);
   }
 
   if (data.length > 0) {
-    prependMain.set('_data', data.map(x => `memcpy(_memory + ${x.offset}, (unsigned char[]){${x.bytes.join(',')}}, ${x.bytes.length});`).join('\n  '));
+    if (Prefs['2cMemcpy']) {
+      prependMain.set('_data', data.map(x => `memcpy(_memory + ${x.offset}, (unsigned char[]){${x.bytes.join(',')}}, ${x.bytes.length});`).join('\n  '));
+      includes.set('string.h', true);
+    } else {
+      prependMain.set('_data', data.map(x => x.bytes.reduce((acc, y, i) => acc + `_memory[${x.offset + i}]=(u8)${y};`, '')).join('\n  '));
+    }
   }
 
   if (importFuncs.find(x => x.name === '__Porffor_readArgv')) {
@@ -362,7 +419,10 @@ export default ({ funcs, globals, tags, data, exceptions, pages }) => {
 
         case Opcodes.return:
           // line(`return${returns ? ` ${removeBrackets(vals.pop())}` : ''}`);
-          line(`return${returns ? ` (struct ReturnValue){ ${removeBrackets(vals.pop())}, ${removeBrackets(vals.pop())} }` : ''}`);
+          const b = vals.pop();
+          const a = vals.pop();
+          line(`return${returns ? ` (struct ReturnValue){ ${removeBrackets(a)}, ${removeBrackets(b)} }` : ''}`);
+          // line(`return${returns ? ` (struct ReturnValue){ ${removeBrackets(vals.pop())}, ${removeBrackets(vals.pop())} }` : ''}`);
           break;
 
         case Opcodes.if: {
@@ -474,7 +534,7 @@ _time_out = _time.tv_nsec / 1000000. + _time.tv_sec * 1000.;`);
     out[read++] = ch;
   }
 
-  memcpy(_memory + outPtr, &read, sizeof(read));
+  *((i32*)(_memory + outPtr)) = (i32)read;
   return read;
 }`);
 
@@ -504,7 +564,7 @@ _time_out = _time.tv_nsec / 1000000. + _time.tv_sec * 1000.;`);
 
   fclose(fp);
 
-  memcpy(_memory + outPtr, &read, sizeof(read));
+  *((i32*)(_memory + outPtr)) = (i32)read;
   return read;
 }`);
                 const outPtr = vals.pop();

package/compiler/allocators.js

@@ -0,0 +1,128 @@
+import { Opcodes, PageSize, Valtype } from './wasmSpec.js';
+import { number } from './embedding.js';
+import Prefs from './prefs.js';
+
+// we currently have 3 allocators:
+// - static (default): a static/compile-time allocator. fast (no grow/run-time alloc needed) but can break some code
+// - grow: perform a memory.grow every allocation. simple but maybe slow?
+// - chunk: perform large memory.grow's in chunks when needed. needs investigation
+
+export default name => {
+  switch (name) {
+    case 'static': return new StaticAllocator();
+    case 'grow': return new GrowAllocator();
+    case 'chunk': return new ChunkAllocator();
+    default: throw new Error(`unknown allocator: ${name}`);
+  }
+};
+
+export class StaticAllocator {
+  constructor() {
+  }
+
+  allocType(itemType) {
+    switch (itemType) {
+      case 'i8': return 'bytestring';
+      case 'i16': return 'string';
+
+      default: return 'array';
+    }
+  }
+
+  ptr(ind) {
+    if (ind === 0) return 4;
+    return ind * PageSize;
+  }
+
+  alloc({ scope, pages }, name, { itemType }) {
+    const reason = `${this.allocType(itemType)}: ${Prefs.scopedPageNames ? (scope.name + '/') : ''}${name}`;
+
+    if (pages.has(reason)) return number(this.ptr(pages.get(reason).ind), Valtype.i32);
+
+    if (reason.startsWith('array:')) pages.hasArray = true;
+    if (reason.startsWith('string:')) pages.hasString = true;
+    if (reason.startsWith('bytestring:')) pages.hasByteString = true;
+    if (reason.includes('string:')) pages.hasAnyString = true;
+
+    let ind = pages.size;
+    pages.set(reason, { ind, type: itemType });
+
+    scope.pages ??= new Map();
+    scope.pages.set(reason, { ind, type: itemType });
+
+    return number(this.ptr(ind), Valtype.i32);
+  }
+}
+
+export class GrowAllocator {
+  constructor() {
+    Prefs.rmUnusedTypes = false;
+  }
+
+  alloc() {
+    return [
+      // grow by 1 page
+      [ Opcodes.i32_const, 1 ],
+      [ Opcodes.memory_grow, 0 ], // returns old page count
+
+      // get ptr (page count * page size)
+      number(65536, Valtype.i32)[0],
+      [ Opcodes.i32_mul ]
+    ];
+  }
+}
+
+export class ChunkAllocator {
+  constructor(chunkSize) {
+    Prefs.rmUnusedTypes = false;
+
+    // 64KiB * chunk size each growth
+    // 16: 1MiB chunks
+    this.chunkSize = chunkSize ?? Prefs.chunkAllocatorSize ?? 16;
+  }
+
+  alloc({ asmFunc, funcIndex }) {
+    const func = funcIndex['#chunkallocator_alloc'] ?? asmFunc('#chunkallocator_alloc', {
+      wasm: [
+        [ Opcodes.global_get, 0 ],
+        [ Opcodes.global_get, 1 ],
+        [ Opcodes.i32_ge_s ],
+        [ Opcodes.if, Valtype.i32 ], // ptr >= next
+          // grow by chunk size pages
+          [ Opcodes.i32_const, this.chunkSize ],
+          [ Opcodes.memory_grow, 0 ],
+
+          // ptr = prev memory size * PageSize
+          number(65536, Valtype.i32)[0],
+          [ Opcodes.i32_mul ],
+          [ Opcodes.global_set, 0 ],
+
+          // next = ptr + ((chunkSize - 1) * PageSize)
+          [ Opcodes.global_get, 0 ],
+          number(65536 * (this.chunkSize - 1), Valtype.i32)[0],
+          [ Opcodes.i32_add ],
+          [ Opcodes.global_set, 1 ],
+
+          // return ptr
+          [ Opcodes.global_get, 0 ],
+        [ Opcodes.else ],
+          // return ptr = ptr + PageSize
+          [ Opcodes.global_get, 0 ],
+          number(65536, Valtype.i32)[0],
+          [ Opcodes.i32_add ],
+          [ Opcodes.global_set, 0 ],
+          [ Opcodes.global_get, 0 ],
+        [ Opcodes.end ],
+      ],
+      params: [],
+      locals: [],
+      globals: [ Valtype.i32, Valtype.i32 ],
+      globalNames: ['#chunkallocator_ptr', '#chunkallocator_next'],
+      returns: [ Valtype.i32 ],
+    }).index;
+
+    return [
+      [ Opcodes.call, func ]
+    ];
+  }
+}

package/compiler/assemble.js

@@ -21,7 +21,7 @@ const chHint = (topTier, baselineTier, strategy) => {
   return (strategy | (baselineTier << 2) | (topTier << 4));
 };
 
-export default (funcs, globals, tags, pages, data, flags) => {
+export default (funcs, globals, tags, pages, data, flags, noTreeshake = false) => {
   const types = [], typeCache = {};
 
   const optLevel = parseInt(process.argv.find(x => x.startsWith('-O'))?.[2] ?? 1);
@@ -44,7 +44,7 @@ export default (funcs, globals, tags, pages, data, flags) => {
 
   let importFuncs = [];
 
-  if (optLevel < 1 || !Prefs.treeshakeWasmImports) {
+  if (optLevel < 1 || !Prefs.treeshakeWasmImports || noTreeshake) {
     importFuncs = importedFuncs;
   } else {
     let imports = new Map();
@@ -94,7 +94,7 @@ export default (funcs, globals, tags, pages, data, flags) => {
 
   const importSection = importFuncs.length === 0 ? [] : createSection(
     Section.import,
-    encodeVector(importFuncs.map(x => [ 0, ...encodeString(x.import), ExportDesc.func, getType(new Array(x.params).fill(x.name.startsWith('profile') ? Valtype.i32 : valtypeBinary), new Array(x.returns).fill(valtypeBinary)) ]))
+    encodeVector(importFuncs.map(x => [ 0, ...encodeString(x.import), ExportDesc.func, getType(typeof x.params === 'object' ? x.params : new Array(x.params).fill(valtypeBinary), new Array(x.returns).fill(valtypeBinary)) ]))
   );
 
   const funcSection = createSection(
@@ -116,7 +116,7 @@ export default (funcs, globals, tags, pages, data, flags) => {
     ] ])
   );
 
-  if (pages.has('func argc lut')) {
+  if (pages.has('func argc lut') && !data.addedFuncArgcLut) {
     // generate func argc lut data
     const bytes = [];
     for (let i = 0; i < funcs.length; i++) {
@@ -128,6 +128,7 @@ export default (funcs, globals, tags, pages, data, flags) => {
       offset: pages.get('func argc lut').ind * pageSize,
       bytes
     });
+    data.addedFuncArgcLut = true;
   }
 
   // const t0 = performance.now();
@@ -239,7 +240,13 @@ export default (funcs, globals, tags, pages, data, flags) => {
 
   const dataSection = data.length === 0 ? [] : createSection(
     Section.data,
-    encodeVector(data.map(x => [ 0x00, Opcodes.i32_const, ...signedLEB128(x.offset), Opcodes.end, ...encodeVector(x.bytes) ]))
+    encodeVector(data.map(x => {
+      // type: active
+      if (x.offset != null) return [ 0x00, Opcodes.i32_const, ...signedLEB128(x.offset), Opcodes.end, ...encodeVector(x.bytes) ];
+
+      // type: passive
+      return [ 0x01, ...encodeVector(x.bytes) ];
+    }))
   );
 
   const dataCountSection = data.length === 0 ? [] : createSection(

package/compiler/builtins/array.ts

@@ -171,12 +171,11 @@ export const __Array_prototype_filter = (_this: any[], callbackFn: any) => {
 };
 
 export const __Array_prototype_map = (_this: any[], callbackFn: any) => {
-  const out: any[] = [];
-
-  const len: i32 = _this.length;
   let i: i32 = 0;
+  const len: i32 = _this.length;
+  const out: any[] = new Array(len);
   while (i < len) {
-    out.push(callbackFn(_this[i], i++, _this));
+    out[i] = callbackFn(_this[i], i++, _this);
   }
 
   return out;

package/compiler/builtins/date.ts

@@ -722,12 +722,9 @@ export const __Porffor_date_allocate = (): Date => {
   const hack: bytestring = '';
 
   if (hack.length == 0) {
-    hack.length = Porffor.wasm`i32.const 1
-memory.grow 0
-drop
-memory.size 0
+    hack.length = Porffor.wasm`
 i32.const 1
-i32.sub
+memory.grow 0
 i32.const 65536
 i32.mul
 i32.from_u`;

package/compiler/builtins/set.ts

@@ -2,12 +2,9 @@ import type {} from './porffor.d.ts';
 
 // dark wasm magic for dealing with memory, sorry.
 export const __Porffor_allocate = (): number => {
-  Porffor.wasm`i32.const 1
-memory.grow 0
-drop
-memory.size 0
+  Porffor.wasm`
 i32.const 1
-i32.sub
+memory.grow 0
 i32.const 65536
 i32.mul
 i32.from_u

package/compiler/builtins.js

@@ -32,13 +32,13 @@ export const importedFuncs = [
   {
     name: 'profile1',
     import: 'y',
-    params: 1,
+    params: [ Valtype.i32 ],
     returns: 0
   },
   {
     name: 'profile2',
     import: 'z',
-    params: 1,
+    params: [ Valtype.i32 ],
     returns: 0
   },
   {