porffor 0.50.20 → 0.50.22

package/README.md

@@ -175,9 +175,6 @@ No particular order and no guarantees, just what could happen soon™
   - PGO
 - Self hosted testing?
 
-## VSCode extension
-There is a vscode extension in `vscode-ext` which tweaks JS syntax highlighting to be nicer with porffor features (eg highlighting wasm inside of inline asm).
-
 ## Wasm proposals used
 Porffor intentionally does not use Wasm proposals which are not commonly implemented yet (eg GC) so it can be used in as many places as possible.
 

package/compiler/assemble.js

@@ -1,5 +1,5 @@
 import { Valtype, FuncType, ExportDesc, Section, Magic, Opcodes, PageSize, Reftype } from './wasmSpec.js';
-import { encodeVector, encodeString, encodeLocal, unsignedLEB128, signedLEB128, unsignedLEB128_into, signedLEB128_into, ieee754_binary64, ieee754_binary64_into } from './encoding.js';
+import { encodeVector, encodeString, encodeLocal, unsignedLEB128, signedLEB128, unsignedLEB128_into, signedLEB128_into, ieee754_binary64, ieee754_binary64_into, unsignedLEB128_length } from './encoding.js';
 import { importedFuncs } from './builtins.js';
 import { log } from './log.js';
 import './prefs.js';
@@ -250,9 +250,7 @@ export default (funcs, globals, tags, pages, data, noTreeshake = false) => {
   let codeSection = [];
   for (let i = 0; i < funcs.length; i++) {
     const x = funcs[i];
-    // time(x.name);
     const locals = Object.values(x.locals).sort((a, b) => a.idx - b.idx);
-    // time('  locals gen');
 
     const paramCount = x.params.length;
     let localDecl = [], typeCount = 0, lastType, declCount = 0;
@@ -269,77 +267,131 @@ export default (funcs, globals, tags, pages, data, noTreeshake = false) => {
       lastType = local?.type;
     }
 
-    // time('  localDecl gen');
-
-    const makeAssembled = Prefs.d;
     let wasm = [], wasmNonFlat = [];
-    for (let i = 0; i < x.wasm.length; i++) {
-      let o = x.wasm[i];
-
-      // encode local/global ops as unsigned leb128 from raw number
-      if (
-        (o[0] >= Opcodes.local_get && o[0] <= Opcodes.global_set) &&
-        o[1] > 127
-      ) {
-        const n = o[1];
-        o = [ o[0] ];
-        unsignedLEB128_into(n, o);
-      }
+    if (Prefs.d) {
+      for (let i = 0; i < x.wasm.length; i++) {
+        let o = x.wasm[i];
+
+        // encode local/global ops as unsigned leb128 from raw number
+        if (
+          (o[0] >= Opcodes.local_get && o[0] <= Opcodes.global_set) &&
+          o[1] > 127
+        ) {
+          const n = o[1];
+          o = [ o[0] ];
+          unsignedLEB128_into(n, o);
+        }
 
-      // encode f64.const ops as ieee754 from raw number
-      if (o[0] === Opcodes.f64_const) {
-        const n = o[1];
-        o = ieee754_binary64(n);
-        if (o.length === 8) o.unshift(Opcodes.f64_const);
-      }
+        // encode f64.const ops as ieee754 from raw number
+        if (o[0] === Opcodes.f64_const) {
+          const n = o[1];
+          o = ieee754_binary64(n);
+          if (o.length === 8) o.unshift(Opcodes.f64_const);
+        }
 
-      // encode call ops as unsigned leb128 from raw number
-      if ((o[0] === Opcodes.call /* || o[0] === Opcodes.return_call */) && o[1] >= importedFuncs.length) {
-        const n = o[1] - importDelta;
-        o = [ Opcodes.call ];
-        unsignedLEB128_into(n, o);
-      }
+        // encode call ops as unsigned leb128 from raw number
+        if ((o[0] === Opcodes.call /* || o[0] === Opcodes.return_call */) && o[1] >= importedFuncs.length) {
+          const n = o[1] - importDelta;
+          o = [ Opcodes.call ];
+          unsignedLEB128_into(n, o);
+        }
 
-      // encode call indirect ops as types from info
-      if (o[0] === Opcodes.call_indirect) {
-        o = [...o];
-        const params = [];
-        for (let i = 0; i < o[1]; i++) {
-          params.push(valtypeBinary, Valtype.i32);
+        // encode call indirect ops as types from info
+        if (o[0] === Opcodes.call_indirect) {
+          o = [...o];
+          const params = [];
+          for (let i = 0; i < o[1]; i++) {
+            params.push(valtypeBinary, Valtype.i32);
+          }
+
+          let returns = [ valtypeBinary, Valtype.i32 ];
+          if (o.at(-1) === 'no_type_return') {
+            o.pop();
+            returns = [ valtypeBinary ];
+          }
+
+          o[1] = getType(params, returns);
         }
 
-        let returns = [ valtypeBinary, Valtype.i32 ];
-        if (o.at(-1) === 'no_type_return') {
-          o.pop();
-          returns = [ valtypeBinary ];
+        for (let j = 0; j < o.length; j++) {
+          const x = o[j];
+          if (x == null || !(x <= 0xff)) continue;
+          wasm.push(x);
         }
 
-        o[1] = getType(params, returns);
+        wasmNonFlat.push(o);
       }
 
-      for (let j = 0; j < o.length; j++) {
-        const x = o[j];
-        if (x == null || !(x <= 0xff)) continue;
-        wasm.push(x);
-      }
+      x.assembled = { localDecl, wasm, wasmNonFlat };
+    } else {
+      for (let i = 0; i < x.wasm.length; i++) {
+        let o = x.wasm[i];
+        const op = o[0];
+
+        // encode local/global ops as unsigned leb128 from raw number
+        if (
+          (op >= Opcodes.local_get && op <= Opcodes.global_set) &&
+          o[1] > 127
+        ) {
+          wasm.push(op);
+          unsignedLEB128_into(o[1], wasm);
+          continue;
+        }
 
-      if (makeAssembled) wasmNonFlat.push(o);
-    }
-    // time('  wasm transform');
+        // encode f64.const ops as ieee754 from raw number
+        if (op === Opcodes.f64_const) {
+          wasm.push(op);
+          ieee754_binary64_into(o[1], wasm);
+          continue;
+        }
 
-    if (makeAssembled) {
-      x.assembled = { localDecl, wasm, wasmNonFlat };
-    }
+        // encode call ops as unsigned leb128 from raw number
+        if ((op === Opcodes.call /* || o[0] === Opcodes.return_call */) && o[1] >= importedFuncs.length) {
+          wasm.push(op);
+          unsignedLEB128_into(o[1] - importDelta, wasm);
+          continue;
+        }
+
+        // encode call indirect ops as types from info
+        if (op === Opcodes.call_indirect) {
+          const params = [];
+          for (let i = 0; i < o[1]; i++) {
+            params.push(valtypeBinary, Valtype.i32);
+          }
 
-    let out = unsignedLEB128(declCount)
-      .concat(localDecl, wasm, Opcodes.end);
+          let returns = [ valtypeBinary, Valtype.i32 ];
+          if (o.at(-1) === 'no_type_return') {
+            returns = [ valtypeBinary ];
+          }
+
+          wasm.push(op, getType(params, returns), o[2]);
+          continue;
+        }
 
-    codeSection.push(
-      ...unsignedLEB128(out.length),
-      ...out
-    );
+        for (let j = 0; j < o.length; j++) {
+          const x = o[j];
+          if (x == null || !(x <= 0xff)) continue;
+          wasm.push(x);
+        }
+      }
+    }
 
-    // time('  finish');
+    if (wasm.length > 100000) {
+      // slow path for handling large arrays which break v8 due to running out of stack size
+      const out = unsignedLEB128(declCount)
+        .concat(localDecl, wasm, Opcodes.end);
+      codeSection = codeSection.concat(unsignedLEB128(out.length), out);
+    } else {
+      codeSection.push(
+        ...unsignedLEB128(unsignedLEB128_length(declCount) + localDecl.length + wasm.length + 1),
+        ...unsignedLEB128(declCount),
+        ...localDecl,
+        ...wasm,
+        Opcodes.end
+      );
+    }
+
+    // globalThis.progress?.(`${i}/${funcs.length}`);
   }
 
   codeSection.unshift(...unsignedLEB128(funcs.length, codeSection));
@@ -368,10 +420,14 @@ export default (funcs, globals, tags, pages, data, noTreeshake = false) => {
         dataSection.push(0x01);
       }
 
-      dataSection.push(
-        ...unsignedLEB128(x.bytes.length),
-        ...x.bytes
-      );
+      if (x.bytes.length > 100000) {
+        codeSection = codeSection.concat(unsignedLEB128(x.bytes.length), x.bytes);
+      } else {
+        dataSection.push(
+          ...unsignedLEB128(x.bytes.length),
+          ...x.bytes
+        );
+      }
     }
 
     dataSection.unshift(...unsignedLEB128(data.length, dataSection));

package/compiler/codegen.js

@@ -6457,8 +6457,6 @@ const generateFunc = (scope, decl, forceNoExpr = false) => {
     return [ func, out ];
   }
 
-  globalThis.progress?.(null, ' ' + name);
-
   const params = decl.params ?? [];
 
   // TODO: share scope/locals between !!!
@@ -6762,15 +6760,18 @@ const generateFunc = (scope, decl, forceNoExpr = false) => {
   return [ func, out ];
 };
 
+let largestBlockBody = 0;
 const generateBlock = (scope, decl) => {
   let out = [];
 
   scope.inferTree ??= [];
   scope.inferTree.push(decl);
 
-  let j = 0;
-  for (let i = 0; i < decl.body.length; i++) {
+  let len = decl.body.length, j = 0;
+  if (len > largestBlockBody) largestBlockBody = len;
+  for (let i = 0; i < len; i++) {
     const x = decl.body[i];
+    if (len >= largestBlockBody) globalThis.progress?.(`${i}/${len}`);
     if (isEmptyNode(x)) continue;
 
     if (j++ > 0) out.push([ Opcodes.drop ]);
@@ -6946,6 +6947,7 @@ export default program => {
   data = [];
   currentFuncIndex = importedFuncs.length;
   typeswitchDepth = 0;
+  largestBlockBody = 0;
   usedTypes = new Set([ TYPES.empty, TYPES.undefined, TYPES.number, TYPES.boolean, TYPES.function ]);
   coctc = new Map();
   globalInfer = new Map();

package/compiler/embedding.js

@@ -8,17 +8,4 @@ export const number = (n, valtype = valtypeBinary) => {
   signedLEB128_into(n, out);
 
   return out;
-};
-
-export const enforceOneByte = arr => [ arr[0] ?? 0 ];
-export const enforceTwoBytes = arr => [ arr[0] ?? 0, arr[1] ?? 0 ];
-export const enforceFourBytes = arr => [ arr[0] ?? 0, arr[1] ?? 0, arr[2] ?? 0, arr[3] ?? 0 ];
-export const enforceEightBytes = arr => [ arr[0] ?? 0, arr[1] ?? 0, arr[2] ?? 0, arr[3] ?? 0, arr[4] ?? 0, arr[5] ?? 0, arr[6] ?? 0, arr[7] ?? 0 ];
-
-export const i32x4 = (a, b, c, d) => [ [
-  ...Opcodes.v128_const,
-  ...enforceFourBytes(signedLEB128(a)),
-  ...enforceFourBytes(signedLEB128(b)),
-  ...enforceFourBytes(signedLEB128(c)),
-  ...enforceFourBytes(signedLEB128(d))
-] ];
+};

package/compiler/encoding.js

@@ -64,6 +64,22 @@ export const unsignedLEB128 = n => {
   return buffer;
 };
 
+export const unsignedLEB128_length = n => {
+  if (n < 0) n = n >>> 0;
+  if (n <= 127) return 1;
+  if (n <= 16383) return 2;
+  if (n <= 2097151) return 3;
+  if (n <= 268435455) return 4;
+  if (n <= 34359738367) return 5;
+
+  let length = 0;
+  do {
+    length++;
+    n >>>= 7;
+  } while (n > 0);
+  return length;
+};
+
 export const big_signedLEB128 = n => {
   // just input for small numbers (for perf as common)
   if (n >= 0n && n <= 63n) return [ Number(n) ];

package/compiler/index.js

@@ -35,7 +35,7 @@ const progressStart = msg => {
 
   const log = (extra, after) => {
     const pre = extra ? `${extra}` : spinner[spin++ % 4];
-    process.stdout.write(`\r\u001b[90m${' '.repeat(10 - pre.length)}${pre}  ${msg}${after ?? ''}\u001b[0m`);
+    process.stdout.write(`\r\u001b[90m${' '.repeat(120)}\r${' '.repeat(12 - pre.length)}${pre}  ${msg}${after ?? ''}\u001b[0m`);
   };
   log();
 
@@ -48,7 +48,7 @@ const progressDone = (msg, start) => {
   clearInterval(progressInterval);
 
   const timeStr = (performance.now() - start).toFixed(0);
-  console.log(`\r${' '.repeat(50)}\r\u001b[90m${' '.repeat(8 - timeStr.length)}${timeStr}ms\u001b[0m  \u001b[92m${msg}\u001b[0m`);
+  console.log(`\r${' '.repeat(120)}\r\u001b[90m${' '.repeat(10 - timeStr.length)}${timeStr}ms\u001b[0m  \u001b[92m${msg}\u001b[0m`);
   progressLines++;
 };
 const progressClear = () => {
@@ -66,9 +66,8 @@ export default (code, module = undefined) => {
   let target = Prefs.target ?? 'wasm';
   if (Prefs.native) target = 'native';
 
-  const logProgress = Prefs.profileCompiler || (target === 'native' && !Prefs.native && globalThis.file);
-
-  let outFile = Prefs.o;
+  const outFile = Prefs.o;
+  const logProgress = Prefs.profileCompiler || (Prefs.target && outFile && !Prefs.native && globalThis.file);
 
   globalThis.valtype = Prefs.valtype ?? 'f64';
   globalThis.valtypeBinary = Valtype[valtype];
@@ -214,6 +213,12 @@ export default (code, module = undefined) => {
   if (target === 'wasm' && outFile) {
     fs.writeFileSync(outFile, Buffer.from(wasm));
 
+    if (logProgress) {
+      const total = performance.now();
+      progressClear();
+      console.log(`\u001b[90m[${total.toFixed(0)}ms]\u001b[0m \u001b[32mcompiled ${globalThis.file} \u001b[90m->\u001b[0m \u001b[92m${outFile}\u001b[90m (${(fs.statSync(outFile).size / 1000).toFixed(1)}KB)\u001b[0m`);
+    }
+
     if (process.version) process.exit();
   }
 
@@ -227,6 +232,12 @@ export default (code, module = undefined) => {
       console.log(c);
     }
 
+    if (logProgress) {
+      const total = performance.now();
+      progressClear();
+      console.log(`\u001b[90m[${total.toFixed(0)}ms]\u001b[0m \u001b[32mcompiled ${globalThis.file} \u001b[90m->\u001b[0m \u001b[92m${outFile}\u001b[90m (${(fs.statSync(outFile).size / 1000).toFixed(1)}KB)\u001b[0m`);
+    }
+
     if (process.version) process.exit();
   }
 
@@ -260,35 +271,33 @@ export default (code, module = undefined) => {
 
     if (logProgress) progressStart(`compiled C to native (using ${compiler})`, t5);
 
-    if (process.version) {
-      if (Prefs.native) {
-        const cleanup = () => {
-          try {
-            fs.unlinkSync(outFile);
-          } catch {}
-        };
-
-        process.on('exit', cleanup);
-        process.on('beforeExit', cleanup);
-        process.on('SIGINT', () => {
-          cleanup();
-          process.exit();
-        });
-
-        const runArgs = process.argv.slice(2).filter(x => !x.startsWith('-'));
+    if (Prefs.native) {
+      const cleanup = () => {
         try {
-          execSync([ outFile, ...runArgs.slice(1) ].join(' '), { stdio: 'inherit' });
+          fs.unlinkSync(outFile);
         } catch {}
-      }
+      };
 
-      if (logProgress) {
-        const total = performance.now();
-        progressClear();
-        console.log(`\u001b[90m[${total.toFixed(0)}ms]\u001b[0m \u001b[32mcompiled ${globalThis.file} \u001b[90m->\u001b[0m \u001b[92m${outFile}\u001b[90m (${(fs.statSync(outFile).size / 1000).toFixed(1)}KB)\u001b[0m`);
-      }
+      process.on('exit', cleanup);
+      process.on('beforeExit', cleanup);
+      process.on('SIGINT', () => {
+        cleanup();
+        process.exit();
+      });
+
+      const runArgs = process.argv.slice(2).filter(x => !x.startsWith('-'));
+      try {
+        execSync([ outFile, ...runArgs.slice(1) ].join(' '), { stdio: 'inherit' });
+      } catch {}
+    }
 
-      process.exit();
+    if (logProgress) {
+      const total = performance.now();
+      progressClear();
+      console.log(`\u001b[90m[${total.toFixed(0)}ms]\u001b[0m \u001b[32mcompiled ${globalThis.file} \u001b[90m->\u001b[0m \u001b[92m${outFile}\u001b[90m (${(fs.statSync(outFile).size / 1000).toFixed(1)}KB)\u001b[0m`);
     }
+
+    process.exit();
   }
 
   return out;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "porffor",
   "description": "a basic experimental wip aot optimizing js -> wasm engine/compiler/runtime in js",
-  "version": "0.50.20",
+  "version": "0.50.22",
   "author": "CanadaHonk",
   "license": "MIT",
   "scripts": {},

package/runner/index.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import fs from 'node:fs';
-globalThis.version = '0.50.20';
+globalThis.version = '0.50.22';
 
 // deno compat
 if (typeof process === 'undefined' && typeof Deno !== 'undefined') {
@@ -94,8 +94,6 @@ if (['precompile', 'run', 'wasm', 'native', 'c', 'flamegraph', 'hotlines', 'debu
   }
 }
 
-globalThis.file = file;
-
 let source = '', printOutput = false;
 if (process.argv.length >= 4) {
   let evalIndex = process.argv.indexOf('-e');
@@ -127,6 +125,20 @@ if (process.argv.length >= 4) {
   }
 }
 
+if (file.startsWith('https://')) { // https only :)
+  // rce warning, make user confirm (disabled)
+  // const rl = (await import('readline')).createInterface({ input: process.stdin, output: process.stdout });
+  // const ans = await new Promise(resolve => rl.question(`\u001b[1mAre you sure you want to download this URL:\u001b[0m ${file} (y/n)? `, ans => {
+  //   rl.close();
+  //   resolve(ans);
+  // }));
+  // if (ans.toLowerCase()[0] !== 'y') process.exit();
+
+  source = await (await fetch(file)).text();
+}
+
+globalThis.file = file;
+
 if (!file && !source) {
   if (process.argv.includes('-v') || process.argv.includes('--version')) {
     // just print version