porffor 0.2.0-2265539 → 0.2.0-22b3092

package/compiler/index.js

@@ -1,8 +1,8 @@
 import { underline, bold, log } from './log.js';
 import parse from './parse.js';
-import codeGen from './codeGen.js';
+import codegen from './codegen.js';
 import opt from './opt.js';
-import produceSections from './sections.js';
+import assemble from './assemble.js';
 import decompile from './decompile.js';
 import toc from './2c.js';
 import Prefs from './prefs.js';
@@ -26,29 +26,29 @@ const logFuncs = (funcs, globals, exceptions) => {
   console.log();
 };
 
-const writeFileSync = (typeof process?.version !== 'undefined' ? (await import('node:fs')).writeFileSync : undefined);
+const fs = (typeof process?.version !== 'undefined' ? (await import('node:fs')) : undefined);
 const execSync = (typeof process?.version !== 'undefined' ? (await import('node:child_process')).execSync : undefined);
 
 export default (code, flags) => {
   const t0 = performance.now();
   const program = parse(code, flags);
-  if (flags.includes('info')) console.log(`1. parsed in ${(performance.now() - t0).toFixed(2)}ms`);
+  if (Prefs.profileCompiler) console.log(`1. parsed in ${(performance.now() - t0).toFixed(2)}ms`);
 
   const t1 = performance.now();
-  const { funcs, globals, tags, exceptions, pages, data } = codeGen(program);
-  if (flags.includes('info')) console.log(`2. generated code in ${(performance.now() - t1).toFixed(2)}ms`);
+  const { funcs, globals, tags, exceptions, pages, data } = codegen(program);
+  if (Prefs.profileCompiler) console.log(`2. generated code in ${(performance.now() - t1).toFixed(2)}ms`);
 
   if (Prefs.funcs) logFuncs(funcs, globals, exceptions);
 
   const t2 = performance.now();
   opt(funcs, globals, pages, tags, exceptions);
-  if (flags.includes('info')) console.log(`3. optimized code in ${(performance.now() - t2).toFixed(2)}ms`);
+  if (Prefs.profileCompiler) console.log(`3. optimized in ${(performance.now() - t2).toFixed(2)}ms`);
 
   if (Prefs.optFuncs) logFuncs(funcs, globals, exceptions);
 
   const t3 = performance.now();
-  const sections = produceSections(funcs, globals, tags, pages, data, flags);
-  if (flags.includes('info')) console.log(`4. produced sections in ${(performance.now() - t3).toFixed(2)}ms`);
+  const wasm = assemble(funcs, globals, tags, pages, data, flags);
+  if (Prefs.profileCompiler) console.log(`4. assembled in ${(performance.now() - t3).toFixed(2)}ms`);
 
   if (Prefs.allocLog) {
     const wasmPages = Math.ceil((pages.size * pageSize) / 65536);
@@ -57,13 +57,13 @@ export default (code, flags) => {
     console.log([...pages.keys()].map(x => `\x1B[36m - ${x}\x1B[0m`).join('\n') + '\n');
   }
 
-  const out = { wasm: sections, funcs, globals, tags, exceptions, pages, data };
+  const out = { wasm, funcs, globals, tags, exceptions, pages, data };
 
   const target = Prefs.target ?? 'wasm';
   const outFile = Prefs.o;
 
   if (target === 'wasm' && outFile) {
-    writeFileSync(outFile, Buffer.from(sections));
+    fs.writeFileSync(outFile, Buffer.from(wasm));
 
     if (process.version) process.exit();
   }
@@ -73,7 +73,7 @@ export default (code, flags) => {
     out.c = c;
 
     if (outFile) {
-      writeFileSync(outFile, c);
+      fs.writeFileSync(outFile, c);
     } else {
       console.log(c);
     }
@@ -88,17 +88,19 @@ export default (code, flags) => {
     if (compiler === 'zig') compiler = [ 'zig', 'cc' ];
       else compiler = [ compiler ];
 
-    const tmpfile = 'tmp.c';
+    const tmpfile = 'porffor_tmp.c';
     // const args = [ compiler, tmpfile, '-o', outFile ?? (process.platform === 'win32' ? 'out.exe' : 'out'), '-' + cO, '-march=native', '-s', '-fno-unwind-tables', '-fno-asynchronous-unwind-tables', '-ffunction-sections', '-fdata-sections', '-Wl', '-fno-ident', '-fno-exceptions', '-ffast-math' ];
     // const args = [ ...compiler, tmpfile, '-o', outFile ?? (process.platform === 'win32' ? 'out.exe' : 'out'), '-' + cO, '-march=native', '-s', '-ffast-math', '-fno-exceptions', '-target', 'x86_64-linux' ];
     const args = [ ...compiler, tmpfile, '-o', outFile ?? (process.platform === 'win32' ? 'out.exe' : 'out'), '-' + cO, '-march=native', '-s', '-ffast-math', '-fno-exceptions' ];
 
     const c = toc(out);
-    writeFileSync(tmpfile, c);
+    fs.writeFileSync(tmpfile, c);
 
     // obvious command escape is obvious
     execSync(args.join(' '), { stdio: 'inherit' });
 
+    fs.unlinkSync(tmpfile);
+
     if (process.version) process.exit();
   }
 

package/compiler/log.js

@@ -5,11 +5,14 @@ export const bold = x => `\u001b[1m${x}\u001b[0m`;
 const areaColors = {
   codegen: [ 20, 80, 250 ],
   opt: [ 250, 20, 80 ],
-  sections: [ 20, 250, 80 ],
+  assemble: [ 20, 250, 80 ],
   alloc: [ 250, 250, 20 ],
-  parser: [ 240, 240, 240 ],
-  '2c': [ 20, 250, 250 ]
+  parse: [ 240, 240, 240 ],
+  '2c': [ 20, 250, 250 ],
+  wrap: [ 250, 100, 20 ]
 };
 
+// for (const x in areaColors) console.log(rgb(areaColors[x][0], areaColors[x][1], areaColors[x][2], x));
+
 export const log = (area, ...args) => console.log(`\u001b[90m[\u001b[0m${rgb(...areaColors[area], area)}\u001b[90m]\u001b[0m`, ...args);
 log.warning = (area, ...args) => log(area, '\u001b[93m' + args[0], ...args.slice(1), '\u001b[0m');

package/compiler/opt.js

@@ -100,8 +100,9 @@ export default (funcs, globals, pages, tags, exceptions) => {
 
   if (Prefs.optInlineOnly) return;
 
-  const tagUse = tags.reduce((acc, x) => { acc[x.idx] = 0; return acc; }, {});
-  const exceptionUse = exceptions.reduce((acc, _, i) => { acc[i] = 0; return acc; }, {});
+  // todo: this breaks exceptions after due to indexes not being adjusted
+  // const tagUse = tags.reduce((acc, x) => { acc[x.idx] = 0; return acc; }, {});
+  // const exceptionUse = exceptions.reduce((acc, _, i) => { acc[i] = 0; return acc; }, {});
 
   // wasm transform pass
   for (const f of funcs) {
@@ -109,7 +110,7 @@ export default (funcs, globals, pages, tags, exceptions) => {
 
     const lastType = f.locals['#last_type'];
 
-    let runs = 2; // how many by default? add arg?
+    let runs = (+Prefs.optWasmRuns) || 2; // todo: how many by default?
     while (runs > 0) {
       runs--;
 
@@ -131,12 +132,12 @@ export default (funcs, globals, pages, tags, exceptions) => {
         if (inst[0] === Opcodes.local_get) getCount[inst[1]]++;
         if (inst[0] === Opcodes.local_set || inst[0] === Opcodes.local_tee) setCount[inst[1]]++;
 
-        if (inst[0] === Opcodes.throw) {
-          tagUse[inst[1]]++;
+        // if (inst[0] === Opcodes.throw) {
+        //   tagUse[inst[1]]++;
 
-          const exceptId = read_signedLEB128(wasm[i - 1].slice(1));
-          exceptionUse[exceptId]++;
-        }
+        //   const exceptId = read_signedLEB128(wasm[i - 1].slice(1));
+        //   exceptionUse[exceptId]++;
+        // }
 
         if (inst[0] === Opcodes.block) {
           // remove unneeded blocks (no brs inside)
@@ -171,14 +172,14 @@ export default (funcs, globals, pages, tags, exceptions) => {
           }
         }
 
-        if (inst[inst.length - 1] === 'string_only' && !pages.hasString) {
+        if (inst[inst.length - 1] === 'string_only' && !pages.hasAnyString) {
           // remove this inst
           wasm.splice(i, 1);
           if (i > 0) i--;
           inst = wasm[i];
         }
 
-        if (inst[inst.length - 1] === 'string_only|start' && !pages.hasString) {
+        if (inst[inst.length - 1] === 'string_only|start' && !pages.hasAnyString) {
           let j = i;
           for (; j < wasm.length; j++) {
             const op = wasm[j];
@@ -553,18 +554,18 @@ export default (funcs, globals, pages, tags, exceptions) => {
     }
   }
 
-  for (const x in tagUse) {
-    if (tagUse[x] === 0) {
-      const el = tags.find(y => y.idx === x);
-      tags.splice(tags.indexOf(el), 1);
-    }
-  }
-
-  for (const x of Object.keys(exceptionUse).sort((a, b) => b - a)) {
-    if (exceptionUse[x] === 0) {
-      exceptions.splice(+x, 1);
-    }
-  }
+  // for (const x in tagUse) {
+  //   if (tagUse[x] === 0) {
+  //     const el = tags.find(y => y.idx === x);
+  //     tags.splice(tags.indexOf(el), 1);
+  //   }
+  // }
+
+  // for (const x of Object.keys(exceptionUse).sort((a, b) => b - a)) {
+  //   if (exceptionUse[x] === 0) {
+  //     exceptions.splice(+x, 1);
+  //   }
+  // }
 
   // return funcs;
 };

package/compiler/parse.js

@@ -20,37 +20,43 @@ globalThis.typedInput = types && Prefs.optTypes;
 // - hermes-parser
 // - @babel/parser
 
-let parser, parse;
+globalThis.parser = '';
+let parse;
 const loadParser = async (fallbackParser = 'acorn', forceParser) => {
   parser = forceParser ?? process.argv.find(x => x.startsWith('-parser='))?.split('=')?.[1] ?? fallbackParser;
-  0, { parse } = (await import((globalThis.document ? 'https://esm.sh/' : '') + parser));
+  0, { parse } = (await import((globalThis.document || globalThis.Deno ? 'https://esm.sh/' : '') + parser));
 };
 globalThis._porf_loadParser = loadParser;
 await loadParser(types ? '@babel/parser' : undefined);
 
-if (types && !['@babel/parser', 'hermes-parser'].includes(parser)) log.warning('parser', `passed -types with a parser (${parser}) which does not support`);
+if (types && !['@babel/parser', 'hermes-parser'].includes(parser)) log.warning('parse', `passed -parse-types with a parser (${parser}) which does not support`);
 
 export default (input, flags) => {
-  const ast = parse(input, {
-    // acorn
-    ecmaVersion: 'latest',
-
-    // meriyah
-    next: true,
-    module: flags.includes('module'),
-    webcompat: true,
-
-    // babel
-    plugins: types ? ['estree', 'typescript'] : ['estree'],
-
-    // multiple
-    sourceType: flags.includes('module') ? 'module' : 'script',
-    ranges: false,
-    tokens: false,
-    comments: false,
-  });
-
-  if (ast.type === 'File') return ast.program;
-
-  return ast;
+  try {
+    const ast = parse(input, {
+      // acorn
+      ecmaVersion: 'latest',
+
+      // meriyah
+      next: true,
+      module: flags.includes('module'),
+      webcompat: true,
+
+      // babel
+      plugins: types ? ['estree', 'typescript'] : ['estree'],
+
+      // multiple
+      sourceType: flags.includes('module') ? 'module' : 'script',
+      ranges: false,
+      tokens: false,
+      comments: false,
+    });
+
+    if (ast.type === 'File') return ast.program;
+
+    return ast;
+  } catch (e) {
+    // normalize error class thrown by 3rd party parsers
+    throw new SyntaxError(e.message, { cause: e });
+  }
 };

package/compiler/precompile.js

@@ -1,4 +1,5 @@
 import { Opcodes } from './wasmSpec.js';
+import { TYPES } from './types.js';
 
 import fs from 'node:fs';
 import { join } from 'node:path';
@@ -6,33 +7,21 @@ import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
 
-const TYPES = {
-  number: 0x00,
-  boolean: 0x01,
-  string: 0x02,
-  undefined: 0x03,
-  object: 0x04,
-  function: 0x05,
-  symbol: 0x06,
-  bigint: 0x07,
-
-  // these are not "typeof" types but tracked internally
-  _array: 0x10,
-  _regexp: 0x11,
-  _bytestring: 0x12
-};
-
 // import porfParse from './parse.js';
 // import porfCodegen from './codeGen.js';
 
 const argv = process.argv.slice();
 
 const compile = async (file, [ _funcs, _globals ]) => {
-  const source = fs.readFileSync(file, 'utf8');
-  const first = source.slice(0, source.indexOf('\n'));
+  let source = fs.readFileSync(file, 'utf8');
+  let first = source.slice(0, source.indexOf('\n'));
+
+  if (first.startsWith('export default')) {
+    source = (await import(file)).default();
+    first = source.slice(0, source.indexOf('\n'));
+  }
 
-  let args = ['-bytestring'];
-  if (file.endsWith('.ts')) args.push('-parse-types', '-opt-types');
+  let args = ['-bytestring', '-todo-time=compile', '-no-aot-pointer-opt', '-scoped-page-names', '-parse-types', '-opt-types'];
   if (first.startsWith('// @porf')) {
     args = args.concat(first.slice('// @porf '.length).split(' '));
   }
@@ -51,7 +40,12 @@ const compile = async (file, [ _funcs, _globals ]) => {
 
   const exports = funcs.filter(x => x.export);
   for (const x of exports) {
-    if (x.data) x.data = x.data.map(x => data[x]);
+    if (x.data) {
+      x.data = x.data.map(x => data[x]);
+      for (const y in x.data) {
+        x.data[y].offset -= x.data[0].offset;
+      }
+    }
     if (x.exceptions) x.exceptions = x.exceptions.map(x => {
       const obj = exceptions[x];
       if (obj) obj.exceptId = x;
@@ -83,7 +77,7 @@ const compile = async (file, [ _funcs, _globals ]) => {
         if (!pageName || allocated.has(pageName)) continue;
         allocated.add(pageName);
 
-        y.splice(0, 10, 'alloc', pageName, x.pages.get(pageName).type);
+        y.splice(0, 10, 'alloc', pageName, x.pages.get(pageName).type, valtypeBinary);
         // y.push(x.pages.get(pageName));
       }
     }
@@ -110,16 +104,16 @@ import { number } from './embedding.js';
 
 export const BuiltinFuncs = function() {
 ${funcs.map(x => `  this.${x.name} = {
-    wasm: (scope, { allocPage, builtin }) => ${JSON.stringify(x.wasm.filter(x => x.length)).replace(/\["alloc","(.*?)","(.*?)"\]/g, (_, reason, type) => `...number(allocPage(scope, '${reason}', '${type}') * pageSize, ${valtypeBinary})`).replace(/\[16,"(.*?)"]/g, (_, name) => `[16, builtin('${name}')]`)},
+    wasm: (scope, { allocPage, builtin }) => ${JSON.stringify(x.wasm.filter(x => x.length && x[0] != null)).replace(/\["alloc","(.*?)","(.*?)",(.*?)\]/g, (_, reason, type, valtype) => `...number(allocPage(scope, '${reason}', '${type}') * pageSize, ${valtype})`).replace(/\[16,"(.*?)"]/g, (_, name) => `[16, builtin('${name}')]`)},
     params: ${JSON.stringify(x.params)},
     typedParams: true,
     returns: ${JSON.stringify(x.returns)},
-    typedReturns: true,
+    ${x.returnType != null ? `returnType: ${JSON.stringify(x.returnType)}` : 'typedReturns: true'},
     locals: ${JSON.stringify(Object.values(x.locals).slice(x.params.length).map(x => x.type))},
     localNames: ${JSON.stringify(Object.keys(x.locals))},
 ${x.data && x.data.length > 0 ? `    data: ${JSON.stringify(x.data)},` : ''}
 ${x.exceptions && x.exceptions.length > 0 ? `    exceptions: ${JSON.stringify(x.exceptions)},` : ''}
-  };`.replaceAll('\n\n', '\n')).join('\n')}
+  };`.replaceAll('\n\n', '\n').replaceAll('\n\n', '\n')).join('\n')}
 };`;
 };
 

package/compiler/prefs.js

@@ -1,6 +1,6 @@
-const onByDefault = [ 'bytestring' ];
+const onByDefault = [ 'bytestring', 'aotPointerOpt' ];
 
-const cache = {};
+let cache = {};
 const obj = new Proxy({}, {
   get(_, p) {
     // intentionally misses with undefined values cached

package/compiler/prototype.js

@@ -3,23 +3,7 @@ import { number } from "./embedding.js";
 import { unsignedLEB128 } from "./encoding.js";
 import { UNDEFINED } from "./builtins.js";
 import Prefs from './prefs.js';
-
-// todo: do not duplicate this
-const TYPES = {
-  number: 0x00,
-  boolean: 0x01,
-  string: 0x02,
-  undefined: 0x03,
-  object: 0x04,
-  function: 0x05,
-  symbol: 0x06,
-  bigint: 0x07,
-
-  // these are not "typeof" types but tracked internally
-  _array: 0x10,
-  _regexp: 0x11,
-  _bytestring: 0x12
-};
+import { TYPES } from './types.js';
 
 // todo: turn these into built-ins once arrays and these become less hacky
 
@@ -150,7 +134,7 @@ export const PrototypeFuncs = function() {
     shift: (pointer, length) => [
       // if length == 0, noop
       ...length.getCachedI32(),
-      Opcodes.i32_eqz,
+      [ Opcodes.i32_eqz ],
       [ Opcodes.if, Blocktype.void ],
       ...number(UNDEFINED),
       [ Opcodes.br, 1 ],

package/compiler/types.js

@@ -0,0 +1,37 @@
+export const TYPES = {
+  number: 0x00,
+  boolean: 0x01,
+  string: 0x02,
+  undefined: 0x03,
+  object: 0x04,
+  function: 0x05,
+  symbol: 0x06,
+  bigint: 0x07
+};
+
+export const TYPE_NAMES = {
+  [TYPES.number]: 'Number',
+  [TYPES.boolean]: 'Boolean',
+  [TYPES.string]: 'String',
+  [TYPES.undefined]: 'undefined',
+  [TYPES.object]: 'Object',
+  [TYPES.function]: 'Function',
+  [TYPES.symbol]: 'Symbol',
+  [TYPES.bigint]: 'BigInt'
+};
+
+export const INTERNAL_TYPE_BASE = 0x10;
+let internalTypeIndex = INTERNAL_TYPE_BASE;
+const registerInternalType = name => {
+  const n = internalTypeIndex++;
+  TYPES['_' + name.toLowerCase()] = n;
+  TYPE_NAMES[n] = name;
+};
+
+// note: when adding a new internal type, please also add a deserializer to wrap.js
+// (it is okay to add a throw todo deserializer for wips)
+
+registerInternalType('Array');
+registerInternalType('RegExp');
+registerInternalType('ByteString');
+registerInternalType('Date');

package/compiler/wasmSpec.js

@@ -1,4 +1,13 @@
-import { enumify } from "../util/enum.js";
+const enumify = (...args) => {
+  const obj = {};
+
+  for (let i = 0; i < args.length; i++) {
+    obj[i] = args[i];
+    obj[args[i]] = i;
+  }
+
+  return obj;
+};
 
 export const Section = enumify('custom', 'type', 'import', 'func', 'table', 'memory', 'global', 'export', 'start', 'element', 'code', 'data', 'data_count', 'tag');
 export const ExportDesc = enumify('func', 'table', 'mem', 'global', 'tag');
@@ -32,17 +41,16 @@ export const Opcodes = {
   throw: 0x08,
   rethrow: 0x09,
 
-  call: 0x10,
-  call_indirect: 0x11,
-  return_call: 0x12,
-  return_call_indirect: 0x13,
-
   end: 0x0b,
   br: 0x0c,
   br_if: 0x0d,
   br_table: 0x0e,
   return: 0x0f,
+
   call: 0x10,
+  call_indirect: 0x11,
+  return_call: 0x12,
+  return_call_indirect: 0x13,
 
   drop: 0x1a,
 
@@ -62,13 +70,21 @@ export const Opcodes = {
   i32_load16_s: 0x2e,
   i32_load16_u: 0x2f,
 
-  i32_store8: 0x3a,
-  i32_store16: 0x3b,
+  i64_load8_s: 0x30,
+  i64_load8_u: 0x31,
+  i64_load16_s: 0x32,
+  i64_load16_u: 0x33,
 
   i32_store: 0x36,
   i64_store: 0x37,
   f64_store: 0x39,
 
+  i32_store8: 0x3a,
+  i32_store16: 0x3b,
+
+  i64_store8: 0x3c,
+  i64_store16: 0x3d,
+
   memory_grow: 0x40,
 
   i32_const: 0x41,
@@ -100,6 +116,8 @@ export const Opcodes = {
   i32_shl: 0x74,
   i32_shr_s: 0x75,
   i32_shr_u: 0x76,
+  i32_rotl: 0x77,
+  i32_rotr: 0x78,
 
   i64_eqz: 0x50,
   i64_eq: 0x51,
@@ -123,6 +141,7 @@ export const Opcodes = {
   i64_shr_s: 0x87,
   i64_shr_u: 0x88,
   i64_rotl: 0x89,
+  i64_rotr: 0x8a,
 
   f64_eq: 0x61,
   f64_ne: 0x62,