porffor 0.1.1 → 0.2.0-c6c8c81

package/compiler/wrap.js

@@ -4,7 +4,8 @@ import decompile from './decompile.js';
 
 const bold = x => `\u001b[1m${x}\u001b[0m`;
 
-const typeBase = 0xffffffffffff0;
+const typeBase = 0x00;
+const internalTypeBase = 0x10;
 const TYPES = {
   [typeBase]: 'number',
   [typeBase + 1]: 'boolean',
@@ -16,7 +17,8 @@ const TYPES = {
   [typeBase + 7]: 'bigint',
 
   // internal
-  [typeBase + 8]: '_array'
+  [internalTypeBase]: '_array',
+  [internalTypeBase + 1]: '_regexp'
 };
 
 export default async (source, flags = [ 'module' ], customImports = {}, print = str => process.stdout.write(str)) => {
@@ -37,7 +39,6 @@ export default async (source, flags = [ 'module' ], customImports = {}, print =
     '': {
       p: valtype === 'i64' ? i => print(Number(i).toString()) : i => print(i.toString()),
       c: valtype === 'i64' ? i => print(String.fromCharCode(Number(i))) : i => print(String.fromCharCode(i)),
-      a: c => { if (!Number(c)) throw new Error(`assert failed`); },
       t: _ => performance.now(),
       ...customImports
     }
@@ -64,11 +65,15 @@ export default async (source, flags = [ 'module' ], customImports = {}, print =
 
     exports[func.name] = function() {
       try {
-        const ret = exp.apply(this, arguments);
+        const _ret = exp.apply(this, arguments);
 
-        if (ret >= typeBase && ret <= typeBase + 8) return ret > (typeBase + 7) ? 'object' : TYPES[ret];
+        if (_ret == null) return undefined;
 
-        switch (TYPES[func.returnType]) {
+        const [ ret, type ] = _ret;
+
+        // if (ret >= typeBase && ret <= typeBase + 8) return ret > (typeBase + 7) ? 'object' : TYPES[ret];
+
+        switch (TYPES[type]) {
           case 'boolean': return Boolean(ret);
           case 'undefined': return undefined;
           case 'object': return ret === 0 ? null : {};
@@ -90,6 +95,15 @@ export default async (source, flags = [ 'module' ], customImports = {}, print =
             return Array.from(new Uint16Array(memory.buffer, pointer + 4, length)).map(x => String.fromCharCode(x)).join('');
           }
 
+          case 'function': {
+            // wasm func index, including all imports
+            const func = funcs.find(x => (x.originalIndex ?? x.index) === ret);
+            if (!func) return ret;
+
+            // make fake empty func for repl/etc
+            return {[func.name]() {}}[func.name];
+          }
+
           default: return ret;
         }
       } catch (e) {

package/package.json

@@ -1,12 +1,17 @@
 {
   "name": "porffor",
   "description": "a basic experimental wip aot optimizing js -> wasm engine/compiler/runtime in js",
-  "version": "0.1.1",
+  "version": "0.2.0-c6c8c81",
   "author": "CanadaHonk",
   "license": "MIT",
   "dependencies": {
     "acorn": "^8.9.0"
   },
+  "optionalDependencies": {
+    "@babel/parser": "^7.23.6",
+    "hermes-parser": "^0.18.2",
+    "meriyah": "^4.3.9"
+  },
   "bin": {
     "porf": "./runner/index.js"
   },

package/porf.cmd

@@ -1,2 +1,2 @@
-@echo off
+@echo off
 node runner %*

package/rhemyn/README.md

@@ -0,0 +1,37 @@
+# Rhemyn
+A basic experimental WIP regex engine/AOT Wasm compiler in JS. Regex engine for Porffor! Uses own regex parser, no dependencies (excluding porffor internals). <br>
+Age: ~1 day (of work)
+
+Made for use with Porffor but could possibly be adapted, implementation/library notes:
+- Exposes functions for each regex "operation" (eg test, match)
+- Given a regex pattern string (eg `a+`), it returns a "function" object
+- Wasm function returned expects an i32 pointer to a UTF-16 string (can add UTF-8 option later if someone else actually wants to use this)
+
+## syntax
+🟢 supported 🟡 partial 🟠 parsed only 🔴 unsupported
+
+- 🟢 literal characters (eg `a`)
+- 🟢 escaping (eg `\.\n\cJ\x0a\u000a`)
+  - 🟢 character itself (eg `\.`)
+  - 🟢 escape sequences (eg `\n`)
+  - 🟢 control character (eg `\cJ`)
+  - 🟢 unicode code points (eg `\x00`, `\u0000`)
+- 🟢 sets (eg `[ab]`)
+  - 🟢 ranges (eg `[a-z]`)
+  - 🟢 negated sets (eg `[^ab]`)
+- 🟢 metacharacters
+  - 🟢 dot (eg `a.b`)
+  - 🟢 digit, not digit (eg `\d\D`)
+  - 🟢 word, not word (eg `\w\W`)
+  - 🟢 whitespace, not whitespace (eg `\s\S`)
+- 🟠 quantifiers
+  - 🟠 star (eg `a*`)
+  - 🟠 plus (eg `a+`)
+  - 🟠 optional (eg `a?`)
+  - 🟠 lazy modifier (eg `a*?`)
+  - 🔴 n repetitions (eg `a{4}`)
+  - 🔴 n-m repetitions (eg `a{2,4}`)
+- 🔴 assertions
+  - 🔴 beginning (eg `^a`)
+  - 🔴 end (eg `a$`)
+  - 🔴 word boundary assertion (eg `\b\B`)

package/rhemyn/compile.js

@@ -0,0 +1,214 @@
+import { Blocktype, Opcodes, Valtype, PageSize, ValtypeSize } from '../compiler/wasmSpec.js';
+import { number } from '../compiler/embedding.js';
+import { signedLEB128, unsignedLEB128 } from '../compiler/encoding.js';
+import parse from './parse.js';
+
+// local indexes
+const BasePointer = 0; // base string pointer
+const IterPointer = 1; // this iteration base pointer
+const EndPointer = 2; // pointer for the end
+const Counter = 3; // what char we are running on
+const Pointer = 4; // next char BYTE pointer
+const Length = 5;
+const Tmp = 6;
+
+let exprLastGet = false;
+const generate = (node, negated = false, get = true, func = 'test') => {
+  let out = [];
+  switch (node.type) {
+    case 'Expression':
+      exprLastGet = false;
+      out = [
+        // set length local
+        [ Opcodes.local_get, BasePointer ],
+        [ Opcodes.i32_load, Math.log2(ValtypeSize.i32) - 1, 0 ],
+        [ Opcodes.local_set, Length ],
+
+        // set iter pointer local as base + sizeof i32 initially
+        [ Opcodes.local_get, BasePointer ],
+        ...number(ValtypeSize.i32, Valtype.i32),
+        [ Opcodes.i32_add ],
+        [ Opcodes.local_set, IterPointer ],
+
+        [ Opcodes.loop, Blocktype.void ],
+
+        // reset pointer as iter pointer
+        [ Opcodes.local_get, IterPointer ],
+        [ Opcodes.local_set, Pointer ],
+
+        [ Opcodes.block, Blocktype.void ],
+
+        // generate checks
+        ...node.body.flatMap((x, i) => {
+          exprLastGet = x.type !== 'Group' && i === (node.body.length - 1);
+          return generate(x, negated);
+        }),
+
+        // reached end without branching out, successful match
+        ...({
+          test: number(1, Valtype.i32),
+          search: [
+            [ Opcodes.local_get, Counter ]
+          ]
+        })[func],
+        [ Opcodes.return ],
+
+        [ Opcodes.end ],
+
+        // increment iter pointer by sizeof i16
+        [ Opcodes.local_get, IterPointer ],
+        ...number(ValtypeSize.i16, Valtype.i32),
+        [ Opcodes.i32_add ],
+        [ Opcodes.local_set, IterPointer ],
+
+        // increment counter by 1, check if eq length, if not loop
+        [ Opcodes.local_get, Counter ],
+        ...number(1, Valtype.i32),
+        [ Opcodes.i32_add ],
+        [ Opcodes.local_tee, Counter ],
+
+        [ Opcodes.local_get, Length ],
+        [ Opcodes.i32_ne ],
+
+        [ Opcodes.br_if, 0 ],
+        [ Opcodes.end ],
+
+        // no match, return 0
+        ...number(({
+          test: 0,
+          search: -1
+        })[func], Valtype.i32)
+      ];
+
+      if (globalThis.regexLog) {
+        const underline = x => `\u001b[4m\u001b[1m${x}\u001b[0m`;
+        console.log(`\n${underline('ast')}`);
+        console.log(node);
+        console.log(`\n${underline('wasm bytecode')}\n` + decompile(out) + '\n');
+      }
+
+      break;
+
+    case 'Character':
+      out = generateChar(node, node.negated ^ negated, get);
+      break;
+
+    case 'Set':
+      out = generateSet(node, node.negated, get);
+      break;
+
+    case 'Group':
+      out = generateGroup(node, negated, get);
+      break;
+
+    case 'Range':
+      out = generateRange(node, negated, get);
+      break;
+  }
+
+  return out;
+};
+
+const getNextChar = () => [
+  // get char from pointer
+  [ Opcodes.local_get, Pointer ],
+  [ Opcodes.i32_load16_u, Math.log2(ValtypeSize.i16) - 1, ...unsignedLEB128(0) ],
+
+  ...(exprLastGet ? [] : [
+    // pointer += sizeof i16
+    [ Opcodes.local_get, Pointer ],
+    ...number(ValtypeSize.i16, Valtype.i32),
+    [ Opcodes.i32_add ],
+    [ Opcodes.local_set, Pointer ]
+  ])
+];
+
+const checkFailure = () => [
+  // surely we do not need to do this for every single mismatch, right?
+  /* [ Opcodes.if, Blocktype.void ],
+  ...number(0, Valtype.i32),
+  [ Opcodes.return ],
+  [ Opcodes.end ], */
+
+  [ Opcodes.br_if, 0 ]
+];
+
+const generateChar = (node, negated, get) => {
+  return [
+    ...(get ? getNextChar() : []),
+    ...number(node.char.charCodeAt(0), Valtype.i32),
+    negated ? [ Opcodes.i32_eq ] : [ Opcodes.i32_ne ],
+    ...(get ? checkFailure(): [])
+  ];
+};
+
+const generateSet = (node, negated, get) => {
+  // for a single char we do not need a tmp, it is like just
+  const singleChar = node.body.length === 1 && node.body[0].type === 'Character';
+
+  let out = [
+    ...(get ? getNextChar() : []),
+    ...(singleChar ? [] : [ [ Opcodes.local_set, Tmp ] ]),
+  ];
+
+  for (const x of node.body) {
+    out = [
+      ...out,
+      ...(singleChar ? [] : [ [ Opcodes.local_get, Tmp ] ]),
+      ...generate(x, negated, false)
+    ];
+  }
+
+  out = out.concat(new Array(node.body.length - 1).fill(negated ? [ Opcodes.i32_or ] : [ Opcodes.i32_and ]));
+
+  return [
+    ...out,
+    ...checkFailure()
+  ];
+};
+
+const generateRange = (node, negated, get) => {
+  return [
+    ...(get ? getNextChar() : []),
+    ...(get ? [ [ Opcodes.local_tee, Tmp ] ] : []),
+
+    ...number(node.from.charCodeAt(0), Valtype.i32),
+    // negated ? [ Opcodes.i32_lt_s ] : [ Opcodes.i32_ge_s ],
+    negated ? [ Opcodes.i32_ge_s ] : [ Opcodes.i32_lt_s ],
+
+    [ Opcodes.local_get, Tmp ],
+    ...number(node.to.charCodeAt(0), Valtype.i32),
+    // negated ? [ Opcodes.i32_gt_s ] : [ Opcodes.i32_le_s ],
+    negated ? [ Opcodes.i32_le_s ] : [ Opcodes.i32_gt_s ],
+
+    negated ? [ Opcodes.i32_and ] : [ Opcodes.i32_or ],
+    ...(get ? checkFailure(): [])
+  ];
+};
+
+const generateGroup = (node, negated, get) => {
+
+};
+
+export const test = (regex, index = 0, name = 'regex_test_' + regex) => outputFunc(generate(parse(regex), false, true, 'test'), name, index);
+export const search = (regex, index = 0, name = 'regex_search_' + regex) => outputFunc(generate(parse(regex), false, true, 'search'), name, index);
+
+const outputFunc = (wasm, name, index) => ({
+  name,
+  index,
+  wasm,
+
+  export: true,
+  params: [ Valtype.i32 ],
+  returns: [ Valtype.i32 ],
+  returnType: 0xffffffffffff1, // boolean - todo: do not hardcode this
+  locals: {
+    basePointer: { idx: 0, type: Valtype.i32 },
+    iterPointer: { idx: 1, type: Valtype.i32 },
+    endPointer: { idx: 2, type: Valtype.i32 },
+    counter: { idx: 3, type: Valtype.i32 },
+    pointer: { idx: 4, type: Valtype.i32 },
+    length: { idx: 5, type: Valtype.i32 },
+    tmp: { idx: 6, type: Valtype.i32 },
+  }
+});

package/rhemyn/parse.js

@@ -0,0 +1,321 @@
+const State = {
+  none: 0,
+  insideSet: 1
+};
+
+const Quantifiers = {
+  '*': [ 0 ], // 0 -
+  '+': [ 1 ], // 1 -
+  '?': [ 0, 1 ], // 0 - 1
+};
+const QuantifierKeys = Object.keys(Quantifiers);
+
+const getArg = (name, def) => {
+  const arg = (typeof process !== 'undefined' ? process.argv : Deno.args).find(x => x.startsWith(`-${name}=`));
+  if (arg) return arg.split('=')[0];
+
+  return def;
+};
+
+// full is spec-compliant but slower. not needed most of the time. (evil)
+const DotChars = () => ({
+  full: [ '\n', '\r', '\u2028', '\u2029' ],
+  simple: [ '\n', '\r' ],
+  fast: [ '\n' ]
+})[getArg('regex-dot', 'fast')];
+
+const WordChars = () => ({
+  full: [ [ 'a', 'z' ], [ 'A', 'Z' ], [ '0', '9' ], '_' ],
+  fast: [ [ '_', 'z' ], [ 'A', 'Z' ], [ '0', '9' ] ] // skip individual _ with _-z BUT it also matches '`'
+})[getArg('regex-word', 'full')];
+
+const WhitespaceChars = () => ({
+  full: [ ' ', '\t', '\n', '\r', '\u2028', '\u2029' ],
+  simple: [ ' ', '\t', '\n', '\r' ]
+})[getArg('regex-ws', 'simple')];
+
+const _Metachars = () => ({
+  unescaped: {
+    '.': [ DotChars(), true ], // dot
+  },
+  escaped: {
+    d: [ [ [ '0', '9' ] ], false ], // digit
+    D: [ [ [ '0', '9' ] ], true ], // not digit
+    w: [ WordChars(), false ], // word
+    W: [ WordChars(), true ], // not word
+    s: [ WhitespaceChars(), false ], // whitespace
+    S: [ WhitespaceChars(), true ], // not whitespace
+  }
+});
+
+const EscapeSequences = {
+  f: '\f',
+  n: '\n',
+  r: '\r',
+  t: '\t',
+  v: '\v',
+  '0': '\0'
+};
+
+const HexDigit = /[0-9a-fA-F]/;
+
+export default str => {
+  const Metachars = _Metachars();
+
+  const out = {
+    type: 'Expression',
+    body: []
+  };
+  let node = out, parents = [];
+
+  let state = State.none, setIndex = 0, escape = false;
+  for (let i = 0; i < str.length; i++) {
+    const c = str[i];
+
+    const charNode = char => ({
+      type: 'Character',
+      char
+    });
+
+    const rangeNode = (from, to) => ({
+      type: 'Range',
+      from,
+      to
+    });
+
+    const addChar = (char = c) => {
+      node.body.push(charNode(char));
+    };
+
+    const addSet = (matches, negated = false) => {
+      let body = matches.map(x => x[1] ? rangeNode(x[0], x[1]) : charNode(x));
+      if (state === State.insideSet) {
+        // if negated, mark each node as negated for merge
+        if (negated) body = body.map(x => {
+          x.negated = true;
+          return x;
+        });
+
+        // already in set, merge bodies
+        node.body.push(...body);
+        return;
+      }
+
+      node.body.push({
+        type: 'Set',
+        body,
+        negated
+      });
+    };
+
+    const addMetachar = meta => {
+      const [ matches, negated = false ] = meta;
+      return addSet(matches, negated);
+    };
+
+    // get next char and consume it
+    const seek = (allowEscaped = true) => {
+      const cNext = str[++i];
+
+      if (cNext === '\\') return !allowEscaped ? undefined : [ str[++i], true ];
+      return !allowEscaped ? cNext : [ cNext, false ];
+    };
+
+    // get next char without consuming
+    const peek = (allowEscaped = true, offset = 0) => {
+      const cNext = str[i + 1 + offset];
+
+      if (cNext === '\\') return !allowEscaped ? undefined : [ str[i + 2 + offset], true ];
+      return !allowEscaped ? cNext : [ cNext, false ];
+    };
+
+    if (escape) {
+      escape = false;
+      if (EscapeSequences[c]) {
+        addChar(EscapeSequences[c]);
+        continue;
+      }
+
+      if (Metachars.escaped[c]) {
+        addMetachar(Metachars.escaped[c]);
+        continue;
+      }
+
+      if (c === 'c') {
+        // \c (not [A-Za-z] ...) = literal \c... (WHY)
+        const next = peek(false);
+        if (next == null || /[^a-zA-Z]/.test(next)) {
+          addChar('\\');
+          addChar('c');
+          continue;
+        }
+
+        // \c[A-Za-z]
+        const code = seek(false).charCodeAt(0);
+        addChar(String.fromCharCode(code % 32));
+        continue;
+      }
+
+      if (c === 'x') {
+        // \x = x
+        // \xH = xH
+        // \x[0-9a-zA-Z][0-9a-zA-Z] = \xAB
+        const next1 = peek(false);
+        const next2 = peek(false, 1);
+
+        // missing a char or invalid hex digit
+        if (next1 == null || next2 == null || !HexDigit.test(next1) || !HexDigit.test(next2)) {
+          addChar('x');
+          continue;
+        }
+
+        const code = parseInt(seek(false) + seek(false), 16);
+        addChar(String.fromCodePoint(code));
+        continue;
+      }
+
+      if (c === 'u') {
+        // '\u' = u
+        // '\uHHH' = uHHH
+        // '\uABCD' = \uABCD
+        const next1 = peek(false);
+        const next2 = peek(false, 1);
+        const next3 = peek(false, 2);
+        const next4 = peek(false, 3);
+
+        // missing a char or invalid hex digit
+        if (next1 == null || next2 == null || next3 == null || next4 == null || !HexDigit.test(next1) || !HexDigit.test(next2) || !HexDigit.test(next3) || !HexDigit.test(next4)) {
+          addChar('u');
+          continue;
+        }
+
+        const code = parseInt(seek(false) + seek(false) + seek(false) + seek(false), 16);
+        addChar(String.fromCodePoint(code));
+        continue;
+      }
+
+      addChar();
+      continue;
+    }
+
+    if (c === '\\') {
+      escape = true;
+      continue;
+    }
+
+    switch (state) {
+      case State.none:
+        if (c === '[') {
+          parents.push(node);
+          node = {
+            type: 'Set',
+            body: [],
+            negated: false
+          };
+
+          parents.at(-1).body.push(node);
+
+          state = State.insideSet;
+          setIndex = 0;
+          continue;
+        }
+
+        if (c === '(') {
+          parents.push(node);
+          node = {
+            type: 'Group',
+            body: []
+          };
+
+          parents.at(-1).body.push(node);
+          continue;
+        }
+
+        if (c === ')') {
+          if (node.type !== 'Group') throw new SyntaxError('Unmatched closing parenthesis');
+
+          node = parents.pop();
+          continue;
+        }
+
+        if (QuantifierKeys.includes(c)) {
+          node.body.at(-1).quantifier = Quantifiers[c];
+
+          // lazy modifier
+          if (peek(false) === '?') node.body.at(-1).lazy = true;
+
+          continue;
+        }
+
+        if (Metachars.unescaped[c]) {
+          addMetachar(Metachars.unescaped[c]);
+          continue;
+        }
+
+        addChar();
+        break;
+
+      case State.insideSet:
+        setIndex++;
+        if (setIndex === 1) {
+          // first char in set
+          if (c === '^') {
+            node.negated = true;
+            continue;
+          }
+        }
+
+        if (c === ']') {
+          state = State.none;
+          node = parents.pop();
+
+          continue;
+        }
+
+        // range
+        if (c === '-') {
+          // start of set (or not char), just literal -
+          if (node.body.at(-1)?.char == null) {
+            addChar(); // add -
+            continue;
+          }
+
+          const from = node.body.pop().char;
+          const [ to, escaped ] = seek();
+
+          // end of set, just literal -
+          if (to == null || (!escaped && to === ']')) {
+            addChar(from); // add from char back
+            i--; // rollback seek
+
+            addChar(); // add -
+            continue;
+          }
+
+          // next char was escaped and a metachar, just literal -
+          if (escaped && Metachars.escaped[to] != null) {
+            i -= 2; // rollback seek
+
+            addChar(); // add -
+            continue;
+          }
+
+          if (to < from) throw new SyntaxError('Range out of order');
+
+          node.body.push(rangeNode(from, to));
+          continue;
+        }
+
+        addChar();
+        break;
+    }
+  }
+
+  // still in a group by the end
+  if (node.type !== 'Expression') throw new SyntaxError('Unmatched opening parenthesis');
+
+  // still in a set by the end
+  if (state === State.insideSet) throw new SyntaxError('Unmatched opening square bracket');
+
+  return out;
+};