rip-lang 3.16.0 → 3.16.2

package/src/grammar/solar.rip

@@ -10,7 +10,6 @@
 import * as fs from 'fs'
 import * as path from 'path'
 import { fileURLToPath, pathToFileURL } from 'url'
-import { install as installLunar } from './lunar.rip'
 
 VERSION = '1.5.0'
 
@@ -90,11 +89,9 @@ class Generator
     # Build parser
     @timing '💥 Total time', =>
       @timing 'processGrammar'   , => @processGrammar grammar # Process grammar rules
-      unless @options.rd
-        @timing 'buildLRAutomaton' , => @buildLRAutomaton()     # Build LR(0) automaton
+      @timing 'buildLRAutomaton' , => @buildLRAutomaton()     # Build LR(0) automaton
       @timing 'processLookaheads', => @processLookaheads()    # Compute FIRST/FOLLOW and assign lookaheads
-      unless @options.rd
-        @timing 'buildParseTable'  , => @buildParseTable()      # Build parse table with default actions
+      @timing 'buildParseTable'  , => @buildParseTable()      # Build parse table with default actions
 
   # ============================================================================
   # Helper Functions
@@ -656,8 +653,6 @@ class Generator
     # Decoder reconstructs [{key: val}, ...] at runtime
     "(()=>{let d=[#{data.join ','}],t=[],p=0,n,o,k,a;while(p<d.length){n=d[p++];o={};k=0;a=[];while(n--)k+=d[p++],a.push(k);for(k of a)o[k]=d[p++];t.push(o)}return t})()"
 
-  # Recursive descent generation — installed by Lunar (see lunar.rip)
-
   # ============================================================================
   # Runtime Parser
   # ============================================================================
@@ -772,8 +767,6 @@ class Generator
 # Exports
 # ==============================================================================
 
-installLunar Generator
-
 export { Generator }
 
 # ==============================================================================
@@ -805,7 +798,6 @@ if isRunAsScript
         -v, --version           Show version
         -i, --info              Show grammar information
         -s, --sexpr             Show grammar as s-expression
-        -r, --rd                Generate recursive descent parser (parser-rd.js)
         -c, --conflicts         Show conflict details (use with --info)
         -o, --output <file>     Output file (default: parser.js)
 
@@ -843,7 +835,7 @@ if isRunAsScript
           console.log "     Resolution: #{conflict.resolution} (by default)"
 
     # Parse command line
-    options = {help: false, version: false, info: false, sexpr: false, conflicts: false, rd: false, output: 'parser.js'}
+    options = {help: false, version: false, info: false, sexpr: false, conflicts: false, output: 'parser.js'}
     grammarFile = null
     i = 0
 
@@ -854,7 +846,6 @@ if isRunAsScript
         when '-v', '--version'   then options.version   = true
         when '-i', '--info'      then options.info      = true
         when '-s', '--sexpr'     then options.sexpr     = true
-        when '-r', '--rd'        then options.rd        = true
         when '-c', '--conflicts' then options.conflicts = true
         when '-o', '--output'    then options.output    = process.argv[++i + 2]
         else grammarFile = arg unless arg.startsWith('-')
@@ -915,11 +906,7 @@ if isRunAsScript
       if options.info
         showStats generator
       else
-        if options.rd
-          options.output = 'parser-rd.js' if options.output is 'parser.js'
-          parserCode = generator.generateRD()
-        else
-          parserCode = generator.generate()
+        parserCode = generator.generate()
         fs.writeFileSync options.output, parserCode
         console.log "\nParser generated: #{options.output}"
 

package/src/lexer.js

@@ -7,7 +7,7 @@
 //
 // Design principles:
 //   - Every token carries .pre (whitespace count before it)
-//   - Every token carries .data (metadata: await, predicate, quote, etc.)
+//   - Every token carries .data (metadata: bang, optional, quote, etc.)
 //   - Every token carries .loc  (location: row, col, len)
 //   - Indentation is derived from .pre, not tracked during lexing
 //   - Token categories use Sets for O(1) membership tests
@@ -23,8 +23,8 @@
 //   token.newLine       — true if preceded by a newline
 //
 // Identifier suffixes:
-//   !  — dammit operator: fetch!() → await fetch()
-//   ?  — predicate:       empty?  → isEmpty (returns boolean convention)
+//   !  — bang: fetch!() → await fetch() (call site) | foo! = -> → void (definition)
+//   ?  — optional:        empty?  → (empty != null) (existence check / optional marker)
 //
 // The 9 tokenizer methods (in priority order):
 //   1. identifier  — variables, keywords, properties, ! and ? suffixes
@@ -129,7 +129,7 @@ let IMPLICIT_UNSPACED_CALL = new Set(['+', '-']);
 // Tokens that end an implicit call
 let IMPLICIT_END = new Set([
   'POST_IF', 'POST_UNLESS', 'FOR', 'WHILE', 'UNTIL',
-  'WHEN', 'BY', 'LOOP', 'TERMINATOR', '||', '&&', 'PIPE',
+  'WHEN', 'BY', 'LOOP', 'TERMINATOR', '||', '&&',
 ]);
 
 // Tokens that trigger implicit comma insertion before arrows
@@ -246,12 +246,12 @@ let UNARY_MATH = new Set(['!', '~']);
 // Regex Patterns
 // ==========================================================================
 
-// Identifier: word chars + optional trailing ! (await) or ? (predicate)
+// Identifier: word chars + optional trailing ! (bang) or ? (optional)
 // The ? suffix is only captured when NOT followed by . ? ! [ ( to avoid
 // conflict with ?. (optional chaining), ?? (nullish), ?! (presence), ?.( and ?.[
 let IDENTIFIER_RE = /^(?!\d)((?:(?!\s)[$\w\x7f-\uffff])+(?:!|[?](?![.?![(]))?)([^\n\S]*:(?![=:]))?/;
 let NUMBER_RE     = /^0b[01](?:_?[01])*n?|^0o[0-7](?:_?[0-7])*n?|^0x[\da-f](?:_?[\da-f])*n?|^\d+(?:_\d+)*n|^(?:\d+(?:_\d+)*)?\.?\d+(?:_\d+)*(?:e[+-]?\d+(?:_\d+)*)?/i;
-let OPERATOR_RE   = /^(?:<=>|::|\*>|[-=]>|~>|~=|:=|=!|===|!==|\?\!|\?\?|=~|\|>|[-+*\/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>*\/%])\2=?|\?\.?|\.{2,3})/;
+let OPERATOR_RE   = /^(?:<=>|<~|::|\*>|[-=]>|~>|~=|:=|=!|===|!==|\?\!|\?\?|=~|[-+*\/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>*\/%])\2=?|\?\.?|\.{2,3})/;
 let WHITESPACE_RE = /^[^\n\S]+/;
 let NEWLINE_RE    = /^(?:\n[^\n\S]*)+/;
 let COMMENT_RE    = /^(\s*)###([^#][\s\S]*?)(?:###([^\n\S]*)|###$)|^((?:\s*#(?!##[^#]).*)+)/;
@@ -519,8 +519,10 @@ export class Lexer {
   // Handles: variables, keywords, properties, aliases
   //
   // Suffix operators on identifiers:
-  //   !  → dammit operator (await): fetch!() → await fetch()
-  //   ?  → predicate (boolean):     empty?   → isEmpty
+  //   !  → bang: a neutral "trailing !" flag. Resolved by context downstream —
+  //        dammit/await at a call site (fetch!() → await fetch()), or the void
+  //        marker at a function definition (foo! = -> / def foo! → no return).
+  //   ?  → optional (existence):    empty?   → (empty != null)
   //
   // The ? suffix is captured by IDENTIFIER_RE only when NOT followed by
   // . ? [ ( — so x?.y (optional chaining) and x?? (nullish coalescing)
@@ -655,16 +657,20 @@ export class Lexer {
       }
     }
 
-    // --- Dammit operator: trailing ! → await ---
+    // --- Bang: trailing ! → context-resolved (await at call site / void at def) ---
     if (id.length > 1 && id.endsWith('!')) {
-      data.await = true;
+      data.bang = true;
       id = id.slice(0, -1);
     }
 
-    // --- Predicate operator: trailing ? → boolean convention ---
-    // empty? → isEmpty, active? → isActive, valid? → isValid
+    // --- Optional marker: trailing ? ---
+    // Identifier-position: existence check (`empty?` → `(empty != null)`).
+    // Property-name position: optional prop / type field (`@label?:: T`,
+    // `{ x?:: T }`, `def f(x?:: T)`). The flag is consumed by the type/
+    // component/schema emitters; the runtime semantics for `name?` as an
+    // existence check are unchanged.
     if (id.length > 1 && id.endsWith('?')) {
-      data.predicate = true;
+      data.optional = true;
       id = id.slice(0, -1);
     }
 
@@ -742,18 +748,6 @@ export class Lexer {
       }
       if (/^\s+#[a-zA-Z_]/.test(this.chunk)) return 0;  // let lineToken handle indentation first
     }
-    // Schema field modifier: `#` adjacent (unspaced) to an identifier acts
-    // as the unique-marker inside schema bodies (e.g. `email!# email`).
-    // Return 0 so literalToken emits a standalone `#` token; rewriteSchema
-    // absorbs it. Outside schema bodies the `#` token is harmless because
-    // nothing else in the grammar accepts `IDENTIFIER #` without a space.
-    if (this.chunk[0] === '#') {
-      let prev = this.prev();
-      if (prev && !prev.spaced && !prev.newLine &&
-          (prev[0] === 'IDENTIFIER' || prev[0] === 'PROPERTY')) {
-        return 0;
-      }
-    }
     let match = COMMENT_RE.exec(this.chunk);
     if (!match) return 0;
     return match[0].length;
@@ -1382,8 +1376,6 @@ export class Lexer {
       this.inTypeAnnotation = false;
       tag = 'TERMINATOR';
     }
-    // Pipe operator
-    else if (val === '|>') tag = 'PIPE';
     // Type operators
     else if (val === '::' && /^[a-zA-Z_$]/.test(this.chunk[2] || '')) {
       // Prototype access: String::trim → String.prototype.trim
@@ -1397,6 +1389,7 @@ export class Lexer {
     else if (val === '~=') tag = 'COMPUTED_ASSIGN';
     else if (val === ':=') tag = 'REACTIVE_ASSIGN';
     else if (val === '<=>') tag = 'BIND';
+    else if (val === '<~') { tag = 'GATE'; this.inTypeAnnotation = false; }
     else if (val === '~>') { tag = 'EFFECT'; this.inTypeAnnotation = false; }
     else if (val === '=!') { tag = 'READONLY_ASSIGN'; this.inTypeAnnotation = false; }
     // Merge assignment: *>config = {a: 1} → Object.assign(config, {a: 1})
@@ -1519,9 +1512,42 @@ export class Lexer {
 
   // Walk back to tag parameters for arrow functions
   tagParameters() {
-    if (this.prevTag() !== ')') return this.tagDoIife();
+    let closeIdx = this.tokens.length - 1;
+    if (this.tokens[closeIdx]?.[0] !== ')') {
+      // Maybe a return-type annotation sits between `)` and the arrow:
+      // `(x:: T):: R ->`. Scan backward over the type expression (balanced
+      // brackets) looking for the trailing `TYPE_ANNOTATION` whose previous
+      // token is `)`. If found, treat that `)` as the param-list close.
+      let n = this.tokens.length;
+      let depth = 0;
+      let j = n - 1;
+      let found = -1;
+      while (j >= 0) {
+        let tk = this.tokens[j];
+        let tg = tk[0];
+        if (tg === ')' || tg === ']' || tg === '}' || tg === 'CALL_END' ||
+            tg === 'PARAM_END' || tg === 'INDEX_END' ||
+            (tg === 'COMPARE' && tk[1] === '>')) {
+          depth++;
+        } else if (tg === '(' || tg === '[' || tg === '{' || tg === 'CALL_START' ||
+                   tg === 'PARAM_START' || tg === 'INDEX_START' ||
+                   (tg === 'COMPARE' && tk[1] === '<')) {
+          depth--;
+        } else if (depth === 0) {
+          if (tg === 'TYPE_ANNOTATION') {
+            if (j > 0 && this.tokens[j - 1][0] === ')') found = j - 1;
+            break;
+          }
+          if (tg === 'TERMINATOR' || tg === 'INDENT' || tg === 'OUTDENT' ||
+              tg === '=' || tg === '->' || tg === '=>') break;
+        }
+        j--;
+      }
+      if (found < 0) return this.tagDoIife();
+      closeIdx = found;
+    }
 
-    let i = this.tokens.length - 1;
+    let i = closeIdx;
     let stack = [];
     this.tokens[i][0] = 'PARAM_END';
 
@@ -1536,7 +1562,7 @@ export class Lexer {
           tok[0] = 'PARAM_START';
           return this.tagDoIife(i - 1);
         } else {
-          this.tokens[this.tokens.length - 1][0] = 'CALL_END';
+          this.tokens[closeIdx][0] = 'CALL_END';
           return;
         }
       }
@@ -1562,10 +1588,14 @@ export class Lexer {
     this.closeMergeAssignments();
     this.closeOpenCalls();
     this.closeOpenIndexes();
+    // rewriteTypes must run BEFORE normalizeLines: otherwise a type-arrow
+    // `=>` inside `(...) => T` gets treated as a single-liner function and
+    // wrapped in spurious INDENT/OUTDENT, which then derails the type
+    // collector (it slurps the whole assignment body as part of the type).
+    this.rewriteTypes();
     this.normalizeLines();
     this.rewriteRender?.();
     this.rewriteSchema?.();
-    this.rewriteTypes();
     this.tagPostfixConditionals();
     this.rewriteTaggedTemplates();
     this.addImplicitBracesAndParens();
@@ -2020,12 +2050,32 @@ export class Lexer {
         } else if (t && t !== 'TERMINATOR' && t !== 'OUTDENT' && t !== ',') j++;
         logicalKeep = tokens[j]?.[0] === ',';
       }
+      // A postfix conditional on the FIRST property line of a multiline
+      // implicit object must bind to the property's value, not close the
+      // object — otherwise `a: 1 if c` followed by more properties splits
+      // the object into separate arguments. Later property lines already
+      // bind to the value (the sameLine flag is false by then), so only
+      // first-line conditionals need the lookahead: keep the object open
+      // when the property list continues after this line.
+      let objectContinues = (j) => {
+        for (let d = 0; j < tokens.length; j++) {
+          let t = tokens[j][0];
+          if (t === '(' || t === '[' || t === '{' || t === 'CALL_START' || t === 'INDEX_START' || t === 'INDENT') d++;
+          else if (t === ')' || t === ']' || t === '}' || t === 'CALL_END' || t === 'INDEX_END' || t === 'OUTDENT') {
+            if (d === 0) return false;
+            d--;
+          }
+          else if (t === 'TERMINATOR' && d === 0) return this.looksObjectish(j + 1);
+        }
+        return false;
+      };
       if ((IMPLICIT_END.has(tag) && !logicalKeep) || (CALL_CLOSERS.has(tag) && newLine)) {
         while (isImplicit(stackTop())) {
           let [stackTag, , {sameLine, startsLine}] = stackTop();
           if (inImplicitCall() && prevTag !== ',') {
             endImplicitCall();
-          } else if (inImplicitObject() && !isLogicalOp && sameLine && tag !== 'TERMINATOR' && prevTag !== ':') {
+          } else if (inImplicitObject() && !isLogicalOp && sameLine && tag !== 'TERMINATOR' && prevTag !== ':' &&
+                     !((tag === 'POST_IF' || tag === 'POST_UNLESS') && objectContinues(i + 1))) {
             endImplicitObject();
           } else if (inImplicitObject() && tag === 'TERMINATOR' && prevTag !== ',' && !(startsLine && this.looksObjectish(i + 1))) {
             endImplicitObject();