zexus 1.6.8 → 1.7.2

package/src/zexus/lexer.py

@@ -2,6 +2,187 @@
 from .zexus_token import *
 from .error_reporter import get_error_reporter, SyntaxError as ZexusSyntaxError
 
+_LITERAL_KEYWORDS = {
+    "true": TRUE,
+    "false": FALSE,
+    "null": NULL,
+}
+
+_STRICT_KEYWORDS = {
+    'if', 'elif', 'else', 'while', 'for', 'each', 'in',
+    'return', 'break', 'continue', 'throw', 'try', 'catch',
+    'await', 'async', 'spawn', 'let', 'const', 'print',
+    'use', 'find', 'load', 'export', 'import', 'debug', 'match', 'lambda',
+    'case', 'default'
+}
+
+_CONTEXTS_ALLOWING_KEYWORD_IDENTS = {
+    LET, CONST, DOT, COMMA, LBRACKET, COLON, ASSIGN
+}
+
+_KEYWORDS = {
+    "let": LET,
+    "const": CONST,
+    "data": DATA,
+    "print": PRINT,
+    "if": IF,
+    "then": THEN,
+    "elif": ELIF,
+    "else": ELSE,
+    "true": TRUE,
+    "false": FALSE,
+    "null": NULL,
+    "return": RETURN,
+    "for": FOR,
+    "each": EACH,
+    "in": IN,
+    "action": ACTION,
+    "function": FUNCTION,
+    "while": WHILE,
+    "use": USE,
+    "find": FIND,
+    "load": LOAD,
+    "exactly": EXACTLY,
+    "embedded": EMBEDDED,
+    "export": EXPORT,
+    "lambda": LAMBDA,
+    "debug": DEBUG,
+    "try": TRY,
+    "catch": CATCH,
+    "finally": FINALLY,
+    "continue": CONTINUE,
+    "break": BREAK,
+    "throw": THROW,
+    "external": EXTERNAL,
+    "screen": SCREEN,
+    "component": COMPONENT,
+    "theme": THEME,
+    "color": COLOR,
+    "canvas": CANVAS,
+    "graphics": GRAPHICS,
+    "animation": ANIMATION,
+    "clock": CLOCK,
+    "async": ASYNC,
+    "await": AWAIT,
+    "channel": CHANNEL,
+    "send": SEND,
+    "receive": RECEIVE,
+    "atomic": ATOMIC,
+    "event": EVENT,
+    "emit": EMIT,
+    "enum": ENUM,
+    "protocol": PROTOCOL,
+    "import": IMPORT,
+    "public": PUBLIC,
+    "private": PRIVATE,
+    "sealed": SEALED,
+    "secure": SECURE,
+    "pure": PURE,
+    "view": VIEW,
+    "payable": PAYABLE,
+    "modifier": MODIFIER,
+    "entity": ENTITY,
+    "verify": VERIFY,
+    "contract": CONTRACT,
+    "protect": PROTECT,
+    "implements": IMPLEMENTS,
+    "this": THIS,
+    "as": AS,
+    "interface": INTERFACE,
+    "capability": CAPABILITY,
+    "grant": GRANT,
+    "revoke": REVOKE,
+    "module": MODULE,
+    "package": PACKAGE,
+    "using": USING,
+    "type_alias": TYPE_ALIAS,
+    "seal": SEAL,
+    "audit": AUDIT,
+    "restrict": RESTRICT,
+    "sandbox": SANDBOX,
+    "trail": TRAIL,
+    "middleware": MIDDLEWARE,
+    "auth": AUTH,
+    "throttle": THROTTLE,
+    "cache": CACHE,
+    "ledger": LEDGER,
+    "state": STATE,
+    "revert": REVERT,
+    "limit": LIMIT,
+    "persistent": PERSISTENT,
+    "storage": STORAGE,
+    "require": REQUIRE,
+    "and": AND,
+    "or": OR,
+    "native": NATIVE,
+    "gc": GC,
+    "inline": INLINE,
+    "buffer": BUFFER,
+    "simd": SIMD,
+    "defer": DEFER,
+    "pattern": PATTERN,
+    "match": MATCH,
+    "case": CASE,
+    "default": DEFAULT,
+    "enum": ENUM,
+    "stream": STREAM,
+    "watch": WATCH,
+    "log": LOG,
+    "inject": INJECT,
+    "validate": VALIDATE,
+    "sanitize": SANITIZE,
+}
+
+_FUNCTION_DECL_KEYWORDS = {"action", "function"}
+
+_FUNCTION_STATEMENT_BOUNDARIES = {
+    None,
+    SEMICOLON,
+    LBRACE,
+    RBRACE,
+    RBRACKET,
+    INT,
+    STRING,
+    FLOAT,
+    RPAREN,
+    TRUE,
+    FALSE,
+    NULL,
+    RETURN,
+    ASSIGN,
+    ASYNC,
+    EXPORT,
+    PUBLIC,
+    PRIVATE,
+    SEALED,
+    INLINE,
+    SECURE,
+    PURE,
+    VIEW,
+    PAYABLE,
+    NATIVE,
+}
+
+_DATA_KEYWORD_CONTRACT_CONTEXTS = {
+    SEMICOLON,
+    LBRACE,
+    RBRACE,
+    RBRACKET,
+    STRING,
+    INT,
+    FLOAT,
+    TRUE,
+    FALSE,
+    NULL,
+    PRIVATE,
+    PUBLIC,
+    SEALED,
+    SECURE,
+    PURE,
+    VIEW,
+    PAYABLE,
+}
+
 class Lexer:
     def __init__(self, source_code, filename="<stdin>"):
         self.input = source_code
@@ -18,6 +199,11 @@ class Lexer:
         self._next_paren_has_lambda = False
         # Track last token type to enable context-aware keyword handling
         self.last_token_type = None
+        # Track statement boundaries and nesting depth to disambiguate keywords vs identifiers
+        self.at_statement_boundary = True
+        self.paren_depth = 0
+        self.bracket_depth = 0
+        self.brace_depth = 0
         
         # Register source with error reporter
         self.error_reporter = get_error_reporter()
@@ -48,308 +234,414 @@ class Lexer:
             return self.input[self.read_position]
 
     def next_token(self):
-        self.skip_whitespace()
+        # NOTE: This method must not recurse. Large files can contain thousands
+        # of consecutive comment/blank lines; using recursion here can hit
+        # Python's recursion limit and cause unpredictable failures.
+        while True:
+            self.skip_whitespace()
 
-        # CRITICAL FIX: Skip single line comments (both # and // styles)
-        if self.ch == '#' and self.peek_char() != '{':
-            self.skip_comment()
-            return self.next_token()
+            # Skip single line comments (both # and // styles)
+            if self.ch == '#' and self.peek_char() != '{':
+                self.skip_comment()
+                continue
 
-        # NEW: Handle // style comments
-        if self.ch == '/' and self.peek_char() == '/':
-            self.skip_double_slash_comment()
-            return self.next_token()
+            # Handle // style comments and /* */ block comments
+            if self.ch == '/' and self.peek_char() == '/':
+                self.skip_double_slash_comment()
+                continue
 
-        tok = None
-        current_line = self.line
-        current_column = self.column
+            # Block comments: /* ... */
+            if self.ch == '/' and self.peek_char() == '*':
+                self.skip_block_comment()
+                continue
 
-        if self.ch == '=':
-            # Equality '=='
-            if self.peek_char() == '=':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(EQ, literal)
-                tok.line = current_line
-                tok.column = current_column
-            # Arrow '=>' (treat as lambda shorthand)
-            elif self.peek_char() == '>':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(LAMBDA, literal)
+            tok = None
+            current_line = self.line
+            current_column = self.column
+
+            if self.ch == '=':
+                # Equality '=='
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(EQ, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                # Arrow '=>' (treat as lambda shorthand)
+                elif self.peek_char() == '>':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(LAMBDA, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(ASSIGN, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '!':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(NOT_EQ, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(BANG, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '&':
+                if self.peek_char() == '&':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(AND, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    # Single '&' is not supported - suggest using '&&'
+                    error = self.error_reporter.report_error(
+                        ZexusSyntaxError,
+                        f"Unexpected character '{self.ch}'",
+                        line=current_line,
+                        column=current_column,
+                        filename=self.filename,
+                        suggestion="Did you mean '&&' for logical AND?"
+                    )
+                    raise error
+            elif self.ch == '|':
+                if self.peek_char() == '|':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(OR, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    # Single '|' is not supported - suggest using '||'
+                    error = self.error_reporter.report_error(
+                        ZexusSyntaxError,
+                        f"Unexpected character '{self.ch}'",
+                        line=current_line,
+                        column=current_column,
+                        filename=self.filename,
+                        suggestion="Did you mean '||' for logical OR?"
+                    )
+                    raise error
+            elif self.ch == '<':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(LTE, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                elif self.peek_char() == '<':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(IMPORT_OP, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(LT, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '>':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(GTE, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                elif self.peek_char() == '>':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(APPEND, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(GT, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '?':
+                # Check for nullish coalescing '??'
+                if self.peek_char() == '?':
+                    ch = self.ch
+                    self.read_char()
+                    literal = ch + self.ch
+                    tok = Token(NULLISH, literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(QUESTION, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '"':
+                # Check for triple-quote multiline string
+                if self.peek_char() == '"' and self.read_position + 1 < len(self.input) and self.input[self.read_position + 1] == '"':
+                    string_literal = self.read_multiline_string()
+                else:
+                    string_literal = self.read_string()
+                # If read_string returned a list, it's an interpolated string
+                if isinstance(string_literal, list):
+                    tok = Token(INTERP_STRING, string_literal)
+                else:
+                    tok = Token(STRING, string_literal)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                tok = Token(ASSIGN, self.ch)
+            elif self.ch == "'":
+                # Single-quoted strings
+                if self.peek_char() == "'" and self.read_position + 1 < len(self.input) and self.input[self.read_position + 1] == "'":
+                    string_literal = self.read_multiline_string(quote_char="'")
+                else:
+                    string_literal = self.read_single_quoted_string()
+                tok = Token(STRING, string_literal)
                 tok.line = current_line
                 tok.column = current_column
-        elif self.ch == '!':
-            if self.peek_char() == '=':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(NOT_EQ, literal)
+            elif self.ch == '[':
+                tok = Token(LBRACKET, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                tok = Token(BANG, self.ch)
+            elif self.ch == ']':
+                tok = Token(RBRACKET, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-        elif self.ch == '&':
-            if self.peek_char() == '&':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(AND, literal)
+            elif self.ch == '@':
+                tok = Token(AT, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                # Single '&' is not supported - suggest using '&&'
-                error = self.error_reporter.report_error(
-                    ZexusSyntaxError,
-                    f"Unexpected character '{self.ch}'",
-                    line=current_line,
-                    column=current_column,
-                    filename=self.filename,
-                    suggestion="Did you mean '&&' for logical AND?"
-                )
-                raise error
-        elif self.ch == '|':
-            if self.peek_char() == '|':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(OR, literal)
+            elif self.ch == '(':
+                # Quick char-level scan: detect if this '(' pairs with a ')' that
+                # is followed by '=>' (arrow). If so, set a hint flag so parser
+                # can treat the parentheses as a lambda-parameter list.
+                try:
+                    src = self.input
+                    i = self.position
+                    depth = 0
+                    found = False
+                    scan_limit = len(src)
+                    while i < scan_limit:
+                        c = src[i]
+                        if c == '(':
+                            depth += 1
+                        elif c == ')':
+                            depth -= 1
+                            if depth == 0:
+                                # look ahead for '=>' skipping whitespace
+                                j = i + 1
+                                while j < len(src) and src[j].isspace():
+                                    j += 1
+                                if j + 1 < len(src) and src[j] == '=' and src[j + 1] == '>':
+                                    found = True
+                                break
+                        i += 1
+                    self._next_paren_has_lambda = found
+                except Exception:
+                    self._next_paren_has_lambda = False
+
+                tok = Token(LPAREN, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                # Single '|' is not supported - suggest using '||'
-                error = self.error_reporter.report_error(
-                    ZexusSyntaxError,
-                    f"Unexpected character '{self.ch}'",
-                    line=current_line,
-                    column=current_column,
-                    filename=self.filename,
-                    suggestion="Did you mean '||' for logical OR?"
-                )
-                raise error
-        elif self.ch == '<':
-            if self.peek_char() == '=':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(LTE, literal)
+            elif self.ch == ')':
+                tok = Token(RPAREN, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            elif self.peek_char() == '<':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(IMPORT_OP, literal)
+            elif self.ch == '{':
+                # Check if this might be start of embedded block
+                lookback = self.input[max(0, self.position-10):self.position]
+                if 'embedded' in lookback:
+                    self.in_embedded_block = True
+                tok = Token(LBRACE, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                tok = Token(LT, self.ch)
+            elif self.ch == '}':
+                if self.in_embedded_block:
+                    self.in_embedded_block = False
+                tok = Token(RBRACE, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-        elif self.ch == '>':
-            if self.peek_char() == '=':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(GTE, literal)
+            elif self.ch == ',':
+                tok = Token(COMMA, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            elif self.peek_char() == '>':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(APPEND, literal)
+            elif self.ch == ';':
+                tok = Token(SEMICOLON, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                tok = Token(GT, self.ch)
+            elif self.ch == ':':
+                tok = Token(COLON, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-        elif self.ch == '?':
-            # Check for nullish coalescing '??'
-            if self.peek_char() == '?':
-                ch = self.ch
-                self.read_char()
-                literal = ch + self.ch
-                tok = Token(NULLISH, literal)
+            elif self.ch == '+':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    tok = Token(PLUS_ASSIGN, ch + self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(PLUS, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '-':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    tok = Token(MINUS_ASSIGN, ch + self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(MINUS, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '*':
+                if self.peek_char() == '*':
+                    ch = self.ch
+                    self.read_char()
+                    if self.peek_char() == '=':
+                        self.read_char()
+                        tok = Token(POWER_ASSIGN, '**=')
+                        tok.line = current_line
+                        tok.column = current_column
+                    else:
+                        tok = Token(POWER, ch + self.ch)
+                        tok.line = current_line
+                        tok.column = current_column
+                elif self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    tok = Token(STAR_ASSIGN, ch + self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(STAR, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '/':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    tok = Token(SLASH_ASSIGN, ch + self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(SLASH, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '%':
+                if self.peek_char() == '=':
+                    ch = self.ch
+                    self.read_char()
+                    tok = Token(MOD_ASSIGN, ch + self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+                else:
+                    tok = Token(MOD, self.ch)
+                    tok.line = current_line
+                    tok.column = current_column
+            elif self.ch == '.':
+                tok = Token(DOT, self.ch)
                 tok.line = current_line
                 tok.column = current_column
-            else:
-                tok = Token(QUESTION, self.ch)
+            elif self.ch == "":
+                tok = Token(EOF, "")
                 tok.line = current_line
                 tok.column = current_column
-        elif self.ch == '"':
-            string_literal = self.read_string()
-            tok = Token(STRING, string_literal)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '[':
-            tok = Token(LBRACKET, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == ']':
-            tok = Token(RBRACKET, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '@':
-            tok = Token(AT, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '(':
-            # Quick char-level scan: detect if this '(' pairs with a ')' that
-            # is followed by '=>' (arrow). If so, set a hint flag so parser
-            # can treat the parentheses as a lambda-parameter list.
-            try:
-                src = self.input
-                i = self.position
-                depth = 0
-                found = False
-                while i < len(src):
-                    c = src[i]
-                    if c == '(':
-                        depth += 1
-                    elif c == ')':
-                        depth -= 1
-                        if depth == 0:
-                            # look ahead for '=>' skipping whitespace
-                            j = i + 1
-                            while j < len(src) and src[j].isspace():
-                                j += 1
-                            if j + 1 < len(src) and src[j] == '=' and src[j + 1] == '>':
-                                found = True
-                            break
-                    i += 1
-                self._next_paren_has_lambda = found
-            except Exception:
-                self._next_paren_has_lambda = False
-
-            tok = Token(LPAREN, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == ')':
-            tok = Token(RPAREN, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '{':
-            # Check if this might be start of embedded block
-            lookback = self.input[max(0, self.position-10):self.position]
-            if 'embedded' in lookback:
-                self.in_embedded_block = True
-            tok = Token(LBRACE, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '}':
-            if self.in_embedded_block:
-                self.in_embedded_block = False
-            tok = Token(RBRACE, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == ',':
-            tok = Token(COMMA, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == ';':
-            tok = Token(SEMICOLON, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == ':':
-            tok = Token(COLON, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '+':
-            tok = Token(PLUS, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '-':
-            tok = Token(MINUS, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '*':
-            tok = Token(STAR, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '/':
-            # Check if this is division or comment
-            if self.peek_char() == '/':
-                # It's a // comment, handle above
-                self.skip_double_slash_comment()
-                return self.next_token()
             else:
-                tok = Token(SLASH, self.ch)
-                tok.line = current_line
-                tok.column = current_column
-        elif self.ch == '%':
-            tok = Token(MOD, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == '.':
-            tok = Token(DOT, self.ch)
-            tok.line = current_line
-            tok.column = current_column
-        elif self.ch == "":
-            tok = Token(EOF, "")
-            tok.line = current_line
-            tok.column = current_column
-        else:
-            if self.is_letter(self.ch):
-                literal = self.read_identifier()
+                if self.is_letter(self.ch):
+                    literal = self.read_identifier()
 
-                if self.in_embedded_block:
-                    token_type = IDENT
-                else:
-                    token_type = self.lookup_ident(literal)
+                    if self.in_embedded_block:
+                        token_type = IDENT
+                    else:
+                        token_type = self.lookup_ident(literal)
 
-                tok = Token(token_type, literal)
-                tok.line = current_line
-                tok.column = current_column
-                self.last_token_type = tok.type
-                return tok
-            elif self.is_digit(self.ch):
-                num_literal = self.read_number()
-                if '.' in num_literal:
-                    tok = Token(FLOAT, num_literal)
-                else:
-                    tok = Token(INT, num_literal)
-                tok.line = current_line
-                tok.column = current_column
-                self.last_token_type = tok.type
-                return tok
-            else:
-                if self.ch in ['\n', '\r']:
-                    self.read_char()
-                    return self.next_token()
-                # For embedded code, treat unknown printable chars as IDENT
-                if self.ch.isprintable():
-                    literal = self.read_embedded_char()
-                    tok = Token(IDENT, literal)
+                    tok = Token(token_type, literal)
                     tok.line = current_line
                     tok.column = current_column
-                    self.last_token_type = tok.type
+                    self._finalize_token(tok)
                     return tok
-                # Unknown character - report helpful error
-                char_desc = f"'{self.ch}'" if self.ch.isprintable() else f"'\\x{ord(self.ch):02x}'"
-                error = self.error_reporter.report_error(
-                    ZexusSyntaxError,
-                    f"Unexpected character {char_desc}",
-                    line=current_line,
-                    column=current_column,
-                    filename=self.filename,
-                    suggestion="Remove or replace this character with valid Zexus syntax."
-                )
-                raise error
+                elif self.is_digit(self.ch):
+                    num_literal = self.read_number()
+                    if '.' in num_literal:
+                        tok = Token(FLOAT, num_literal)
+                    else:
+                        tok = Token(INT, num_literal)
+                    tok.line = current_line
+                    tok.column = current_column
+                    self._finalize_token(tok)
+                    return tok
+                else:
+                    if self.ch in ['\n', '\r']:
+                        self.read_char()
+                        continue
+                    # For embedded code, treat unknown printable chars as IDENT
+                    if self.ch.isprintable():
+                        literal = self.read_embedded_char()
+                        tok = Token(IDENT, literal)
+                        tok.line = current_line
+                        tok.column = current_column
+                        self._finalize_token(tok)
+                        return tok
+                    # Unknown character - report helpful error
+                    char_desc = f"'{self.ch}'" if self.ch.isprintable() else f"'\\x{ord(self.ch):02x}'"
+                    error = self.error_reporter.report_error(
+                        ZexusSyntaxError,
+                        f"Unexpected character {char_desc}",
+                        line=current_line,
+                        column=current_column,
+                        filename=self.filename,
+                        suggestion="Remove or replace this character with valid Zexus syntax."
+                    )
+                    raise error
 
-        self.read_char()
-        # Track the token type for context-aware keyword handling
-        self.last_token_type = tok.type
-        return tok
+            self.read_char()
+            self._finalize_token(tok)
+            return tok
+
+    def _finalize_token(self, tok):
+        """Update lexer state after producing a token."""
+        if tok is None:
+            return
+
+        token_type = tok.type
+
+        # Maintain nesting depth for parentheses and brackets to help newline handling
+        if token_type == LPAREN:
+            self.paren_depth += 1
+        elif token_type == RPAREN:
+            if self.paren_depth > 0:
+                self.paren_depth -= 1
+        elif token_type == LBRACKET:
+            self.bracket_depth += 1
+        elif token_type == RBRACKET:
+            if self.bracket_depth > 0:
+                self.bracket_depth -= 1
+        elif token_type == LBRACE:
+            self.brace_depth += 1
+        elif token_type == RBRACE:
+            if self.brace_depth > 0:
+                self.brace_depth -= 1
+
+        # Update last token type for context-aware keyword handling
+        self.last_token_type = token_type
+
+        # Determine whether the next non-whitespace token is at a statement boundary
+        if token_type in {SEMICOLON, RBRACE, LBRACE, EOF}:
+            self.at_statement_boundary = True
+        elif token_type in {COMMA, DOT, ASSIGN, COLON, LPAREN, LBRACKET, AT}:
+            self.at_statement_boundary = False
+        elif token_type in {LET, CONST}:
+            # Declarations expect an identifier next
+            self.at_statement_boundary = False
+        else:
+            # Default: remain in the current statement
+            self.at_statement_boundary = False
 
     def read_embedded_char(self):
         """Read a single character as identifier for embedded code compatibility"""
@@ -379,6 +671,8 @@ class Lexer:
         start_line = self.line
         start_column = self.column
         result = []
+        has_interpolation = False
+        parts = []  # list of ("str", text) or ("expr", text)
         while True:
             self.read_char()
             if self.ch == "":
@@ -412,16 +706,170 @@ class Lexer:
                     'r': '\r',
                     '\\': '\\',
                     '"': '"',
-                    "'": "'"
+                    "'": "'",
+                    '$': '$'
                 }
                 result.append(escape_map.get(self.ch, self.ch))
+            elif self.ch == '$' and self.peek_char() == '{':
+                # String interpolation: ${expr}
+                has_interpolation = True
+                # Save current string part
+                if result:
+                    parts.append(("str", ''.join(result)))
+                    result = []
+                else:
+                    parts.append(("str", ""))
+                # Skip the '{'
+                self.read_char()
+                # Read expression until matching '}'
+                expr_chars = []
+                brace_depth = 1
+                while brace_depth > 0:
+                    self.read_char()
+                    if self.ch == "":
+                        error = self.error_reporter.report_error(
+                            ZexusSyntaxError,
+                            "Unterminated interpolation expression in string",
+                            line=start_line,
+                            column=start_column,
+                            filename=self.filename,
+                            suggestion="Add a closing } to terminate the interpolation."
+                        )
+                        raise error
+                    elif self.ch == '{':
+                        brace_depth += 1
+                        expr_chars.append(self.ch)
+                    elif self.ch == '}':
+                        brace_depth -= 1
+                        if brace_depth > 0:
+                            expr_chars.append(self.ch)
+                    else:
+                        expr_chars.append(self.ch)
+                parts.append(("expr", ''.join(expr_chars)))
             elif self.ch == '"':
                 # End of string
                 break
             else:
                 result.append(self.ch)
+
+        if has_interpolation:
+            # Add trailing string part
+            parts.append(("str", ''.join(result)))
+            return parts  # Return list of parts for interpolation
         return ''.join(result)
 
+    def read_single_quoted_string(self):
+        """Read a single-quoted string literal ('...')"""
+        start_line = self.line
+        start_column = self.column
+        result = []
+        while True:
+            self.read_char()
+            if self.ch == "":
+                error = self.error_reporter.report_error(
+                    ZexusSyntaxError,
+                    "Unterminated string literal",
+                    line=start_line,
+                    column=start_column,
+                    filename=self.filename,
+                    suggestion="Add a closing quote ' to terminate the string."
+                )
+                raise error
+            elif self.ch == '\\':
+                self.read_char()
+                if self.ch == '':
+                    error = self.error_reporter.report_error(
+                        ZexusSyntaxError,
+                        "Incomplete escape sequence at end of file",
+                        line=self.line,
+                        column=self.column,
+                        filename=self.filename,
+                        suggestion="Remove the backslash or complete the escape sequence."
+                    )
+                    raise error
+                escape_map = {
+                    'n': '\n', 't': '\t', 'r': '\r',
+                    '\\': '\\', "'": "'", '"': '"'
+                }
+                result.append(escape_map.get(self.ch, self.ch))
+            elif self.ch == "'":
+                break
+            else:
+                result.append(self.ch)
+        return ''.join(result)
+
+    def read_multiline_string(self, quote_char='"'):
+        """Read a triple-quoted multiline string (\"\"\"...\"\"\" or '''...''')"""
+        start_line = self.line
+        start_column = self.column
+        # Skip the three opening quotes
+        self.read_char()  # skip 2nd quote
+        self.read_char()  # skip 3rd quote
+        result = []
+        while True:
+            self.read_char()
+            if self.ch == "":
+                error = self.error_reporter.report_error(
+                    ZexusSyntaxError,
+                    "Unterminated multiline string literal",
+                    line=start_line,
+                    column=start_column,
+                    filename=self.filename,
+                    suggestion=f"Add closing {quote_char}{quote_char}{quote_char} to terminate the multiline string."
+                )
+                raise error
+            elif self.ch == '\\':
+                self.read_char()
+                if self.ch == '':
+                    break
+                escape_map = {
+                    'n': '\n', 't': '\t', 'r': '\r',
+                    '\\': '\\', quote_char: quote_char
+                }
+                result.append(escape_map.get(self.ch, self.ch))
+            elif self.ch == quote_char:
+                # Check for triple close
+                if self.peek_char() == quote_char and self.read_position + 1 < len(self.input) and self.input[self.read_position + 1] == quote_char:
+                    self.read_char()  # skip 2nd closing quote
+                    self.read_char()  # skip 3rd closing quote
+                    break
+                else:
+                    result.append(self.ch)
+            else:
+                result.append(self.ch)
+        return ''.join(result)
+
+    def skip_block_comment(self):
+        """Skip /* ... */ block comments (can be nested)"""
+        start_line = self.line
+        start_column = self.column
+        # Skip the opening /*
+        self.read_char()  # skip *
+        self.read_char()  # move past *
+        depth = 1
+        while depth > 0:
+            if self.ch == "":
+                error = self.error_reporter.report_error(
+                    ZexusSyntaxError,
+                    "Unterminated block comment",
+                    line=start_line,
+                    column=start_column,
+                    filename=self.filename,
+                    suggestion="Add closing */ to terminate the block comment."
+                )
+                raise error
+            elif self.ch == '/' and self.peek_char() == '*':
+                depth += 1
+                self.read_char()
+                self.read_char()
+            elif self.ch == '*' and self.peek_char() == '/':
+                depth -= 1
+                self.read_char()
+                self.read_char()
+            else:
+                self.read_char()
+        self.skip_whitespace()
+
     def read_identifier(self):
         start_position = self.position
         while self.is_letter(self.ch) or self.is_digit(self.ch):
@@ -448,200 +896,32 @@ class Lexer:
         return number_str
 
     def lookup_ident(self, ident):
-        # Special case: Always treat true, false, and null as keywords
-        if ident in ['true', 'false', 'null']:
-            keywords = {
-                "true": TRUE,
-                "false": FALSE,
-                "null": NULL,
-            }
-            return keywords[ident]
-        
-        # Define strict keywords that should NEVER be treated as identifiers
-        # These are control flow, operators, and modifiers that must always be keywords
-        strict_keywords = {
-            'if', 'elif', 'else', 'while', 'for', 'each', 'in',
-            'return', 'break', 'continue', 'throw', 'try', 'catch',
-            'await', 'async', 'spawn', 'let', 'const', 'print',
-            'use', 'export', 'import', 'debug', 'match'
-        }
-        
-        # If this is a strict keyword, always treat as keyword
-        if ident in strict_keywords:
-            # Fall through to normal keyword lookup at the end
-            pass
-        else:
-            # Context-aware keyword recognition: allow non-strict keywords as identifiers in certain contexts
-            # These contexts are where variable names are expected:
-            # - After LET, CONST (variable declarations)
-            # - After DOT (property/method names)
-            # - After COMMA (function parameters, after first param)
-            # - After LBRACKET (map keys when used as identifiers)
-            # - After ASSIGN (right-hand side can use keywords as identifiers: x = data)
-            # - After COLON (map keys, type annotations)
-            #
-            # Note: LPAREN removed - it was causing keywords after '(' to become identifiers
-            # even at the start of new statements. Instead, keywords as param names will
-            # work after the first parameter (via COMMA).
-            contexts_allowing_keywords_as_idents = {
-                LET, CONST, DOT, COMMA, LBRACKET, COLON, ASSIGN
-            }
-            
-            if self.last_token_type in contexts_allowing_keywords_as_idents:
-                # In these contexts, treat non-strict keywords as identifiers
-                return IDENT
-        
-        # Special case: ACTION and FUNCTION keywords should only be recognized
-        # when they actually start a definition, not when used as variable names in expressions
-        # Allow them as keywords at statement boundaries or after contract/data blocks
-        if ident in ['action', 'function']:
-            # These should be keywords at the start of a statement or after RETURN
-            # Allow after: None, SEMICOLON, LBRACE, RBRACE, INT, STRING, RPAREN (end of previous statement)
-            # Also allow after RETURN for function expressions: return function() {...}
-            # Also allow after ASYNC for async functions: async function name() {...}
-            # Also allow after EXPORT for exported functions: export function name() {...}
-            statement_boundaries = {None, SEMICOLON, LBRACE, RBRACE, INT, STRING, FLOAT, RPAREN, TRUE, FALSE, NULL, RETURN, ASYNC, EXPORT}
-            if self.last_token_type in statement_boundaries:
-                # Treat as keyword
-                pass  # Fall through to keyword lookup
-            else:
-                # In expression context, treat as identifier
-                return IDENT
-        
-        # Special case: DATA keyword should only be recognized in contract storage contexts
-        # When used as a parameter name, variable name, or in expressions, treat as identifier
-        if ident == 'data':
-            # Allow as keyword only in contract contexts (after CONTRACT or in contract body)
-            # In all other contexts (parameters, variables, expressions), treat as identifier
-            # Safe contexts for DATA keyword: after statement boundaries and value literals in contracts
-            # This includes: LBRACE (contract start), RBRACE (after Map {}), RBRACKET (after List []),
-            # STRING, INT, FLOAT, TRUE, FALSE (after literal values), SEMICOLON
-            contract_contexts = {SEMICOLON, LBRACE, RBRACE, RBRACKET, STRING, INT, FLOAT, TRUE, FALSE, NULL}
-            if self.last_token_type in contract_contexts:
-                # Might be a data declaration in contract, allow as keyword
-                pass  # Fall through to keyword lookup
-            else:
-                # In expression context, parameter list, or other contexts, treat as identifier
-                return IDENT
-        
-        # keyword lookup mapping (string -> token constant)
-        keywords = {
-            "let": LET,
-            "const": CONST,             # NEW: Const keyword for immutable variables
-            "data": DATA,               # NEW: Data keyword for dataclass definitions
-            "print": PRINT,
-            "if": IF,
-            "then": THEN,              # NEW: Then keyword for if-then-else expressions
-            "elif": ELIF,               # NEW: Elif keyword for else-if conditionals
-            "else": ELSE,
-            "true": TRUE,
-            "false": FALSE,
-            "null": NULL,
-            "return": RETURN,
-            "for": FOR,
-            "each": EACH,
-            "in": IN,
-            "action": ACTION,
-            "function": FUNCTION,
-            "while": WHILE,
-            "use": USE,
-            "exactly": EXACTLY,
-            "embedded": EMBEDDED,
-            "export": EXPORT,
-            "lambda": LAMBDA,
-            "debug": DEBUG,      # DUAL-MODE: Works as both statement (debug x;) and function (debug(x))
-            "try": TRY,          # NEW: Try keyword  
-            "catch": CATCH,      # NEW: Catch keyword
-            "continue": CONTINUE, # NEW: Continue on error keyword
-            "break": BREAK,      # NEW: Break loop keyword
-            "throw": THROW,      # NEW: Throw error keyword
-            "external": EXTERNAL, # NEW: External keyword
-            # "from": FROM,        # NOT a keyword - only recognized contextually in import statements
-            "screen": SCREEN,         # NEW: renderer keyword
-            "component": COMPONENT,   # NEW: renderer keyword
-            "theme": THEME,           # NEW: renderer keyword
-            "canvas": CANVAS,         # NEW (optional recognition)
-            "graphics": GRAPHICS,     # NEW (optional recognition)
-            "animation": ANIMATION,   # NEW (optional recognition)
-            "clock": CLOCK,           # NEW (optional recognition)
-            "async": ASYNC,
-            "await": AWAIT,
-            "channel": CHANNEL,       # NEW: Channel for concurrent communication
-            "send": SEND,             # NEW: Send to channel
-            "receive": RECEIVE,       # NEW: Receive from channel
-            "atomic": ATOMIC,         # NEW: Atomic operations
-            "event": EVENT,
-            "emit": EMIT,
-            "enum": ENUM,
-            "protocol": PROTOCOL,
-            "import": IMPORT,
-            # Modifiers
-            "public": PUBLIC,
-            "private": PRIVATE,
-            "sealed": SEALED,
-            "secure": SECURE,
-            "pure": PURE,
-            "view": VIEW,
-            "payable": PAYABLE,
-            "modifier": MODIFIER,
-            # NEW: Entity, Verify, Contract, Protect
-            "entity": ENTITY,
-            "verify": VERIFY,
-            "contract": CONTRACT,
-            "protect": PROTECT,
-            "implements": IMPLEMENTS,
-            "this": THIS,
-            "interface": INTERFACE,
-            "capability": CAPABILITY,  # NEW: Capability keyword for security
-            "grant": GRANT,             # NEW: Grant keyword for capability grants
-            "revoke": REVOKE,           # NEW: Revoke keyword for capability revocation
-            "module": MODULE,           # NEW: Module keyword for code organization
-            "package": PACKAGE,         # NEW: Package keyword for package definition
-            "using": USING,             # NEW: Using keyword for resource management
-            "type_alias": TYPE_ALIAS,   # NEW: Type alias keyword for type definitions
-            "seal": SEAL,               # NEW: Seal keyword for immutable objects
-            "audit": AUDIT,             # NEW: Audit keyword for compliance logging
-            "restrict": RESTRICT,       # NEW: Restrict keyword for field-level access control
-            "sandbox": SANDBOX,         # NEW: Sandbox keyword for isolated execution
-            "trail": TRAIL,             # NEW: Trail keyword for real-time logging
-            # Advanced features
-            "middleware": MIDDLEWARE,
-            "auth": AUTH,
-            "throttle": THROTTLE,
-            "cache": CACHE,
-            # Blockchain & Smart Contract keywords
-            "ledger": LEDGER,           # Immutable state ledger
-            "state": STATE,             # State management
-            "revert": REVERT,           # Revert transaction
-            # NOTE: "tx" removed as keyword - users can use it as variable name
-            # Only uppercase "TX" is reserved for transaction context
-            "limit": LIMIT,             # Gas/resource limit
-            # NOTE: hash, signature, verify_sig, gas are BUILTINS, not keywords
-            # NEW: Persistent storage keywords
-            "persistent": PERSISTENT,   # NEW: Persistent keyword
-            "storage": STORAGE,         # NEW: Storage keyword
-            "require": REQUIRE,         # Already defined in zexus_token.py
-            # Logical operators as keywords (alternative to && and ||)
-            "and": AND,                 # Logical AND (alternative to &&)
-            "or": OR,                   # Logical OR (alternative to ||)
-            # Performance optimization keywords
-            "native": NATIVE,           # Performance: call C/C++ code
-            "gc": GC,                   # Performance: control garbage collection
-            "inline": INLINE,           # Performance: function inlining
-            "buffer": BUFFER,           # Performance: direct memory access
-            "simd": SIMD,               # Performance: vector operations
-            "defer": DEFER,             # Convenience: cleanup code execution
-            "pattern": PATTERN,         # Convenience: pattern matching
-            "match": MATCH,             # Match expression for pattern matching
-            "enum": ENUM,               # Advanced: type-safe enumerations
-            "stream": STREAM,           # Advanced: event streaming
-            "watch": WATCH,             # Advanced: reactive state management
-            "log": LOG,                 # Output logging to file
-            "inject": INJECT,           # Advanced: dependency injection
-            "validate": VALIDATE,       # Data validation
-            "sanitize": SANITIZE,       # Data sanitization
-        }
-        return keywords.get(ident, IDENT)
+        # Always treat literal keywords as reserved regardless of context.
+        literal_token = _LITERAL_KEYWORDS.get(ident)
+        if literal_token is not None:
+            return literal_token
+
+        token = _KEYWORDS.get(ident)
+        if token is None:
+            return IDENT
+
+        if ident in _FUNCTION_DECL_KEYWORDS:
+            if self.last_token_type in _FUNCTION_STATEMENT_BOUNDARIES:
+                return token
+            return IDENT
+
+        if ident == "data":
+            if self.last_token_type in _DATA_KEYWORD_CONTRACT_CONTEXTS:
+                return token
+            return IDENT
+
+        if ident in _STRICT_KEYWORDS:
+            return token
+
+        if not self.at_statement_boundary and self.last_token_type in _CONTEXTS_ALLOWING_KEYWORD_IDENTS:
+            return IDENT
+
+        return token
 
     def is_letter(self, char):
         return 'a' <= char <= 'z' or 'A' <= char <= 'Z' or char == '_'
@@ -651,4 +931,8 @@ class Lexer:
 
     def skip_whitespace(self):
         while self.ch in [' ', '\t', '\n', '\r']:
+            if self.ch in ['\n', '\r']:
+                # Treat newline as potential statement boundary when not inside paren/bracket expressions
+                if self.paren_depth == 0 and self.bracket_depth == 0:
+                    self.at_statement_boundary = True
             self.read_char()