IncludeCPP 3.4.2__py3-none-any.whl → 3.4.10__py3-none-any.whl

includecpp/core/cssl/cssl_parser.py

@@ -216,6 +216,9 @@ class CSSLLexer:
                 self.column = 1
             elif char in '"\'':
                 self._read_string(char)
+            elif char == '`':
+                # Raw string (no escape processing) - useful for JSON
+                self._read_raw_string()
             elif char.isdigit() or (char == '-' and self._peek(1).isdigit()):
                 self._read_number()
             elif char == 'r' and self._peek(1) == '@':
@@ -224,7 +227,7 @@ class CSSLLexer:
             elif char == 's' and self._peek(1) == '@':
                 # s@<name> self-reference to global struct
                 self._read_self_ref()
-            elif char.isalpha() or char == '_' or char == '-':
+            elif char.isalpha() or char == '_':
                 self._read_identifier()
             elif char == '@':
                 self._add_token(TokenType.AT, '@')
@@ -346,13 +349,52 @@ class CSSLLexer:
     def _read_string(self, quote_char: str):
         self._advance()
         start = self.pos
+        result = []
         while self.pos < len(self.source) and self.source[self.pos] != quote_char:
-            if self.source[self.pos] == '\\':
+            if self.source[self.pos] == '\\' and self.pos + 1 < len(self.source):
+                # Handle escape sequences
+                next_char = self.source[self.pos + 1]
+                if next_char == 'n':
+                    result.append('\n')
+                elif next_char == 't':
+                    result.append('\t')
+                elif next_char == 'r':
+                    result.append('\r')
+                elif next_char == '\\':
+                    result.append('\\')
+                elif next_char == quote_char:
+                    result.append(quote_char)
+                elif next_char == '"':
+                    result.append('"')
+                elif next_char == "'":
+                    result.append("'")
+                else:
+                    result.append(self.source[self.pos])
+                    result.append(next_char)
+                self._advance()
+                self._advance()
+            else:
+                result.append(self.source[self.pos])
                 self._advance()
+        value = ''.join(result)
+        self._add_token(TokenType.STRING, value)
+        self._advance()
+
+    def _read_raw_string(self):
+        """Read raw string with backticks - no escape processing.
+
+        Useful for JSON: `{"id": "2819e1", "name": "test"}`
+        """
+        self._advance()  # Skip opening backtick
+        start = self.pos
+        while self.pos < len(self.source) and self.source[self.pos] != '`':
+            if self.source[self.pos] == '\n':
+                self.line += 1
+                self.column = 0
             self._advance()
         value = self.source[start:self.pos]
         self._add_token(TokenType.STRING, value)
-        self._advance()
+        self._advance()  # Skip closing backtick
 
     def _read_number(self):
         start = self.pos
@@ -368,7 +410,7 @@ class CSSLLexer:
 
     def _read_identifier(self):
         start = self.pos
-        while self.pos < len(self.source) and (self.source[self.pos].isalnum() or self.source[self.pos] in '_-'):
+        while self.pos < len(self.source) and (self.source[self.pos].isalnum() or self.source[self.pos] == '_'):
             self._advance()
         value = self.source[start:self.pos]
 
@@ -588,6 +630,286 @@ class CSSLParser:
         self._match(TokenType.BLOCK_END)
         return root
 
+    def _is_function_modifier(self, value: str) -> bool:
+        """Check if a keyword is a function modifier"""
+        return value in ('undefined', 'open', 'meta', 'super', 'closed', 'private', 'virtual', 'sqlbased')
+
+    def _is_type_keyword(self, value: str) -> bool:
+        """Check if a keyword is a type declaration"""
+        return value in ('int', 'string', 'float', 'bool', 'void', 'json', 'array', 'vector', 'stack',
+                        'dynamic', 'datastruct', 'dataspace', 'shuffled', 'iterator', 'combo', 'structure')
+
+    def _looks_like_function_declaration(self) -> bool:
+        """Check if current position looks like a C-style function declaration.
+
+        Patterns:
+        - int funcName(...)
+        - undefined int funcName(...)
+        - vector<string> funcName(...)
+        - undefined void funcName(...)
+        """
+        saved_pos = self.pos
+
+        # Skip modifiers (undefined, open, meta, super, closed, private, virtual)
+        while self._check(TokenType.KEYWORD) and self._is_function_modifier(self._current().value):
+            self._advance()
+
+        # Check for type keyword
+        if self._check(TokenType.KEYWORD) and self._is_type_keyword(self._current().value):
+            self._advance()
+
+            # Skip generic type parameters <T>
+            if self._check(TokenType.COMPARE_LT):
+                depth = 1
+                self._advance()
+                while depth > 0 and not self._is_at_end():
+                    if self._check(TokenType.COMPARE_LT):
+                        depth += 1
+                    elif self._check(TokenType.COMPARE_GT):
+                        depth -= 1
+                    self._advance()
+
+            # Check for identifier followed by (
+            if self._check(TokenType.IDENTIFIER):
+                self._advance()
+                is_func = self._check(TokenType.PAREN_START)
+                self.pos = saved_pos
+                return is_func
+
+        self.pos = saved_pos
+        return False
+
+    def _looks_like_typed_variable(self) -> bool:
+        """Check if current position looks like a typed variable declaration.
+
+        Patterns:
+        - int x;
+        - stack<string> myStack;
+        - vector<int> nums = [1,2,3];
+
+        Distinguishes from function declarations by checking for '(' after identifier.
+        """
+        saved_pos = self.pos
+
+        # Check for type keyword
+        if self._check(TokenType.KEYWORD) and self._is_type_keyword(self._current().value):
+            self._advance()
+
+            # Skip generic type parameters <T>
+            if self._check(TokenType.COMPARE_LT):
+                depth = 1
+                self._advance()
+                while depth > 0 and not self._is_at_end():
+                    if self._check(TokenType.COMPARE_LT):
+                        depth += 1
+                    elif self._check(TokenType.COMPARE_GT):
+                        depth -= 1
+                    self._advance()
+
+            # Check for identifier NOT followed by ( (that would be a function)
+            if self._check(TokenType.IDENTIFIER):
+                self._advance()
+                # If followed by '(' it's a function, not a variable
+                is_var = not self._check(TokenType.PAREN_START)
+                self.pos = saved_pos
+                return is_var
+
+        self.pos = saved_pos
+        return False
+
+    def _parse_typed_function(self) -> ASTNode:
+        """Parse C-style typed function declaration.
+
+        Patterns:
+        - int Add(int a, int b) { }
+        - undefined int Func() { }
+        - open void Handler(open Params) { }
+        - vector<string> GetNames() { }
+        """
+        modifiers = []
+        return_type = None
+        generic_type = None
+
+        # Collect modifiers (undefined, open, meta, super, closed, private, virtual)
+        while self._check(TokenType.KEYWORD) and self._is_function_modifier(self._current().value):
+            modifiers.append(self._advance().value)
+
+        # Get return type
+        if self._check(TokenType.KEYWORD) and self._is_type_keyword(self._current().value):
+            return_type = self._advance().value
+
+            # Check for generic type <T>
+            if self._check(TokenType.COMPARE_LT):
+                self._advance()  # skip <
+                generic_parts = []
+                depth = 1
+                while depth > 0 and not self._is_at_end():
+                    if self._check(TokenType.COMPARE_LT):
+                        depth += 1
+                        generic_parts.append('<')
+                    elif self._check(TokenType.COMPARE_GT):
+                        depth -= 1
+                        if depth > 0:
+                            generic_parts.append('>')
+                    elif self._check(TokenType.COMMA):
+                        generic_parts.append(',')
+                    else:
+                        generic_parts.append(self._current().value)
+                    self._advance()
+                generic_type = ''.join(generic_parts)
+
+        # Get function name
+        name = self._advance().value
+
+        # Parse parameters
+        params = []
+        self._expect(TokenType.PAREN_START)
+
+        while not self._check(TokenType.PAREN_END) and not self._is_at_end():
+            param_info = {}
+
+            # Handle 'open' keyword for open parameters
+            if self._match_keyword('open'):
+                param_info['open'] = True
+
+            # Handle type annotations
+            if self._check(TokenType.KEYWORD) and self._is_type_keyword(self._current().value):
+                param_info['type'] = self._advance().value
+
+                # Check for generic type parameter <T>
+                if self._check(TokenType.COMPARE_LT):
+                    self._advance()
+                    generic_parts = []
+                    depth = 1
+                    while depth > 0 and not self._is_at_end():
+                        if self._check(TokenType.COMPARE_LT):
+                            depth += 1
+                            generic_parts.append('<')
+                        elif self._check(TokenType.COMPARE_GT):
+                            depth -= 1
+                            if depth > 0:
+                                generic_parts.append('>')
+                        elif self._check(TokenType.COMMA):
+                            generic_parts.append(',')
+                        else:
+                            generic_parts.append(self._current().value)
+                        self._advance()
+                    param_info['generic'] = ''.join(generic_parts)
+
+            # Handle reference operator &
+            if self._match(TokenType.AMPERSAND):
+                param_info['ref'] = True
+
+            # Get parameter name
+            if self._check(TokenType.IDENTIFIER):
+                param_name = self._advance().value
+                if param_info:
+                    params.append({'name': param_name, **param_info})
+                else:
+                    params.append(param_name)
+            elif self._check(TokenType.KEYWORD):
+                # Parameter name could be a keyword
+                param_name = self._advance().value
+                if param_info:
+                    params.append({'name': param_name, **param_info})
+                else:
+                    params.append(param_name)
+
+            self._match(TokenType.COMMA)
+
+        self._expect(TokenType.PAREN_END)
+
+        # Parse function body
+        node = ASTNode('function', value={
+            'name': name,
+            'params': params,
+            'return_type': return_type,
+            'generic_type': generic_type,
+            'modifiers': modifiers
+        }, children=[])
+
+        self._expect(TokenType.BLOCK_START)
+
+        while not self._check(TokenType.BLOCK_END) and not self._is_at_end():
+            stmt = self._parse_statement()
+            if stmt:
+                node.children.append(stmt)
+
+        self._expect(TokenType.BLOCK_END)
+        return node
+
+    def _looks_like_typed_variable(self) -> bool:
+        """Check if current position looks like a typed variable declaration:
+        type_name varName; or type_name<T> varName; or type_name varName = value;
+        """
+        # Save position
+        saved_pos = self.pos
+
+        # Must start with a type keyword (int, string, stack, vector, etc.)
+        if not self._check(TokenType.KEYWORD):
+            return False
+
+        type_name = self._current().value
+
+        # Skip known type keywords
+        type_keywords = {'int', 'string', 'float', 'bool', 'dynamic', 'void',
+                        'stack', 'vector', 'datastruct', 'dataspace', 'shuffled',
+                        'iterator', 'combo', 'array', 'openquote', 'json'}
+        if type_name not in type_keywords:
+            return False
+
+        self._advance()
+
+        # Check for optional generic <T>
+        if self._match(TokenType.COMPARE_LT):
+            # Skip until >
+            depth = 1
+            while depth > 0 and not self._is_at_end():
+                if self._check(TokenType.COMPARE_LT):
+                    depth += 1
+                elif self._check(TokenType.COMPARE_GT):
+                    depth -= 1
+                self._advance()
+
+        # Next should be an identifier (variable name), not '(' (function) or ';'
+        result = self._check(TokenType.IDENTIFIER)
+
+        # Restore position
+        self.pos = saved_pos
+        return result
+
+    def _parse_typed_variable(self) -> Optional[ASTNode]:
+        """Parse a typed variable declaration: type varName; or type<T> varName = value;"""
+        # Get type name
+        type_name = self._advance().value  # Consume type keyword
+
+        # Check for generic type <T>
+        element_type = None
+        if self._match(TokenType.COMPARE_LT):
+            # Get element type
+            if self._check(TokenType.KEYWORD) or self._check(TokenType.IDENTIFIER):
+                element_type = self._advance().value
+            self._expect(TokenType.COMPARE_GT)
+
+        # Get variable name
+        if not self._check(TokenType.IDENTIFIER):
+            return None
+        var_name = self._advance().value
+
+        # Check for assignment or just declaration
+        value = None
+        if self._match(TokenType.EQUALS):
+            value = self._parse_expression()
+
+        self._match(TokenType.SEMICOLON)
+
+        return ASTNode('typed_declaration', value={
+            'type': type_name,
+            'element_type': element_type,
+            'name': var_name,
+            'value': value
+        })
+
     def parse_program(self) -> ASTNode:
         """Parse a standalone program (no service wrapper)"""
         root = ASTNode('program', children=[])
@@ -597,6 +919,39 @@ class CSSLParser:
                 root.children.append(self._parse_struct())
             elif self._match_keyword('define'):
                 root.children.append(self._parse_define())
+            # Check for C-style typed function declarations
+            elif self._looks_like_function_declaration():
+                root.children.append(self._parse_typed_function())
+            # Check for typed variable declarations (int x;, stack<string> s;)
+            elif self._looks_like_typed_variable():
+                decl = self._parse_typed_variable()
+                if decl:
+                    root.children.append(decl)
+            # Handle service blocks
+            elif self._match_keyword('service-init'):
+                root.children.append(self._parse_service_init())
+            elif self._match_keyword('service-include'):
+                root.children.append(self._parse_service_include())
+            elif self._match_keyword('service-run'):
+                root.children.append(self._parse_service_run())
+            elif self._match_keyword('package'):
+                root.children.append(self._parse_package())
+            elif self._match_keyword('package-includes'):
+                root.children.append(self._parse_package_includes())
+            # Handle global declarations
+            elif self._match_keyword('global'):
+                stmt = self._parse_expression_statement()
+                if stmt:
+                    # Wrap in global_assignment to mark as global variable
+                    global_stmt = ASTNode('global_assignment', value=stmt)
+                    root.children.append(global_stmt)
+            elif self._check(TokenType.GLOBAL_REF):
+                stmt = self._parse_expression_statement()
+                if stmt:
+                    # Wrap in global_assignment to mark as global variable (same as 'global' keyword)
+                    global_stmt = ASTNode('global_assignment', value=stmt)
+                    root.children.append(global_stmt)
+            # Handle statements
             elif self._check(TokenType.IDENTIFIER) or self._check(TokenType.AT) or self._check(TokenType.SELF_REF):
                 stmt = self._parse_expression_statement()
                 if stmt:
@@ -609,6 +964,9 @@ class CSSLParser:
                 root.children.append(self._parse_for())
             elif self._match_keyword('foreach'):
                 root.children.append(self._parse_foreach())
+            # Skip comments and newlines
+            elif self._check(TokenType.COMMENT) or self._check(TokenType.NEWLINE):
+                self._advance()
             else:
                 self._advance()
 
@@ -837,8 +1195,31 @@ class CSSLParser:
 
         if self._match(TokenType.PAREN_START):
             while not self._check(TokenType.PAREN_END):
+                param_info = {}
+                # Handle 'open' keyword for open parameters
+                if self._match_keyword('open'):
+                    param_info['open'] = True
+                # Handle type annotations (e.g., string, int, dynamic, etc.)
+                if self._check(TokenType.KEYWORD):
+                    param_info['type'] = self._advance().value
+                # Handle reference operator &
+                if self._match(TokenType.AMPERSAND):
+                    param_info['ref'] = True
+                # Get parameter name
                 if self._check(TokenType.IDENTIFIER):
-                    params.append(self._advance().value)
+                    param_name = self._advance().value
+                    if param_info:
+                        params.append({'name': param_name, **param_info})
+                    else:
+                        params.append(param_name)
+                    self._match(TokenType.COMMA)
+                elif self._check(TokenType.KEYWORD):
+                    # Parameter name could be a keyword like 'Params'
+                    param_name = self._advance().value
+                    if param_info:
+                        params.append({'name': param_name, **param_info})
+                    else:
+                        params.append(param_name)
                     self._match(TokenType.COMMA)
                 else:
                     break
@@ -878,6 +1259,15 @@ class CSSLParser:
             return self._parse_try()
         elif self._match_keyword('await'):
             return self._parse_await()
+        elif self._match_keyword('define'):
+            # Nested define function
+            return self._parse_define()
+        elif self._looks_like_typed_variable():
+            # Typed variable declaration (e.g., stack<string> myStack;)
+            return self._parse_typed_variable()
+        elif self._looks_like_function_declaration():
+            # Nested typed function (e.g., void Level2() { ... })
+            return self._parse_typed_function()
         elif self._check(TokenType.IDENTIFIER) or self._check(TokenType.AT):
             return self._parse_expression_statement()
         else:
@@ -1077,6 +1467,9 @@ class CSSLParser:
             # Check for define statements inside action block
             if self._match_keyword('define'):
                 node.children.append(self._parse_define())
+            # Check for typed function definitions (nested functions)
+            elif self._looks_like_function_declaration():
+                node.children.append(self._parse_typed_function())
             else:
                 stmt = self._parse_statement()
                 if stmt:
@@ -1320,12 +1713,39 @@ class CSSLParser:
         if self._match(TokenType.MINUS):
             operand = self._parse_unary()
             return ASTNode('unary', value={'op': '-', 'operand': operand})
+        if self._match(TokenType.AMPERSAND):
+            # Reference operator: &variable or &@module
+            operand = self._parse_unary()
+            return ASTNode('reference', value=operand)
 
         return self._parse_primary()
 
     def _parse_primary(self) -> ASTNode:
         if self._match(TokenType.AT):
-            return self._parse_module_reference()
+            node = self._parse_module_reference()
+            # Continue to check for calls, indexing, member access on module refs
+            while True:
+                if self._match(TokenType.PAREN_START):
+                    # Function call on module ref: @Module.method()
+                    args = []
+                    while not self._check(TokenType.PAREN_END) and not self._is_at_end():
+                        args.append(self._parse_expression())
+                        if not self._check(TokenType.PAREN_END):
+                            self._expect(TokenType.COMMA)
+                    self._expect(TokenType.PAREN_END)
+                    node = ASTNode('call', value={'callee': node, 'args': args})
+                elif self._match(TokenType.DOT):
+                    # Member access: @Module.property
+                    member = self._advance().value
+                    node = ASTNode('member_access', value={'object': node, 'member': member})
+                elif self._match(TokenType.BRACKET_START):
+                    # Index access: @Module[index]
+                    index = self._parse_expression()
+                    self._expect(TokenType.BRACKET_END)
+                    node = ASTNode('index_access', value={'object': node, 'index': index})
+                else:
+                    break
+            return node
 
         if self._check(TokenType.SELF_REF):
             # s@<name> self-reference to global struct
@@ -1342,6 +1762,31 @@ class CSSLParser:
                 node = ASTNode('call', value={'callee': node, 'args': args})
             return node
 
+        if self._check(TokenType.GLOBAL_REF):
+            # r@<name> global variable reference/declaration
+            token = self._advance()
+            node = ASTNode('global_ref', value=token.value, line=token.line, column=token.column)
+            # Check for member access, calls, indexing
+            while True:
+                if self._match(TokenType.PAREN_START):
+                    args = []
+                    while not self._check(TokenType.PAREN_END):
+                        args.append(self._parse_expression())
+                        if not self._check(TokenType.PAREN_END):
+                            self._expect(TokenType.COMMA)
+                    self._expect(TokenType.PAREN_END)
+                    node = ASTNode('call', value={'callee': node, 'args': args})
+                elif self._match(TokenType.DOT):
+                    member = self._advance().value
+                    node = ASTNode('member_access', value={'object': node, 'member': member})
+                elif self._match(TokenType.BRACKET_START):
+                    index = self._parse_expression()
+                    self._expect(TokenType.BRACKET_END)
+                    node = ASTNode('index_access', value={'object': node, 'index': index})
+                else:
+                    break
+            return node
+
         if self._check(TokenType.NUMBER):
             return ASTNode('literal', value=self._advance().value)