IncludeCPP 4.3.0__py3-none-any.whl → 4.6.0__py3-none-any.whl

includecpp/core/cssl/cssl_parser.py

@@ -158,6 +158,7 @@ KEYWORDS = {
     'public',       # Explicitly public (default)
     'static',       # Static method/function
     'embedded',     # Immediate &target replacement at registration (v4.2.5)
+    'native',       # Force C++ execution (no Python fallback)
     # CSSL Include Keywords
     'include', 'get',
     # Multi-language support (v4.1.0)
@@ -167,7 +168,8 @@ KEYWORDS = {
 # Function modifiers that can appear in any order before function name
 FUNCTION_MODIFIERS = {
     'undefined', 'open', 'meta', 'super', 'closed', 'private', 'virtual',
-    'sqlbased', 'const', 'public', 'static', 'global', 'shuffled', 'embedded'
+    'sqlbased', 'const', 'public', 'static', 'global', 'shuffled', 'embedded',
+    'native'  # Force C++ execution
 }
 
 # Type literals that create empty instances
@@ -1280,7 +1282,8 @@ class CSSLParser:
             'append_mode': append_mode,
             'append_ref_class': append_ref_class,
             'append_ref_member': append_ref_member,
-            'enforce_return_type': return_type is not None and 'meta' not in modifiers
+            # v4.3.2: Also disable strict return type enforcement for 'shuffled' modifier (returns tuple)
+            'enforce_return_type': return_type is not None and 'meta' not in modifiers and 'shuffled' not in modifiers and return_type != 'shuffled'
         }, children=[])
 
         self._expect(TokenType.BLOCK_START)
@@ -1420,6 +1423,31 @@ class CSSLParser:
                 root.children.append(self._parse_bytearrayed())
             elif self._match_keyword('define'):
                 root.children.append(self._parse_define())
+            # v4.5.1: Handle function modifiers (private, const, static, etc.) before define
+            elif self._check(TokenType.KEYWORD) and self._is_function_modifier(self._current().value):
+                modifiers = []
+                is_embedded = False
+                is_global = False
+                has_open_params = False
+                while self._check(TokenType.KEYWORD) and self._is_function_modifier(self._current().value):
+                    mod = self._advance().value
+                    modifiers.append(mod)
+                    if mod == 'embedded':
+                        is_embedded = True
+                    elif mod == 'global':
+                        is_global = True
+                    elif mod == 'open':
+                        has_open_params = True
+                # Now check what follows
+                if self._match_keyword('define'):
+                    root.children.append(self._parse_define(
+                        is_global=is_global, is_embedded=is_embedded,
+                        has_open_params=has_open_params, modifiers=modifiers
+                    ))
+                elif self._looks_like_function_declaration():
+                    root.children.append(self._parse_typed_function(modifiers=modifiers))
+                else:
+                    self.error(f"Expected 'define' or function declaration after modifiers: {modifiers}")
             # Check for C-style typed function declarations
             elif self._looks_like_function_declaration():
                 root.children.append(self._parse_typed_function())
@@ -1441,13 +1469,44 @@ class CSSLParser:
                 root.children.append(self._parse_package_includes())
             # Handle global declarations
             # v4.2.5: Handle 'embedded' keyword for immediate &target replacement
+            # v4.3.2: Extended to support enums: embedded EnumName &TargetEnum { ... }
+            # v4.3.2: Support 'open embedded define' syntax
+            elif self._match_keyword('open'):
+                # open can be followed by embedded or define
+                if self._match_keyword('embedded'):
+                    if self._match_keyword('define'):
+                        root.children.append(self._parse_define(is_embedded=True, has_open_params=True))
+                    elif self._looks_like_function_declaration():
+                        root.children.append(self._parse_typed_function(is_embedded=True, has_open_params=True))
+                    else:
+                        self.error("Expected 'define' or function declaration after 'open embedded'")
+                elif self._match_keyword('define'):
+                    root.children.append(self._parse_define(has_open_params=True))
+                else:
+                    self.error("Expected 'embedded' or 'define' after 'open'")
             elif self._match_keyword('embedded'):
-                if self._match_keyword('class'):
+                # v4.3.2: Support both 'embedded open define' and 'embedded define'
+                if self._match_keyword('open'):
+                    # embedded open define ...
+                    if self._match_keyword('define'):
+                        root.children.append(self._parse_define(is_embedded=True, has_open_params=True))
+                    elif self._looks_like_function_declaration():
+                        root.children.append(self._parse_typed_function(is_embedded=True, has_open_params=True))
+                    else:
+                        self.error("Expected 'define' or function declaration after 'embedded open'")
+                elif self._match_keyword('class'):
                     root.children.append(self._parse_class(is_embedded=True))
+                elif self._match_keyword('enum'):
+                    root.children.append(self._parse_enum(is_embedded=True))
+                elif self._match_keyword('define'):
+                    root.children.append(self._parse_define(is_embedded=True))
                 elif self._looks_like_function_declaration():
                     root.children.append(self._parse_typed_function(is_embedded=True))
+                elif self._check(TokenType.IDENTIFIER):
+                    # embedded Name &Target { ... } - could be enum override
+                    root.children.append(self._parse_embedded_override())
                 else:
-                    self.error("Expected 'class' or function declaration after 'embedded'")
+                    self.error("Expected 'class', 'enum', 'open', 'define', function declaration, or identifier after 'embedded'")
             elif self._match_keyword('global'):
                 # Check if followed by class or define (global class/function)
                 if self._match_keyword('class'):
@@ -1478,6 +1537,11 @@ class CSSLParser:
                 root.children.append(self._parse_for())
             elif self._match_keyword('foreach'):
                 root.children.append(self._parse_foreach())
+            # v4.3.2: Add try/catch and switch at top-level
+            elif self._match_keyword('try'):
+                root.children.append(self._parse_try())
+            elif self._match_keyword('switch'):
+                root.children.append(self._parse_switch())
             # Handle statements - keywords like 'instance', 'list', 'map' can be variable names
             # v4.2.1: Added LANG_INSTANCE_REF for lang$instance statements (js$GameData.score = 1337)
             elif (self._check(TokenType.IDENTIFIER) or self._check(TokenType.AT) or
@@ -1711,7 +1775,7 @@ class CSSLParser:
         self._expect(TokenType.BLOCK_END)
         return node
 
-    def _parse_enum(self) -> ASTNode:
+    def _parse_enum(self, is_embedded: bool = False) -> ASTNode:
         """Parse enum declaration.
 
         Syntax:
@@ -1722,9 +1786,22 @@ class CSSLParser:
                 VALUE4          // Auto value 11
             }
 
+            embedded enum NewEnum &OldEnum { ... }  // Replace OldEnum with NewEnum values
+
         Access values via EnumName::VALUE1
         """
         enum_name = self._advance().value
+
+        # v4.3.2: Check for &Target reference (enum replacement)
+        replace_target = None
+        if self._match(TokenType.AMPERSAND):
+            if self._check(TokenType.IDENTIFIER):
+                replace_target = self._advance().value
+            elif self._check(TokenType.AT):
+                self._advance()
+                if self._check(TokenType.IDENTIFIER):
+                    replace_target = '@' + self._advance().value
+
         self._expect(TokenType.BLOCK_START)
 
         members = []
@@ -1764,7 +1841,88 @@ class CSSLParser:
 
         return ASTNode('enum', value={
             'name': enum_name,
-            'members': members
+            'members': members,
+            'is_embedded': is_embedded,
+            'replace_target': replace_target
+        })
+
+    def _parse_embedded_override(self) -> ASTNode:
+        """Parse embedded override for enums/structs without explicit type keyword.
+
+        Syntax:
+            embedded __NewName &OldEnum { ... }      // Replace OldEnum
+            embedded __NewName &OldEnum ++ { ... }   // Add to OldEnum
+            embedded __NewName &OldEnum -- { ... }   // Remove from OldEnum
+
+        This creates a new definition that modifies the target.
+        """
+        # Get the new name
+        new_name = self._advance().value
+
+        # Expect &Target
+        if not self._match(TokenType.AMPERSAND):
+            self.error("Expected '&' followed by target name after embedded identifier")
+
+        # Get target name
+        target_name = None
+        if self._check(TokenType.IDENTIFIER):
+            target_name = self._advance().value
+        elif self._check(TokenType.AT):
+            self._advance()
+            if self._check(TokenType.IDENTIFIER):
+                target_name = '@' + self._advance().value
+        else:
+            self.error("Expected target name after '&'")
+
+        # Check for mode modifier: ++ (add) or -- (remove)
+        mode = 'replace'
+        if self._match(TokenType.PLUS_PLUS):
+            mode = 'add'
+        elif self._match(TokenType.MINUS_MINUS):
+            mode = 'remove'
+
+        self._expect(TokenType.BLOCK_START)
+
+        # Parse members (enum-style: NAME = value, NAME, etc.)
+        members = []
+        current_value = 0
+
+        while not self._check(TokenType.BLOCK_END) and not self._is_at_end():
+            if self._check(TokenType.NEWLINE):
+                self._advance()
+                continue
+
+            if self._check(TokenType.IDENTIFIER):
+                member_name = self._advance().value
+
+                # Check for explicit value
+                if self._match(TokenType.EQUALS):
+                    value_node = self._parse_expression()
+                    if isinstance(value_node, ASTNode) and value_node.type == 'literal':
+                        val = value_node.value
+                        if isinstance(val, dict) and 'value' in val:
+                            current_value = val['value']
+                        else:
+                            current_value = val
+                    else:
+                        current_value = value_node
+
+                members.append({'name': member_name, 'value': current_value})
+                if isinstance(current_value, int):
+                    current_value = current_value + 1
+
+                self._match(TokenType.COMMA)
+            else:
+                self._advance()
+
+        self._expect(TokenType.BLOCK_END)
+
+        return ASTNode('enum', value={
+            'name': new_name,
+            'members': members,
+            'is_embedded': True,
+            'replace_target': target_name,
+            'mode': mode  # 'replace', 'add', or 'remove'
         })
 
     def _parse_bytearrayed(self) -> ASTNode:
@@ -1823,6 +1981,29 @@ class CSSLParser:
                         # Wildcard - matches any value
                         pattern.append({'type': 'wildcard'})
                         self._advance()
+                    elif self._check(TokenType.BRACKET_START):
+                        # v4.3.2: List pattern - matches a list value: ["read", "write"]
+                        self._advance()  # consume [
+                        list_elements = []
+                        while not self._check(TokenType.BRACKET_END) and not self._is_at_end():
+                            if self._check(TokenType.COMMA):
+                                self._advance()
+                                continue
+                            if self._check(TokenType.STRING):
+                                list_elements.append(self._advance().value)
+                            elif self._check(TokenType.NUMBER):
+                                list_elements.append(self._advance().value)
+                            elif self._check(TokenType.KEYWORD):
+                                kw = self._current().value
+                                if kw in ('true', 'True'):
+                                    list_elements.append(True)
+                                elif kw in ('false', 'False'):
+                                    list_elements.append(False)
+                                self._advance()
+                            else:
+                                self._advance()
+                        self._expect(TokenType.BRACKET_END)
+                        pattern.append({'type': 'list', 'values': list_elements})
                     elif self._check(TokenType.NUMBER):
                         # Check for hex format like 0x28
                         token = self._current()
@@ -1839,6 +2020,10 @@ class CSSLParser:
                     elif self._check(TokenType.STRING):
                         value = self._advance().value
                         pattern.append({'type': 'value', 'value': value})
+                    elif self._check(TokenType.BOOLEAN):
+                        # v4.3.2: Handle true/false which are tokenized as BOOLEAN
+                        value = self._advance().value
+                        pattern.append({'type': 'value', 'value': value})
                     elif self._check(TokenType.KEYWORD):
                         kw = self._current().value
                         if kw in ('true', 'True'):
@@ -1916,15 +2101,26 @@ class CSSLParser:
                     self._expect(TokenType.BLOCK_END)
                 default_block = body_children
 
-            # Parse function reference: &funcName;
+            # Parse function reference: &funcName; or &funcName(arg1, arg2);
             elif self._check(TokenType.AMPERSAND):
                 self._advance()  # consume &
                 if self._check(TokenType.IDENTIFIER):
                     func_name = self._advance().value
+                    # v4.3.2: Support function references with parameters: &testfunc(1, 2)
+                    func_args = []
+                    if self._check(TokenType.PAREN_START):
+                        self._advance()  # consume (
+                        while not self._check(TokenType.PAREN_END) and not self._is_at_end():
+                            arg = self._parse_expression()
+                            func_args.append(arg)
+                            if not self._check(TokenType.PAREN_END):
+                                self._expect(TokenType.COMMA)
+                        self._expect(TokenType.PAREN_END)
                     func_refs.append({
                         'position': len(func_refs),
                         'hex_pos': f"0x{len(func_refs):x}",
-                        'func_ref': func_name
+                        'func_ref': func_name,
+                        'args': func_args  # v4.3.2: Store arguments for simulation
                     })
                 self._match(TokenType.SEMICOLON)
 
@@ -2322,18 +2518,21 @@ class CSSLParser:
 
         return params
 
-    def _parse_define(self, is_global: bool = False) -> ASTNode:
+    def _parse_define(self, is_global: bool = False, is_embedded: bool = False, has_open_params: bool = False, modifiers: list = None) -> ASTNode:
         """Parse define function declaration.
 
         Syntax:
             define MyFunc(args) { }                    // Local function
             global define MyFunc(args) { }             // Global function
+            private define MyFunc(args) { }            // Private function (v4.5.1)
             define @MyFunc(args) { }                   // Global function (alternative)
             define *MyFunc(args) { }                   // Non-null: must never return None
             define MyFunc(args) : extends OtherFunc { }     // Inherit local vars
             define MyFunc(args) : overwrites OtherFunc { }  // Replace OtherFunc
             define MyFunc(args) : supports python { }       // Multi-language syntax
             define MyFunc(args) :: extends Parent::Method { }  // Method-level inheritance
+            embedded define MyFunc(args) &target { }       // Immediate &target replacement
+            open embedded define MyFunc(open Input) &target { }  // Open params + embedded
         """
         # Check for * prefix (non-null function - must return non-null)
         # Also *[type] for type exclusion (must NOT return that type)
@@ -2552,7 +2751,8 @@ class CSSLParser:
         node = ASTNode('function', value={
             'name': name,
             'is_global': is_global,
-            'is_embedded': False,  # v4.2.5: define uses delayed &target replacement
+            'is_embedded': is_embedded,  # v4.2.5: immediate &target replacement (v4.3.2: use param)
+            'has_open_params': has_open_params,  # v4.3.2: open embedded define support
             'params': params,
             'non_null': non_null,
             'exclude_type': exclude_type,  # *[type] - must NOT return this type
@@ -2572,7 +2772,9 @@ class CSSLParser:
             # v4.1.0: Multi-language support
             'supports_language': supports_language,
             # v4.2.0: Raw body for language transformation
-            'raw_body': raw_body
+            'raw_body': raw_body,
+            # v4.5.1: Function modifiers (private, const, static, etc.)
+            'modifiers': modifiers or []
         }, children=children)
 
         return node
@@ -2596,6 +2798,9 @@ class CSSLParser:
         elif self._match_keyword('continue'):
             self._match(TokenType.SEMICOLON)
             return ASTNode('continue')
+        # v4.5.1: Add throw statement parsing
+        elif self._match_keyword('throw'):
+            return self._parse_throw()
         elif self._match_keyword('try'):
             return self._parse_try()
         elif self._match_keyword('await'):
@@ -2988,9 +3193,39 @@ class CSSLParser:
         value = self._parse_expression()
         self._expect(TokenType.PAREN_END)
 
-        # v4.2.5: Check if this is a param switch (switch on open params identifier)
+        # v4.2.5: Check if this is a param switch (switch on open params)
         # Syntax: switch(Params): case name: ... except age: ... always: ... finally: ...
-        is_param_switch = (value.type == 'identifier')
+        # v4.3.2: Detect param_switch by:
+        #   1. Explicit "Params" identifier, OR
+        #   2. Any identifier if case uses param-style conditions (& / not)
+        is_param_switch = (value.type == 'identifier' and value.value == 'Params')
+
+        # v4.3.2: Also check if case syntax uses param-style conditions
+        # Look ahead to see if first case uses & or 'not' operators
+        if not is_param_switch and value.type == 'identifier':
+            saved_pos = self.pos
+            # Skip optional : and {
+            if self._check(TokenType.COLON):
+                self._advance()
+            if self._check(TokenType.BLOCK_START):
+                self._advance()
+            # Check for 'case' keyword
+            if self._check_keyword('case'):
+                self._advance()  # skip 'case'
+                # Look for & or 'not' before : (indicates param switch)
+                depth = 0
+                while not self._is_at_end():
+                    if self._check(TokenType.COLON) and depth == 0:
+                        break
+                    if self._check(TokenType.PAREN_START):
+                        depth += 1
+                    if self._check(TokenType.PAREN_END):
+                        depth -= 1
+                    if self._check(TokenType.AMPERSAND) or self._check_keyword('not') or self._check(TokenType.NOT):
+                        is_param_switch = True
+                        break
+                    self._advance()
+            self.pos = saved_pos  # Restore position
 
         if is_param_switch:
             return self._parse_param_switch(value)
@@ -3112,14 +3347,17 @@ class CSSLParser:
     def _parse_param_condition(self) -> dict:
         """Parse param switch condition.
 
-        Syntax:
+        v4.3.2: Enhanced to support multiple styles:
             name            -> {'type': 'exists', 'param': 'name'}
             not name        -> {'type': 'not', 'param': 'name'}
+            !name           -> {'type': 'not', 'param': 'name'}
             name & age      -> {'type': 'and', 'left': {...}, 'right': {...}}
             name & not age  -> {'type': 'and', 'left': {...}, 'right': {'type': 'not', ...}}
+            name & !age     -> {'type': 'and', 'left': {...}, 'right': {'type': 'not', ...}}
+            name || age     -> {'type': 'or', 'left': {...}, 'right': {...}}
         """
-        # Check for 'not' prefix
-        negated = self._match_keyword('not')
+        # Check for 'not' or '!' prefix
+        negated = self._match_keyword('not') or self._match(TokenType.NOT)
         param_name = self._advance().value
 
         if negated:
@@ -3127,18 +3365,33 @@ class CSSLParser:
         else:
             condition = {'type': 'exists', 'param': param_name}
 
-        # Check for & (AND) combinations
-        while self._match(TokenType.AMPERSAND):
-            # Parse right side
-            right_negated = self._match_keyword('not')
-            right_param = self._advance().value
+        # Check for & (AND) or || (OR) combinations
+        while True:
+            if self._match(TokenType.AMPERSAND):
+                # AND operator - check for 'not' or '!' prefix on right side
+                right_negated = self._match_keyword('not') or self._match(TokenType.NOT)
+                right_param = self._advance().value
 
-            if right_negated:
-                right_condition = {'type': 'not', 'param': right_param}
-            else:
-                right_condition = {'type': 'exists', 'param': right_param}
+                if right_negated:
+                    right_condition = {'type': 'not', 'param': right_param}
+                else:
+                    right_condition = {'type': 'exists', 'param': right_param}
+
+                condition = {'type': 'and', 'left': condition, 'right': right_condition}
 
-            condition = {'type': 'and', 'left': condition, 'right': right_condition}
+            elif self._match(TokenType.OR):
+                # OR operator (||) - check for 'not' or '!' prefix on right side
+                right_negated = self._match_keyword('not') or self._match(TokenType.NOT)
+                right_param = self._advance().value
+
+                if right_negated:
+                    right_condition = {'type': 'not', 'param': right_param}
+                else:
+                    right_condition = {'type': 'exists', 'param': right_param}
+
+                condition = {'type': 'or', 'left': condition, 'right': right_condition}
+            else:
+                break
 
         return condition
 
@@ -3226,8 +3479,11 @@ class CSSLParser:
         self._expect(TokenType.BLOCK_END)
         node.children.append(try_block)
 
-        # v4.2.6: Skip optional semicolon between try block and catch
-        self._match(TokenType.SEMICOLON)
+        # v4.2.6: Skip optional semicolons between try block and catch
+        # v4.5.1: Also skip comments for better .cssl-pl file support
+        while self._match(TokenType.SEMICOLON) or self._check(TokenType.COMMENT):
+            if self._check(TokenType.COMMENT):
+                self._advance()
 
         if self._match_keyword('catch'):
             error_var = None
@@ -3244,8 +3500,11 @@ class CSSLParser:
             self._expect(TokenType.BLOCK_END)
             node.children.append(catch_block)
 
-        # v4.2.6: Skip optional semicolon between catch and finally
-        self._match(TokenType.SEMICOLON)
+        # v4.2.6: Skip optional semicolons between catch and finally
+        # v4.5.1: Also skip comments for better .cssl-pl file support
+        while self._match(TokenType.SEMICOLON) or self._check(TokenType.COMMENT):
+            if self._check(TokenType.COMMENT):
+                self._advance()
 
         # v4.2.6: Add finally support
         if self._match_keyword('finally'):
@@ -3260,6 +3519,26 @@ class CSSLParser:
 
         return node
 
+    def _parse_throw(self) -> ASTNode:
+        """Parse throw statement: throw expression;
+
+        v4.5.1: Added throw statement for exception handling.
+
+        Syntax:
+            throw "Error message";
+            throw error_variable;
+            throw MyException("details");
+        """
+        # Parse the expression to throw (can be string, variable, or function call)
+        if self._check(TokenType.SEMICOLON):
+            # throw; - re-throw current exception
+            self._advance()
+            return ASTNode('throw', value=None)
+
+        expr = self._parse_expression()
+        self._match(TokenType.SEMICOLON)
+        return ASTNode('throw', value=expr)
+
     def _parse_await(self) -> ASTNode:
         """Parse await statement: await expression;"""
         expr = self._parse_expression()
@@ -3458,7 +3737,8 @@ class CSSLParser:
             filter_info = {}
             # Parse type::helper=value patterns within this bracket
             while not self._check(TokenType.BRACKET_END) and not self._is_at_end():
-                if self._check(TokenType.IDENTIFIER) or self._check(TokenType.KEYWORD):
+                # Accept IDENTIFIER, KEYWORD, or TYPE_LITERAL (dict, list) as filter type
+                if self._check(TokenType.IDENTIFIER) or self._check(TokenType.KEYWORD) or self._check(TokenType.TYPE_LITERAL):
                     filter_type = self._advance().value
                     if self._match(TokenType.DOUBLE_COLON):
                         helper = self._advance().value
@@ -4245,7 +4525,7 @@ class CSSLParser:
     def _is_object_literal(self) -> bool:
         """Check if current position is an object literal { key = value } vs action block { expr; }
 
-        Object literal: { name = value; } or { "key" = value; }
+        Object literal: { name = value } or { "key" = value } or { "key": value } (Python-style)
         Action block: { %version; } or { "1.0.0" } or { call(); }
         """
         # Empty block is action block
@@ -4255,12 +4535,12 @@ class CSSLParser:
         # Save position for lookahead
         saved_pos = self.pos
 
-        # Check if it looks like key = value pattern
+        # Check if it looks like key = value or key: value pattern
         is_object = False
         if self._check(TokenType.IDENTIFIER) or self._check(TokenType.STRING):
             self._advance()  # skip key
-            if self._check(TokenType.EQUALS):
-                # Looks like object literal: { key = ...
+            if self._check(TokenType.EQUALS) or self._check(TokenType.COLON):
+                # Looks like object literal: { key = ... } or { "key": ... }
                 is_object = True
 
         # Restore position
@@ -4300,12 +4580,22 @@ class CSSLParser:
         return ASTNode('action_block', children=children)
 
     def _parse_object(self) -> ASTNode:
+        """Parse object/dict literal.
+
+        Supports both CSSL-style and Python-style syntax:
+            { key = value }       // CSSL style
+            { "key" = value }     // CSSL style with string key
+            { "key": value }      // Python style
+            { key: value }        // Python style with identifier key
+        """
         properties = {}
 
         while not self._check(TokenType.BLOCK_END) and not self._is_at_end():
             if self._check(TokenType.IDENTIFIER) or self._check(TokenType.STRING):
                 key = self._advance().value
-                self._expect(TokenType.EQUALS)
+                # Accept both = (CSSL) and : (Python) as key-value separator
+                if not self._match(TokenType.EQUALS) and not self._match(TokenType.COLON):
+                    self.error(f"Expected '=' or ':' after key '{key}' in object literal")
                 value = self._parse_expression()
                 properties[key] = value
                 self._match(TokenType.SEMICOLON)
@@ -4367,12 +4657,117 @@ def parse_cssl_program(source: str) -> ASTNode:
     return parser.parse_program()
 
 
-def tokenize_cssl(source: str) -> List[Token]:
-    """Tokenize CSSL source code (useful for syntax highlighting)"""
+def tokenize_cssl(source: str, use_cpp: bool = True) -> List[Token]:
+    """
+    Tokenize CSSL source code (useful for syntax highlighting).
+
+    Args:
+        source: CSSL source code
+        use_cpp: If True, use C++ acceleration when available (default True)
+
+    Returns:
+        List of Token objects
+    """
+    # Try C++ tokenization if available and requested
+    if use_cpp:
+        try:
+            from . import _CPP_AVAILABLE, _cpp_module
+            if _CPP_AVAILABLE and _cpp_module and hasattr(_cpp_module, 'Lexer'):
+                lexer = _cpp_module.Lexer(source)
+                cpp_tokens = lexer.tokenize()
+                # Convert C++ tokens to Python Token objects
+                return _convert_cpp_tokens(cpp_tokens)
+        except (ImportError, Exception):
+            pass
+
+    # Fall back to Python lexer
     lexer = CSSLLexer(source)
     return lexer.tokenize()
 
 
+def _convert_cpp_tokens(cpp_tokens: list) -> List[Token]:
+    """Convert C++ Token objects to Python Token objects."""
+    result = []
+    for ct in cpp_tokens:
+        # Map C++ token type to Python TokenType
+        token_type = _map_cpp_token_type(ct.type)
+
+        # Get value based on value_type
+        if ct.value_type == 'string':
+            value = ct.str_value
+        elif ct.value_type == 'number':
+            value = ct.num_value
+        elif ct.value_type == 'bool':
+            value = ct.bool_value
+        else:
+            value = ct.str_value
+
+        result.append(Token(token_type, value, ct.line, ct.column))
+
+    return result
+
+
+def _map_cpp_token_type(cpp_type: int) -> TokenType:
+    """Map C++ token type integer to Python TokenType enum."""
+    # This mapping must match the C++ TokenType enum in cssl_core.cpp
+    type_map = {
+        0: TokenType.KEYWORD,
+        1: TokenType.IDENTIFIER,
+        2: TokenType.STRING,
+        3: TokenType.NUMBER,
+        4: TokenType.BOOLEAN,
+        5: TokenType.NULL,
+        6: TokenType.OPERATOR,
+        7: TokenType.INJECT_LEFT,
+        8: TokenType.INJECT_RIGHT,
+        9: TokenType.INJECT_PLUS_LEFT,
+        10: TokenType.INJECT_PLUS_RIGHT,
+        11: TokenType.INJECT_MINUS_LEFT,
+        12: TokenType.INJECT_MINUS_RIGHT,
+        13: TokenType.INFUSE_LEFT,
+        14: TokenType.INFUSE_RIGHT,
+        15: TokenType.FLOW_RIGHT,
+        16: TokenType.FLOW_LEFT,
+        17: TokenType.EQUALS,
+        18: TokenType.COMPARE_EQ,
+        19: TokenType.COMPARE_NE,
+        20: TokenType.COMPARE_LT,
+        21: TokenType.COMPARE_GT,
+        22: TokenType.COMPARE_LE,
+        23: TokenType.COMPARE_GE,
+        24: TokenType.PLUS,
+        25: TokenType.MINUS,
+        26: TokenType.MULTIPLY,
+        27: TokenType.DIVIDE,
+        28: TokenType.MODULO,
+        29: TokenType.AND,
+        30: TokenType.OR,
+        31: TokenType.NOT,
+        32: TokenType.AMPERSAND,
+        33: TokenType.BLOCK_START,
+        34: TokenType.BLOCK_END,
+        35: TokenType.PAREN_START,
+        36: TokenType.PAREN_END,
+        37: TokenType.BRACKET_START,
+        38: TokenType.BRACKET_END,
+        39: TokenType.SEMICOLON,
+        40: TokenType.COLON,
+        41: TokenType.DOUBLE_COLON,
+        42: TokenType.COMMA,
+        43: TokenType.DOT,
+        44: TokenType.AT,
+        45: TokenType.GLOBAL_REF,
+        46: TokenType.SNAPSHOT_REF,
+        47: TokenType.ARROW,
+        48: TokenType.LAMBDA,
+        49: TokenType.TERNARY,
+        50: TokenType.INCREMENT,
+        51: TokenType.DECREMENT,
+        52: TokenType.EOF,
+    }
+    return type_map.get(cpp_type, TokenType.IDENTIFIER)
+
+
 # Export public API
 __all__ = [
     'TokenType', 'Token', 'ASTNode',