IncludeCPP 4.3.0__py3-none-any.whl → 4.6.0__py3-none-any.whl

includecpp/core/cssl/cpp/cssl_core.cp

@@ -0,0 +1,108 @@
+SOURCE(include/cssl_core.cpp) cssl_core
+
+PUBLIC(
+    // Full Interpreter - Main Entry Point
+    cssl_core FUNC(run_cssl)
+    cssl_core FUNC(version)
+    cssl_core FUNC(is_keyword)
+
+    // Lexer & Token Classes
+    cssl_core CLASS(Lexer) {
+        CONSTRUCTOR(const std::string&)
+        METHOD(tokenize)
+    }
+    cssl_core CLASS(Token) {
+        CONSTRUCTOR()
+        CONSTRUCTOR(int, int, int)
+        CONSTRUCTOR(int, const std::string&, int, int)
+        CONSTRUCTOR(int, double, int, int)
+        CONSTRUCTOR(int, bool, int, int)
+        FIELD(type)
+        FIELD(str_value)
+        FIELD(num_value)
+        FIELD(bool_value)
+        FIELD(value_type)
+        FIELD(line)
+        FIELD(column)
+    }
+
+    // Full Interpreter Class
+    cssl_core CLASS(Interpreter) {
+        CONSTRUCTOR()
+        METHOD(run)
+        METHOD(run_string)
+    }
+
+    // Value Class (for advanced usage)
+    cssl_core CLASS(Value) {
+        CONSTRUCTOR()
+        CONSTRUCTOR(bool)
+        CONSTRUCTOR(int64_t)
+        CONSTRUCTOR(double)
+        CONSTRUCTOR(const std::string&)
+        METHOD_CONST(as_bool)
+        METHOD_CONST(as_float)
+        METHOD_CONST(as_int)
+        METHOD_CONST(as_string)
+        METHOD_CONST(get_type)
+        METHOD_CONST(is_bool)
+        METHOD_CONST(is_dict)
+        METHOD_CONST(is_float)
+        METHOD_CONST(is_int)
+        METHOD_CONST(is_list)
+        METHOD_CONST(is_null)
+        METHOD_CONST(is_number)
+        METHOD_CONST(is_string)
+        METHOD_CONST(to_bool)
+        METHOD_CONST(to_string)
+    }
+
+    // String Operations
+    cssl_core FUNC(str_concat)
+    cssl_core FUNC(str_contains)
+    cssl_core FUNC(str_join)
+    cssl_core FUNC(str_lower)
+    cssl_core FUNC(str_replace)
+    cssl_core FUNC(str_split)
+    cssl_core FUNC(str_trim)
+    cssl_core FUNC(str_upper)
+    cssl_core FUNC(str_reverse)
+    cssl_core FUNC(str_len)
+    cssl_core FUNC(str_repeat)
+    cssl_core FUNC(str_startswith)
+    cssl_core FUNC(str_endswith)
+    cssl_core FUNC(str_indexof)
+    cssl_core FUNC(str_substr)
+    cssl_core FUNC(str_cmp)
+
+    // Math Operations
+    cssl_core FUNC(math_clamp)
+    cssl_core FUNC(math_ipow)
+    cssl_core FUNC(math_pow)
+    cssl_core FUNC(math_mod)
+    cssl_core FUNC(math_abs)
+    cssl_core FUNC(math_min)
+    cssl_core FUNC(math_max)
+    cssl_core FUNC(math_floor)
+    cssl_core FUNC(math_ceil)
+
+    // Array Operations
+    cssl_core FUNC(array_sum)
+    cssl_core FUNC(array_isum)
+    cssl_core FUNC(array_avg)
+    cssl_core FUNC(array_min)
+    cssl_core FUNC(array_max)
+    cssl_core FUNC(range)
+
+    // Loop Helpers
+    cssl_core FUNC(loop_check_lt)
+    cssl_core FUNC(loop_check_le)
+    cssl_core FUNC(loop_check_gt)
+    cssl_core FUNC(loop_check_ge)
+
+    // Comparison Helpers
+    cssl_core FUNC(num_cmp)
+    cssl_core FUNC(eq_int)
+    cssl_core FUNC(eq_float)
+    cssl_core FUNC(eq_str)
+)

includecpp/core/cssl/cpp/cssl_lexer.hpp

@@ -0,0 +1,280 @@
+/**
+ * CSSL Lexer - High-performance tokenizer for CSSL language
+ *
+ * Part of IncludeCPP's self-hosting optimization strategy.
+ * This C++ implementation provides 10-20x speedup over Python.
+ *
+ * Copyright (c) 2024-2026 Lilias Hatterscheidt
+ * Licensed under MIT License
+ */
+
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <vector>
+#include <variant>
+#include <unordered_set>
+#include <cstdint>
+#include <stdexcept>
+
+namespace cssl {
+
+/**
+ * Token types matching Python cssl_parser.py TokenType enum
+ */
+enum class TokenType {
+    // Basic types
+    KEYWORD,
+    IDENTIFIER,
+    STRING,
+    STRING_INTERP,      // <variable> in strings
+    NUMBER,
+    BOOLEAN,
+    NULL_T,
+    TYPE_LITERAL,       // list, dict as type literals
+    TYPE_GENERIC,       // datastruct<T>, shuffled<T>, etc.
+
+    // Injection operators
+    INJECT_LEFT,        // <==
+    INJECT_RIGHT,       // ==>
+    INJECT_PLUS_LEFT,   // +<==
+    INJECT_PLUS_RIGHT,  // ==>+
+    INJECT_MINUS_LEFT,  // -<==
+    INJECT_MINUS_RIGHT, // ===>-
+
+    // Code Infusion operators
+    INFUSE_LEFT,        // <<==
+    INFUSE_RIGHT,       // ==>>
+    INFUSE_PLUS_LEFT,   // +<<==
+    INFUSE_PLUS_RIGHT,  // ==>>+
+    INFUSE_MINUS_LEFT,  // -<<==
+    INFUSE_MINUS_RIGHT, // ==>>-
+
+    // Flow operators
+    FLOW_RIGHT,         // ->
+    FLOW_LEFT,          // <-
+
+    // Assignment and comparison
+    EQUALS,             // =
+    COMPARE_EQ,         // ==
+    COMPARE_NE,         // !=
+    COMPARE_LT,         // <
+    COMPARE_GT,         // >
+    COMPARE_LE,         // <=
+    COMPARE_GE,         // >=
+
+    // Arithmetic
+    PLUS,               // +
+    MINUS,              // -
+    MULTIPLY,           // *
+    DIVIDE,             // /
+    MODULO,             // %
+
+    // Logical
+    AND,                // && or 'and'
+    OR,                 // || or 'or'
+    NOT,                // ! or 'not'
+    AMPERSAND,          // & (reference/bitwise)
+
+    // Delimiters
+    BLOCK_START,        // {
+    BLOCK_END,          // }
+    PAREN_START,        // (
+    PAREN_END,          // )
+    BRACKET_START,      // [
+    BRACKET_END,        // ]
+    SEMICOLON,          // ;
+    COLON,              // :
+    DOUBLE_COLON,       // ::
+    COMMA,              // ,
+    DOT,                // .
+
+    // References
+    AT,                 // @
+    GLOBAL_REF,         // r@name
+    SELF_REF,           // s@name
+    SHARED_REF,         // $name
+    CAPTURED_REF,       // %name
+    THIS_REF,           // this->
+
+    // Package/module
+    PACKAGE,
+    PACKAGE_INCLUDES,
+    AS,
+
+    // Special
+    COMMENT,
+    NEWLINE,
+    EOF_T,
+
+    // Super-functions (payload files)
+    SUPER_FUNC,         // #$run(), #$exec(), etc.
+
+    // Append/extension operators
+    PLUS_PLUS,          // ++ (constructor/function extension)
+    MINUS_MINUS,        // --
+
+    // Multi-language support
+    LANG_INSTANCE_REF   // cpp$InstanceName, py$Object
+};
+
+/**
+ * Token value - can hold string, number, or boolean
+ */
+using TokenValue = std::variant<std::monostate, std::string, double, bool>;
+
+/**
+ * Token structure
+ */
+struct Token {
+    TokenType type;
+    TokenValue value;
+    uint32_t line;
+    uint32_t column;
+
+    // Convenience getters
+    bool has_string() const { return std::holds_alternative<std::string>(value); }
+    bool has_number() const { return std::holds_alternative<double>(value); }
+    bool has_bool() const { return std::holds_alternative<bool>(value); }
+
+    const std::string& as_string() const { return std::get<std::string>(value); }
+    double as_number() const { return std::get<double>(value); }
+    bool as_bool() const { return std::get<bool>(value); }
+};
+
+/**
+ * Lexer error with source location
+ */
+class LexerError : public std::runtime_error {
+public:
+    LexerError(const std::string& message, uint32_t line, uint32_t column,
+               const std::string& source_line = "")
+        : std::runtime_error(build_message(message, line, column, source_line)),
+          line_(line), column_(column) {}
+
+    uint32_t line() const { return line_; }
+    uint32_t column() const { return column_; }
+
+private:
+    uint32_t line_;
+    uint32_t column_;
+
+    static std::string build_message(const std::string& msg, uint32_t line, uint32_t col,
+                                     const std::string& src_line) {
+        std::string result = "CSSL Lexer Error at line " + std::to_string(line) +
+                            ", column " + std::to_string(col) + ": " + msg;
+        if (!src_line.empty()) {
+            result += "\n  " + src_line;
+            if (col > 0) {
+                result += "\n  " + std::string(col - 1, ' ') + "^";
+            }
+        }
+        return result;
+    }
+};
+
+/**
+ * High-performance CSSL Lexer
+ *
+ * Converts CSSL source code into a vector of tokens.
+ * Optimized for speed with minimal allocations.
+ */
+class Lexer {
+public:
+    /**
+     * Construct lexer with source code
+     * @param source CSSL source code (must remain valid during tokenization)
+     */
+    explicit Lexer(std::string_view source);
+
+    /**
+     * Tokenize entire source and return all tokens
+     * @return Vector of tokens including EOF token at end
+     */
+    std::vector<Token> tokenize();
+
+    /**
+     * Get next token (streaming mode)
+     * @return Next token, EOF_T when done
+     */
+    Token next_token();
+
+    /**
+     * Check if more tokens available
+     */
+    bool has_more() const { return pos_ < source_.size(); }
+
+    /**
+     * Reset lexer to beginning
+     */
+    void reset();
+
+private:
+    std::string_view source_;
+    size_t pos_ = 0;
+    uint32_t line_ = 1;
+    uint32_t column_ = 1;
+
+    // Current character helpers
+    char current() const;
+    char peek(size_t offset = 1) const;
+    void advance(size_t count = 1);
+    bool at_end() const { return pos_ >= source_.size(); }
+
+    // Whitespace and comment handling
+    void skip_whitespace();
+    void skip_comment();
+    void skip_block_comment();
+
+    // Token readers
+    Token read_string(char quote);
+    Token read_raw_string();
+    Token read_number();
+    Token read_identifier();
+    Token read_operator();
+
+    // Special token readers
+    Token read_super_function();
+    Token read_global_ref();
+    Token read_self_ref();
+    Token read_shared_ref();
+    Token read_captured_ref();
+
+    // Operator readers
+    Token read_less_than();
+    Token read_greater_than();
+    Token read_equals();
+    Token read_not();
+    Token read_minus();
+    Token read_plus();
+
+    // Token creation helpers
+    Token make_token(TokenType type, TokenValue value = {}) const;
+
+    // Error helper
+    [[noreturn]] void error(const std::string& message) const;
+
+    // Static keyword sets (initialized once)
+    static const std::unordered_set<std::string> keywords_;
+    static const std::unordered_set<std::string> type_literals_;
+    static const std::unordered_set<std::string> type_generics_;
+    static const std::unordered_set<std::string> language_ids_;
+};
+
+/**
+ * Utility function: Tokenize source code
+ * @param source CSSL source code
+ * @return Vector of tokens
+ */
+inline std::vector<Token> tokenize(const std::string& source) {
+    Lexer lexer(source);
+    return lexer.tokenize();
+}
+
+/**
+ * Get string representation of token type
+ */
+const char* token_type_name(TokenType type);
+
+} // namespace cssl

includecpp/core/cssl/cssl_builtins.py

@@ -41,6 +41,7 @@ class CSSLBuiltins:
         self._functions['error'] = self.builtin_error
         self._functions['warn'] = self.builtin_warn
         self._functions['log'] = self.builtin_log
+        self._functions['encode'] = self.builtin_encode  # v4.3.2: Safe unicode encoding
 
         # Type conversion
         self._functions['int'] = self.builtin_int
@@ -380,6 +381,44 @@ class CSSLBuiltins:
                 encoded = output.encode(sys.stdout.encoding or 'utf-8', errors='replace')
                 print(encoded.decode(sys.stdout.encoding or 'utf-8', errors='replace'))
 
+    def builtin_encode(self, content: str, fallback: str = "?") -> str:
+        """Encode unicode/emoji characters for safe console output.
+
+        v4.3.2: Safely encode emojis and special unicode characters that may not
+        be supported by the current console encoding (e.g., Windows cp1252).
+
+        Usage:
+            printl("Status: " + encode("✅"));
+            printl("Result: " + encode("😂", "[emoji]"));
+
+            // With custom fallback
+            msg = encode("🎉 Success!", "[party]");
+
+        Args:
+            content: String containing unicode/emoji characters
+            fallback: Replacement string for unencodable characters (default: "?")
+
+        Returns:
+            String safe for console output
+        """
+        import sys
+        encoding = sys.stdout.encoding or 'utf-8'
+
+        try:
+            # Test if content can be encoded with current console encoding
+            content.encode(encoding)
+            return content
+        except UnicodeEncodeError:
+            # Replace unencodable characters with fallback
+            result = []
+            for char in content:
+                try:
+                    char.encode(encoding)
+                    result.append(char)
+                except UnicodeEncodeError:
+                    result.append(fallback)
+            return ''.join(result)
+
     def builtin_input(self, prompt: str = "") -> str:
         """Read user input from console.
 
@@ -645,19 +684,85 @@ class CSSLBuiltins:
     # ============= List Functions =============
 
     def builtin_push(self, lst: list, *items) -> list:
+        """Push items onto a list/stack/vector.
+
+        v4.5.1: For Stack/Vector/Array types, modifies in place.
+        For regular lists, creates a copy (immutable behavior).
+        """
+        if lst is None:
+            lst = []
+            lst.extend(items)
+            return lst
+
+        # v4.5.1: For CSSL typed containers, modify in place
+        from .cssl_types import Stack, Vector, Array, DataStruct
+        if isinstance(lst, (Stack, Vector, Array, DataStruct)):
+            for item in items:
+                lst.append(item)
+            return lst
+
+        # For regular lists, create a copy (immutable behavior)
         lst = list(lst)
         lst.extend(items)
         return lst
 
     def builtin_pop(self, lst: list, index: int = -1) -> Any:
+        """Pop item from list/stack/vector.
+
+        v4.5.1: For Stack/Vector/Array types, modifies in place.
+        """
+        if lst is None:
+            return None
+
+        # v4.5.1: For CSSL typed containers, modify in place
+        from .cssl_types import Stack, Vector, Array, DataStruct
+        if isinstance(lst, (Stack, Vector, Array, DataStruct)):
+            if len(lst) == 0:
+                return None
+            return lst.pop(index)
+
+        # For regular lists, create a copy
         lst = list(lst)
-        return lst.pop(index)
+        return lst.pop(index) if lst else None
 
     def builtin_shift(self, lst: list) -> Any:
+        """Remove and return first element.
+
+        v4.5.1: For Stack/Vector/Array types, modifies in place.
+        """
+        if lst is None:
+            return None
+
+        # v4.5.1: For CSSL typed containers, modify in place
+        from .cssl_types import Stack, Vector, Array, DataStruct
+        if isinstance(lst, (Stack, Vector, Array, DataStruct)):
+            if len(lst) == 0:
+                return None
+            return lst.pop(0)
+
+        # For regular lists, create a copy
         lst = list(lst)
         return lst.pop(0) if lst else None
 
     def builtin_unshift(self, lst: list, *items) -> list:
+        """Add items to the front of a list/stack/vector.
+
+        v4.5.1: For Stack/Vector/Array types, modifies in place.
+        """
+        if lst is None:
+            lst = []
+            for item in reversed(items):
+                lst.insert(0, item)
+            return lst
+
+        # v4.5.1: For CSSL typed containers, modify in place
+        from .cssl_types import Stack, Vector, Array, DataStruct
+        if isinstance(lst, (Stack, Vector, Array, DataStruct)):
+            for item in reversed(items):
+                lst.insert(0, item)
+            return lst
+
+        # For regular lists, create a copy
         lst = list(lst)
         for item in reversed(items):
             lst.insert(0, item)
@@ -2250,17 +2355,20 @@ class CSSLBuiltins:
 
         All definitions go into the namespace dict.
         embedded replacements are scoped to the namespace only.
+        Variables declared with 'global' keyword remain truly global (accessible via @).
         """
         # Create namespace dict
         namespace = {}
 
-        # Save current scope state
-        old_scope = self.runtime.scope.copy()
-        old_global = self.runtime.global_scope.copy()
+        # Save current scope state (copy the variables dicts)
+        old_scope_vars = dict(self.runtime.scope.variables)
+        old_global_vars = dict(self.runtime.global_scope.variables)
+        old_promoted_globals = dict(self.runtime._promoted_globals)
 
         # Track what gets added during payload execution
-        scope_before = set(self.runtime.scope.keys())
-        global_before = set(self.runtime.global_scope.keys())
+        scope_before = set(self.runtime.scope.variables.keys())
+        global_before = set(self.runtime.global_scope.variables.keys())
+        promoted_before = set(self.runtime._promoted_globals.keys())
 
         # Set a flag to tell runtime we're in namespace mode
         self.runtime._payload_namespace_mode = libname
@@ -2269,46 +2377,53 @@ class CSSLBuiltins:
             # Execute the payload
             self.runtime._execute_node(ast)
 
-            # Collect new definitions from scope
-            for key in self.runtime.scope:
-                if key not in scope_before:
-                    namespace[key] = self.runtime.scope[key]
+            # Find which variables were explicitly promoted to global (via 'global' keyword)
+            new_promoted = set(self.runtime._promoted_globals.keys()) - promoted_before
 
-            # Collect new definitions from global_scope (but not the namespace itself)
-            for key in self.runtime.global_scope:
-                if key not in global_before and key != libname:
-                    namespace[key] = self.runtime.global_scope[key]
+            # Collect new definitions from scope (but not promoted globals)
+            for key in self.runtime.scope.variables:
+                if key not in scope_before and key not in new_promoted:
+                    namespace[key] = self.runtime.scope.get(key)
+
+            # Collect new definitions from global_scope (but not namespace itself, not promoted globals)
+            for key in self.runtime.global_scope.variables:
+                if key not in global_before and key != libname and key not in new_promoted:
+                    namespace[key] = self.runtime.global_scope.get(key)
 
             # Handle embedded replacements: move replaced targets into namespace
             if hasattr(self.runtime, '_namespace_replacements'):
                 for target_name, new_impl in self.runtime._namespace_replacements.items():
                     # Get the original from before execution
-                    original = old_scope.get(target_name) or old_global.get(target_name)
+                    original = old_scope_vars.get(target_name) or old_global_vars.get(target_name)
                     if original is not None:
                         # Restore original globally
-                        if target_name in old_scope:
-                            self.runtime.scope[target_name] = original
-                        if target_name in old_global:
-                            self.runtime.global_scope[target_name] = original
+                        if target_name in old_scope_vars:
+                            self.runtime.scope.set(target_name, original)
+                        if target_name in old_global_vars:
+                            self.runtime.global_scope.set(target_name, original)
                     # Put the replacement in namespace
                     namespace[target_name] = new_impl
                 del self.runtime._namespace_replacements
 
-            # Remove new definitions from global scope (they're in namespace now)
-            for key in list(self.runtime.scope.keys()):
-                if key not in scope_before:
-                    del self.runtime.scope[key]
+            # Remove new definitions from scope (they're in namespace now)
+            # BUT keep promoted globals - they stay global!
+            for key in list(self.runtime.scope.variables.keys()):
+                if key not in scope_before and key not in new_promoted:
+                    del self.runtime.scope.variables[key]
+
+            for key in list(self.runtime.global_scope.variables.keys()):
+                if key not in global_before and key != libname and key not in new_promoted:
+                    del self.runtime.global_scope.variables[key]
 
-            for key in list(self.runtime.global_scope.keys()):
-                if key not in global_before and key != libname:
-                    del self.runtime.global_scope[key]
+            # Promoted globals stay in _promoted_globals and global_scope (not moved to namespace)
+            # They are accessible via @name from anywhere
 
         finally:
             # Clear namespace mode flag
             self.runtime._payload_namespace_mode = None
 
         # Register namespace in global scope
-        self.runtime.global_scope[libname] = namespace
+        self.runtime.global_scope.set(libname, namespace)
 
     def builtin_get(self, source: Any, key: str = None) -> Any:
         """