koskript 1.4.1__tar.gz → 1.4.2__tar.gz

{koskript-1.4.1 → koskript-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: koskript
-Version: 1.4.1
+Version: 1.4.2
 Summary: A programming language fully embbeded on python.
 Author: alesis.buzz
 License: Mozilla Public License Version 2.0
@@ -378,7 +378,7 @@ License: Mozilla Public License Version 2.0
           defined by the Mozilla Public License, v. 2.0.
         
 Project-URL: Homepage, https://github.com/alesis-buzz/koskript
-Keywords: scripting,programming-language,embedded,interpreter,lark
+Keywords: scripting,programming-language,embedded,interpreter,parser,lark
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -429,7 +429,7 @@ array.each(names, (name) { print("hello " + name) })
 - Closures that capture their defining scope, including `this` inside methods.
 - Constants with `const`.
 - Classes with inheritance, visibility, static methods and constructors.
-- Standard library: core builtins plus `map`, `array`, `string`, `math` and `json`.
+- Standard library: core builtins plus `map`, `array`, `string`, `bytes`, `math` and `json`.
 - Module imports with per-module resolution (`import "utils"`).
 - Configurable import search paths (`runtime.add_import_path`).
 - Native Python interop for functions, objects and classes.
@@ -457,6 +457,8 @@ git clone https://github.com/alesis-buzz/koskript.git
 ```
 
 Koskript requires Python 3.10+ and [Lark](https://github.com/lark-parser/lark).
+There is an experimental dependency-free parser behind
+`KoskriptRuntime(native_parser_experiment=True)`.
 
 ---
 
@@ -465,9 +467,9 @@ Koskript requires Python 3.10+ and [Lark](https://github.com/lark-parser/lark).
 | Document | What it covers |
 |---|---|
 | [Getting started](docs/getting-started.md) | Installation, first script, a complete example |
-| [Language reference](docs/language.md) | Types, variables, operators, control flow, errors, functions, wrappers, namespaces, closures, modules, loops, strings |
+| [Language reference](docs/language.md) | Types, variables, operators, control flow, errors, functions, wrappers, namespaces, closures, modules, loops, strings, bytes |
 | [Classes](docs/classes.md) | Inheritance, visibility, static members, constructors, `super` |
-| [Standard library](docs/standard-library.md) | Core builtins and the `map`, `array`, `string`, `math` and `json` namespaces |
+| [Standard library](docs/standard-library.md) | Core builtins and the `map`, `array`, `string`, `bytes`, `math` and `json` namespaces |
 | [Python interop](docs/python-interop.md) | Exposing Python values, callables and classes to scripts |
 | [Embedding guide](docs/embedding.md) | `KoskriptRuntime`, `run()`, error handling and stdlib activation |
 | [Roadmap](docs/roadmap.md) | What is done, what is planned, known limitations |

{koskript-1.4.1 → koskript-1.4.2}/README.md

@@ -32,7 +32,7 @@ array.each(names, (name) { print("hello " + name) })
 - Closures that capture their defining scope, including `this` inside methods.
 - Constants with `const`.
 - Classes with inheritance, visibility, static methods and constructors.
-- Standard library: core builtins plus `map`, `array`, `string`, `math` and `json`.
+- Standard library: core builtins plus `map`, `array`, `string`, `bytes`, `math` and `json`.
 - Module imports with per-module resolution (`import "utils"`).
 - Configurable import search paths (`runtime.add_import_path`).
 - Native Python interop for functions, objects and classes.
@@ -60,6 +60,8 @@ git clone https://github.com/alesis-buzz/koskript.git
 ```
 
 Koskript requires Python 3.10+ and [Lark](https://github.com/lark-parser/lark).
+There is an experimental dependency-free parser behind
+`KoskriptRuntime(native_parser_experiment=True)`.
 
 ---
 
@@ -68,9 +70,9 @@ Koskript requires Python 3.10+ and [Lark](https://github.com/lark-parser/lark).
 | Document | What it covers |
 |---|---|
 | [Getting started](docs/getting-started.md) | Installation, first script, a complete example |
-| [Language reference](docs/language.md) | Types, variables, operators, control flow, errors, functions, wrappers, namespaces, closures, modules, loops, strings |
+| [Language reference](docs/language.md) | Types, variables, operators, control flow, errors, functions, wrappers, namespaces, closures, modules, loops, strings, bytes |
 | [Classes](docs/classes.md) | Inheritance, visibility, static members, constructors, `super` |
-| [Standard library](docs/standard-library.md) | Core builtins and the `map`, `array`, `string`, `math` and `json` namespaces |
+| [Standard library](docs/standard-library.md) | Core builtins and the `map`, `array`, `string`, `bytes`, `math` and `json` namespaces |
 | [Python interop](docs/python-interop.md) | Exposing Python values, callables and classes to scripts |
 | [Embedding guide](docs/embedding.md) | `KoskriptRuntime`, `run()`, error handling and stdlib activation |
 | [Roadmap](docs/roadmap.md) | What is done, what is planned, known limitations |

{koskript-1.4.1 → koskript-1.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "koskript"
-version = "1.4.1"
+version = "1.4.2"
 description = "A programming language fully embbeded on python."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -19,6 +19,7 @@ keywords = [
     "programming-language",
     "embedded",
     "interpreter",
+    "parser",
     "lark"
 ]
 
@@ -37,6 +38,8 @@ classifiers = [
 
 
 dependencies = [
+    # only the default parser needs it; `native_parser_experiment=True`
+    # runs without it
     "lark>=1.3.1"
 ]
 

{koskript-1.4.1 → koskript-1.4.2}/src/koskript/__init__.py

@@ -1,23 +1,65 @@
-from lark import Lark
-from lark.exceptions import LarkError, UnexpectedInput
+import pathlib
+
 from .lang.emtypes import (KoskriptObject, Module, Namespace, Scope, ScopeInfo,
                            ThrownSignal)
 from .lang.interpreter import KoskriptInterpreter
-from .lang.astgen import KoskriptTransformer
+from .lang.parser import parse
 from .lang.errors import Errors
 from .stdlib import build_globals
-import pathlib, os
+import os
 
-_grammar_path = os.path.join(pathlib.Path(__file__).resolve().parent, "grammar.lark")
-with open(_grammar_path, "r") as _grammar_file:
-    grammar = Lark(_grammar_file, parser="lalr", maybe_placeholders=False)
+_grammar_path = os.path.join(pathlib.Path(__file__).resolve().parent,
+                             "grammar.lark")
 
 
 def _wrap(value):
     return value if isinstance(value, KoskriptObject) else KoskriptObject(value=value)
 
 
-def _format_syntax_error(code: str, error: LarkError) -> str:
+def _require_lark():
+    """Import Lark, with a message that points at the experimental parser."""
+    try:
+        from lark import Lark
+    except ImportError:
+        raise Errors.RuntimeError(
+            "the default parser needs the 'lark' package: install it, or pass "
+            "native_parser_experiment=True to use the native parser") from None
+    return Lark
+
+
+class _Grammar(object):
+    """The Lark grammar, with the LALR tables built on first use.
+
+    Building them takes about 200 ms, so it is deferred until a script is
+    actually parsed: runtimes created with ``native_parser_experiment=True``
+    never pay for them.
+    """
+
+    __slots__ = ("path", "parser")
+
+    def __init__(self, path):
+        self.path = path
+        self.parser = None
+
+    def parse(self, code: str):
+        """Parse ``code`` into a Lark tree."""
+        if self.parser is None:
+            with open(self.path, "r") as grammar_file:
+                self.parser = _require_lark()(
+                    grammar_file.read(), parser="lalr", maybe_placeholders=False)
+        return self.parser.parse(code)
+
+
+grammar = _Grammar(_grammar_path)
+
+# How many compiled sources a runtime keeps. Running the same script again is
+# then free of parsing and compiling; the oldest entry is dropped past this.
+_CHUNK_CACHE = 32
+
+
+def _format_syntax_error(code: str, error) -> str:
+    from lark.exceptions import UnexpectedInput
+
     if not isinstance(error, UnexpectedInput) or error.line is None:
         return str(error)
 
@@ -44,22 +86,35 @@ class KoskriptRuntime(object):
     Any Python value or callable exposed to scripts is wrapped automatically,
     so you can pass plain functions without building ``KoskriptObject`` by hand.
 
+    Scripts are parsed with Lark, the default and stable front end, which
+    needs the ``lark`` package. ``native_parser_experiment=True`` switches to
+    the experimental pure Python scanner and parser instead: it needs no
+    dependency, starts faster and parses about 6x quicker, and it is what
+    will become the default once it has been validated in production.
+
     >>> rt = KoskriptRuntime({"print": print})
     >>> rt.execute("local x = 10\\nx + 26")
     36
     """
 
-    def __init__(self, globals_map=None, _globals_=None, stdlib=True):
+    def __init__(self, globals_map=None, _globals_=None, stdlib=True,
+                 native_parser_experiment=False):
         if _globals_ is not None:
             globals_map = {**(globals_map or {}), **_globals_}
 
         self.globals = {}
+        self.native_parser_experiment = bool(native_parser_experiment)
         self.__interpreter__ = KoskriptInterpreter()
-        self.__ast__ = KoskriptTransformer()
+        self._ast = None
         self.__interpreter__.module_loader = self._load_module
         self._modules = {}
         self._loading = []
         self._import_paths = []
+        self._chunks = {}
+
+        if not self.native_parser_experiment:
+            # fail here instead of on the first script
+            _require_lark()
 
         if stdlib:
             self.register_many(build_globals(self.__interpreter__.call_value))
@@ -67,6 +122,23 @@ class KoskriptRuntime(object):
         if globals_map:
             self.register_many(globals_map)
 
+    @property
+    def __ast__(self):
+        """The Lark AST transformer, created the first time it is needed.
+
+        It lives in :mod:`koskript.lang.astgen`, which imports Lark, so
+        runtimes using the native parser never load it.
+        """
+        transformer = self._ast
+        if transformer is None:
+            from .lang.astgen import KoskriptTransformer
+            transformer = self._ast = KoskriptTransformer()
+        return transformer
+
+    @__ast__.setter
+    def __ast__(self, transformer):
+        self._ast = transformer
+
     def register(self, name: str, value):
         """Expose a Python value or callable to scripts.
 
@@ -114,9 +186,29 @@ class KoskriptRuntime(object):
         working directory of the Python process, then against the directories
         added with :meth:`add_import_path`.
 
+        The compiled form of a source is cached, so running the same source
+        again skips parsing and compiling it.
+
         Raises ``Errors.SyntaxError`` if the code cannot be parsed.
         """
-        return self.__interpreter__.run(self._compile(code))
+        try:
+            return self.__interpreter__.run_chunk(self._chunk(code))
+        except ThrownSignal as signal:
+            raise self.__interpreter__._uncaught(signal) from None
+
+    def _chunk(self, code: str):
+        """The compiled chunk of ``code``, compiled once per source."""
+        chunk = self._chunks.get(code)
+        if chunk is not None:
+            return chunk
+
+        chunk = self.__interpreter__.compiler.compile_chunk(self._compile(code))
+        if len(self._chunks) >= _CHUNK_CACHE:
+            # bounded, so a host that runs many different scripts does not
+            # grow the cache without limit
+            del self._chunks[next(iter(self._chunks))]
+        self._chunks[code] = chunk
+        return chunk
 
     def execute_module(self, path: str):
         """Execute ``path`` as a Koskript module and return the module.
@@ -132,12 +224,30 @@ class KoskriptRuntime(object):
         except ThrownSignal as signal:
             raise self.__interpreter__._uncaught(signal) from None
 
+    def _transform(self, tree):
+        """Run the AST transformer, unwrapping clean syntax errors."""
+        from lark.exceptions import VisitError
+
+        transformer = self.__ast__
+        try:
+            return transformer.transform(tree)
+        except VisitError as e:
+            if isinstance(e.orig_exc, Errors.SyntaxError):
+                raise e.orig_exc from None
+            raise
+
     def _compile(self, code: str):
+        """Parse ``code`` into the list of statements the compiler consumes."""
+        if self.native_parser_experiment:
+            return parse(code)
+
+        from lark.exceptions import LarkError
+
         try:
             tree = grammar.parse(code)
         except LarkError as e:
             raise Errors.SyntaxError(_format_syntax_error(code, e)) from e
-        ast = self.__ast__.transform(tree)
+        ast = self._transform(tree)
 
         if not isinstance(ast, list):
             ast = [ast]
@@ -182,14 +292,9 @@ class KoskriptRuntime(object):
                 f"cannot read module '{path}': {e}") from e
 
         try:
-            tree = grammar.parse(code)
-        except LarkError as e:
-            raise Errors.SyntaxError(
-                f"module '{path}':\n{_format_syntax_error(code, e)}") from e
-
-        ast = self.__ast__.transform(tree)
-        if not isinstance(ast, list):
-            ast = [ast]
+            ast = self._compile(code)
+        except Errors.SyntaxError as e:
+            raise Errors.SyntaxError(f"module '{path}':\n{e}") from None
 
         interpreter = self.__interpreter__
         info = ScopeInfo(interpreter.root_info)

{koskript-1.4.1 → koskript-1.4.2}/src/koskript/grammar.lark

@@ -3,6 +3,9 @@
 NAME      : /(?!(?:if|elseif|else|while|for|foreach|fn|wrapper|return|local|const|true|false|null|and|or|not|in|break|continue|class|extends|new|static|public|private|this|super|constructor|import|as|error|throw|try|catch|finally|namespace)\b)[a-zA-Z_][a-zA-Z0-9_]*/
 NUMBER    : /[0-9]+(\.[0-9]+)?/
 STRING    : /"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'/
+// Higher priority than NAME so `b"..."` lexes as a single token instead
+// of NAME + STRING.
+BYTES.2   : /[bB]"(?:\\.|[^"\\])*"|[bB]'(?:\\.|[^'\\])*'/
 COMMENT   : /\/\/[^\n]*/
 // Matches a bare `return` at the end of a line (or before a comment/EOF).
 // Higher priority so it wins over the `return` keyword on equal length.
@@ -188,6 +191,7 @@ block        : "{" statement* "}"
 
 ?atom         : NUMBER
               | STRING
+              | BYTES
               | array
               | map
               | bool

{koskript-1.4.1 → koskript-1.4.2}/src/koskript/lang/astgen.py

@@ -1,5 +1,6 @@
 from lark import Transformer
 from .emtypes import *
+from .errors import Errors
 
 _ESCAPES = {
     "n": "\n",
@@ -11,6 +12,8 @@ _ESCAPES = {
     "'": "'",
 }
 
+_HEX_DIGITS = "0123456789abcdefABCDEF"
+
 
 def _unescape(raw: str) -> str:
     out = []
@@ -27,6 +30,37 @@ def _unescape(raw: str) -> str:
     return "".join(out)
 
 
+def _unescape_bytes(raw: str) -> bytes:
+    """Turn the body of a ``b"..."`` literal into bytes.
+
+    Supports the same escapes as strings plus ``\\xNN`` for arbitrary byte
+    values. Characters outside ASCII are encoded as UTF-8.
+    """
+    out = bytearray()
+    i = 0
+    length = len(raw)
+    while i < length:
+        ch = raw[i]
+        if ch == "\\" and i + 1 < length:
+            nxt = raw[i + 1]
+            if nxt == "x":
+                digits = raw[i + 2:i + 4]
+                if len(digits) != 2 \
+                        or digits[0] not in _HEX_DIGITS \
+                        or digits[1] not in _HEX_DIGITS:
+                    raise ValueError(
+                        f"invalid bytes escape '\\x{digits}', expected \\xNN")
+                out.append(int(digits, 16))
+                i += 4
+                continue
+            out.extend(_ESCAPES.get(nxt, nxt).encode("utf-8"))
+            i += 2
+        else:
+            out.extend(ch.encode("utf-8"))
+            i += 1
+    return bytes(out)
+
+
 def _qualified_name(node) -> str:
     """Display name for a simple or dotted reference node."""
     if type(node) is NameRef:
@@ -47,6 +81,17 @@ class KoskriptTransformer(Transformer):
 
     def NAME(self, token): return NameRef(name=str(token))
     def STRING(self, token): return StrLit(value=_unescape(str(token)[1:-1]))
+
+    def BYTES(self, token):
+        text = str(token)
+        try:
+            value = _unescape_bytes(text[2:-1])
+        except ValueError as exc:
+            raise Errors.SyntaxError(
+                f"Syntax error at line {token.line}, "
+                f"column {token.column}: {exc}") from None
+        return BytesLit(value=value)
+
     def bool_true(self, tree): return BoolLit(value=True)
     def bool_false(self, tree): return BoolLit(value=False)
     def null_lit(self, tree): return NullLit(value=None)