klongpy 0.6.9__py3-none-any.whl → 0.7.1__py3-none-any.whl

klongpy/parser.py

@@ -0,0 +1,328 @@
+"""
+KlongPy parser and lexer functions.
+
+This module contains all the parsing functions for the Klong language:
+- Lexeme reading (numbers, strings, symbols, operators)
+- List parsing
+- Conditional expression parsing
+- Comment handling
+"""
+import copy
+
+from .types import (
+    KGSym, KGChar, KGOp, KGCond, KGCall, KGLambda,
+    reserved_fn_symbol_map,
+    safe_eq, is_symbolic, is_adverb
+)
+
+
+# Character matching utilities
+
+def cmatch(t, i, c):
+    return i < len(t) and t[i] == c
+
+
+def cmatch2(t, i, a, b):
+    return cmatch(t, i, a) and cmatch(t, i+1, b)
+
+
+def cpeek(t, i):
+    return t[i] if i < len(t) else None
+
+
+def cpeek2(t, i):
+    return t[i:i+2] if i < (len(t)-1) else None
+
+
+class UnexpectedChar(Exception):
+    def __init__(self, t, i, c):
+        super().__init__(f"t: {t[i-10:i+10]} pos: {i} char: {c}")
+
+
+class UnexpectedEOF(Exception):
+    def __init__(self, t, i):
+        self.t = t
+        self.i = i
+        super().__init__(f"t: {t[i-10:]} pos: {i}")
+
+
+def cexpect(t, i, c):
+    if cmatch(t, i, c):
+        return i + 1
+    raise UnexpectedChar(t, i, c)
+
+
+def cexpect2(t, i, a, b):
+    if cmatch(t, i, a) and cmatch(t, i+1, b):
+        return i + 2
+    raise UnexpectedChar(t, i, b)
+
+
+# Comment handling
+
+def read_shifted_comment(t, i=0):
+    while i < len(t):
+        c = t[i]
+        if c == '"':
+            i += 1
+            if not cmatch(t, i, '"'):
+                break
+        i += 1
+    return i
+
+
+def read_sys_comment(t, i, a):
+    """
+        .comment(x)                                            [Comment]
+
+        Read and discard lines until the current line starts with the
+        string specified in "x". Also discard the line containing the
+        end-of-comment marker and return "x".
+
+        Example: .comment("end-of-comment")
+                 this will be ignored
+                 this, too: *%(*^#)&(#
+                 end-of-comment
+
+        NOTE: this is handled in the parsing phase and is not a runtime function
+    """
+    try:
+        j = t[i:].index(a)
+        while t[i+j+1:].startswith(a):
+            j += 1
+        return i + j + len(a)
+    except ValueError:
+        return RuntimeError("end of comment not found")
+
+
+# Whitespace handling
+
+def skip_space(t, i=0, ignore_newline=False):
+    """
+        NOTE: a newline character translates to a semicolon in Klong,
+        except in functions, dictionaries, conditional expressions,
+        and lists. So
+    """
+    while i < len(t) and (t[i].isspace() and (ignore_newline or t[i] != '\n')):
+        i += 1
+    return i
+
+
+def skip(t, i=0, ignore_newline=False):
+    i = skip_space(t, i, ignore_newline=ignore_newline)
+    if cmatch2(t, i, ':', '"'):
+        i = read_shifted_comment(t, i+2)
+        i = skip(t, i)
+    return i
+
+
+# Lexeme readers
+
+def read_num(t, i=0):
+    p = i
+    use_float = False
+    if t[i] == '-':
+        i += 1
+    while i < len(t):
+        if t[i] == '.':
+            use_float = True
+        elif t[i] == 'e':
+            use_float = True
+            if cmatch(t, i+1, '-'):
+                i += 2
+        elif not t[i].isnumeric():
+            break
+        i += 1
+    return i, float(t[p:i]) if use_float else int(t[p:i])
+
+
+def read_char(t, i):
+    i = cexpect2(t, i, '0', 'c')
+    if i >= len(t):
+        raise UnexpectedEOF(t, i)
+    return i+1, KGChar(t[i])
+
+
+def read_sym(t, i=0, module=None):
+    p = i
+    while i < len(t) and is_symbolic(t[i]):
+        i += 1
+    x = t[p:i]
+    return i, reserved_fn_symbol_map.get(x) or KGSym(x if x.startswith('.') or module is None else f"{x}`{module}")
+
+
+def read_op(t, i=0):
+    if cmatch2(t, i, '\\', '~') or cmatch2(t, i, '\\', '*'):
+        return i+2, KGOp(t[i:i+2], arity=0)
+    return i+1, KGOp(t[i:i+1], arity=0)
+
+
+def read_string(t, i=0):
+    """
+    ".*"                                                    [String]
+
+    A string is (almost) any sequence of characters enclosed by
+    double quote characters. To include a double quote character in
+    a string, it has to be duplicated, so the above regex is not
+    entirely correct. A comment is a shifted string (see below).
+    Examples: ""
+              "hello, world"
+              "say ""hello""!"
+
+    Note: this comforms to the KG read_string impl.
+          perf tests show that the final join is fast for short strings
+    """
+    r = []
+    while i < len(t):
+        c = t[i]
+        if c == '"':
+            i += 1
+            if not cmatch(t, i, '"'):
+                break
+        r.append(c)
+        i += 1
+    return i, "".join(r)
+
+
+# Dictionary helper
+
+def list_to_dict(a):
+    return {x[0]:x[1] for x in a}
+
+
+# Lambda for copy operations (used in dict parsing)
+copy_lambda = KGLambda(lambda x: copy.deepcopy(x))
+
+
+def read_list(t, delim, i=0, module=None):
+    """
+    Parse a list from string t starting at position i.
+    Returns a Python list (caller converts to array if needed).
+
+    L := '[' (C|L)* ']'
+    """
+    arr = []
+    i = skip(t, i, ignore_newline=True)
+    while not cmatch(t, i, delim) and i < len(t):
+        i, q = kg_read(t, i, read_neg=True, ignore_newline=True, module=module)
+        if q is None:
+            break
+        if safe_eq(q, '['):
+            i, q = read_list(t, ']', i=i, module=module)
+        arr.append(q)
+        i = skip(t, i, ignore_newline=True)
+    if cmatch(t, i, delim):
+        i += 1
+    return i, arr
+
+
+def kg_read(t, i, read_neg=False, ignore_newline=False, module=None):
+    """
+    Read a Klong lexeme from string t starting at position i.
+
+    C := I | H | R | S | V | Y
+    """
+    i = skip(t, i, ignore_newline=ignore_newline)
+    if i >= len(t):
+        return i, None
+    a = t[i]
+    if a == '\n':
+        a = ';'
+    if a in [';', '(', ')', '{', '}', ']']:
+        return i+1, a
+    elif cmatch2(t, i, '0', 'c'):
+        return read_char(t, i)
+    elif a.isnumeric() or (read_neg and (a == '-' and (i+1) < len(t) and t[i+1].isnumeric())):
+        return read_num(t, i)
+    elif a == '"':
+        return read_string(t, i+1)
+    elif a == ':' and (i+1 < len(t)):
+        aa = t[i+1]
+        if aa.isalpha() or aa == '.':
+            return read_sym(t, i=i+1, module=module)
+        elif aa.isnumeric() or aa == '"':
+            return kg_read(t, i+1, ignore_newline=ignore_newline, module=module)
+        elif aa == '{':
+            i, d = read_list(t, '}', i=i+2, module=module)
+            d = list_to_dict(d)
+            return i, KGCall(copy_lambda, args=d, arity=0)
+        elif aa == '[':
+            return i+2, ':['
+        elif aa == '|':
+            return i+2, ':|'
+        return i+2, KGOp(f":{aa}", arity=0)
+    elif safe_eq(a, '['):
+        return read_list(t, ']', i=i+1, module=module)
+    elif is_symbolic(a):
+        return read_sym(t, i, module=module)
+    return read_op(t, i)
+
+
+def kg_read_array(t, i, backend, **kwargs):
+    """
+    Read a value and convert lists to arrays using the provided backend.
+
+    This is a helper that wraps kg_read and handles list-to-array conversion,
+    centralizing the pattern used by the interpreter and eval_sys.
+
+    Parameters
+    ----------
+    t : str
+        The string to read from.
+    i : int
+        Starting position in the string.
+    backend : BackendProvider
+        The backend to use for array conversion.
+    **kwargs
+        Additional arguments passed to kg_read (read_neg, ignore_newline, module).
+
+    Returns
+    -------
+    tuple
+        (new_position, value) where value is converted to an array if it was a list.
+    """
+    i, a = kg_read(t, i, **kwargs)
+    if isinstance(a, list):
+        a = backend.kg_asarray(a)
+    return i, a
+
+
+def read_cond(klong, t, i=0):
+    """
+        # A conditional expression has two forms: :[e1;e2;e3] means "if
+        # e1 is true, evaluate to e2, else evaluate to e3".
+        # :[e1;e2:|e3;e4;e5] is short for :[e1;e2:[e3;e4;e5]], i.e. the
+        # ":|" acts as an "else-if" operator. There may be any number of
+        # ":|" operators in a conditional.
+
+        c := ':[' ( e ';' e ':|' )* e ';' e ';' e ']'
+    """
+    r = []
+    i, n = klong._expr(t, i, ignore_newline=True)
+    r.append(n)
+    i = cexpect(t, i, ';')
+    i, n = klong._expr(t, i, ignore_newline=True)
+    r.append(n)
+    i = skip(t, i, ignore_newline=True)
+    if cmatch2(t, i, ':', '|'):
+        i, n = read_cond(klong, t, i+2)
+        r.append(n)
+    else:
+        i = cexpect(t, i, ';')
+        i, n = klong._expr(t, i, ignore_newline=True)
+        r.append(n)
+        i = skip(t, i, ignore_newline=True)
+        i = cexpect(t, i, ']')
+    return i, KGCond(r)
+
+
+# Adverb peeking
+
+def peek_adverb(t, i=0):
+    x = cpeek2(t, i)
+    if is_adverb(x):
+        return i+2, x
+    x = cpeek(t, i)
+    if is_adverb(x):
+        return i+1, x
+    return i, None

klongpy/repl.py

@@ -3,12 +3,30 @@ import threading
 import time
 import os
 import importlib.resources
+from typing import Optional
 
 from . import KlongInterpreter
 from .utils import CallbackEvent
 
 
-def start_loop(loop: asyncio.AbstractEventLoop, stop_event: asyncio.Event) -> None:
+class LoopStopper:
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        self._loop = loop
+        self._future = loop.create_future()
+
+    def set(self) -> None:
+        if self._future.done():
+            return
+        if self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._future.set_result, None)
+        else:
+            self._future.set_result(None)
+
+    async def wait(self) -> None:
+        await self._future
+
+
+def start_loop(loop: asyncio.AbstractEventLoop, stop_event: LoopStopper) -> None:
     asyncio.set_event_loop(loop)
     loop.run_until_complete(stop_event.wait())
 
@@ -18,13 +36,13 @@ def setup_async_loop(debug: bool = False, slow_callback_duration: float = 86400.
     loop.slow_callback_duration = slow_callback_duration
     if debug:
         loop.set_debug(True)
-    stop_event = asyncio.Event()
+    stop_event = LoopStopper(loop)
     thread = threading.Thread(target=start_loop, args=(loop, stop_event), daemon=True)
     thread.start()
     return loop, thread, stop_event
 
 
-def cleanup_async_loop(loop: asyncio.AbstractEventLoop, loop_thread: threading.Thread, stop_event: asyncio.Event, debug: bool = False, name: str | None = None) -> None:
+def cleanup_async_loop(loop: asyncio.AbstractEventLoop, loop_thread: threading.Thread, stop_event: LoopStopper, debug: bool = False, name: Optional[str] = None) -> None:
     if loop.is_closed():
         return
 
@@ -54,13 +72,13 @@ def append_pkg_resource_path_KLONGPATH() -> None:
         os.environ['KLONGPATH'] = klongpath
 
 
-def create_repl(debug: bool = False):
+def create_repl(debug: bool = False, backend: Optional[str] = None, device: Optional[str] = None):
     io_loop, io_thread, io_stop = setup_async_loop(debug=debug)
     klong_loop, klong_thread, klong_stop = setup_async_loop(debug=debug)
 
     append_pkg_resource_path_KLONGPATH()
 
-    klong = KlongInterpreter()
+    klong = KlongInterpreter(backend=backend, device=device)
     shutdown_event = CallbackEvent()
     klong['.system'] = {'ioloop': io_loop, 'klongloop': klong_loop, 'closeEvent': shutdown_event}
 

klongpy/sys_fn.py

@@ -12,10 +12,24 @@ from inspect import Parameter
 import numpy
 
 from .core import (KGChannel, KGChannelDir, KGLambda, KGSym, KlongException,
-                   is_dict, is_empty, is_list, kg_asarray, kg_read, kg_write, np,
+                   bknp, is_dict, is_empty, is_list, kg_read_array, kg_write,
                    reserved_fn_args, reserved_fn_symbol_map, safe_eq, safe_inspect)
 
 
+def _to_display_value(x, backend):
+    """Convert backend tensors to numpy for cleaner display."""
+    # Convert backend arrays (tensors) to numpy
+    if backend.is_backend_array(x):
+        return backend.to_numpy(x)
+    # Handle numpy arrays with tensors inside (object arrays)
+    if isinstance(x, numpy.ndarray) and x.dtype == object:
+        return numpy.array([_to_display_value(item, backend) for item in x], dtype=object)
+    # Handle lists with tensors
+    if isinstance(x, list):
+        return [_to_display_value(item, backend) for item in x]
+    return x
+
+
 def eval_sys_append_channel(x):
     """
 
@@ -47,10 +61,25 @@ def eval_sys_display(klong, x):
 
         .d(x)                                                  [Display]
 
-        See [Write].
+        Display the object "x". Backend arrays are converted to numpy for cleaner output.
+        Use .bkd() for raw backend-specific display.
 
     """
-    r = kg_write(x, display=True)
+    x = _to_display_value(x, klong._backend)
+    r = kg_write(x, klong._backend, display=True)
+    klong['.sys.cout'].raw.write(r)
+    return r
+
+
+def eval_sys_backend_display(klong, x):
+    """
+
+        .bkd(x)                                        [Backend-Display]
+
+        Display the object "x" in raw backend format (tensors shown as-is).
+
+    """
+    r = kg_write(x, klong._backend, display=True)
     klong['.sys.cout'].raw.write(r)
     return r
 
@@ -272,10 +301,26 @@ def eval_sys_print(klong, x):
         .p(x)                                                    [Print]
 
         Pretty-print the object "x" (like Display) and then print a
-        newline sequence. .p("") will just print a newline.
+        newline sequence. Backend arrays are converted to numpy for cleaner output.
+        Use .bkp() for raw backend-specific print.
 
     """
-    o = kg_write(x, display=True)
+    x = _to_display_value(x, klong._backend)
+    o = kg_write(x, klong._backend, display=True)
+    klong['.sys.cout'].raw.write(o+"\n")
+    return o
+
+
+def eval_sys_backend_print(klong, x):
+    """
+
+        .bkp(x)                                          [Backend-Print]
+
+        Pretty-print the object "x" in raw backend format (tensors shown as-is)
+        and then print a newline sequence.
+
+    """
+    o = kg_write(x, klong._backend, display=True)
     klong['.sys.cout'].raw.write(o+"\n")
     return o
 
@@ -341,20 +386,30 @@ def _handle_import(item):
             if n_args <= len(reserved_fn_args):
                 item = KGLambda(item, args=reserved_fn_args[:n_args])
         else:
-            args = safe_inspect(item, follow_wrapped=True)
-            if 'args' in args:
+            sig_args = safe_inspect(item, follow_wrapped=True)
+            if 'args' in sig_args:
                 item = KGLambda(item, args=None, wildcard=True)
                 n_args = 3
             else:
-                args = [k for k,v in args.items() if (v.kind == Parameter.POSITIONAL_OR_KEYWORD and v.default == Parameter.empty) or (v.kind == Parameter.POSITIONAL_ONLY)]
-                n_args = len(args)
-                # if there are kwargs, then .pyc() must be used to call this function to override them
-                if 'klong' in args:
-                    n_args -= 1
-                    assert n_args <= len(reserved_fn_args)
-                    item = KGLambda(item, args=reserved_fn_args[:n_args], provide_klong=True)
-                elif n_args <= len(reserved_fn_args):
-                    item = KGLambda(item, args=reserved_fn_args[:n_args])
+                # Get required args (no default)
+                required_args = [k for k,v in sig_args.items() if (v.kind == Parameter.POSITIONAL_OR_KEYWORD and v.default == Parameter.empty) or (v.kind == Parameter.POSITIONAL_ONLY)]
+                # Get optional args (have default)
+                optional_args = [k for k,v in sig_args.items() if v.kind == Parameter.POSITIONAL_OR_KEYWORD and v.default != Parameter.empty]
+                # Use required args count, but if there are optional args and no required args,
+                # use wildcard mode so the function can accept 0-3 args
+                if not required_args and optional_args:
+                    item = KGLambda(item, args=None, wildcard=True)
+                    n_args = 3
+                else:
+                    args = required_args
+                    n_args = len(args)
+                    # if there are kwargs, then .pyc() must be used to call this function to override them
+                    if 'klong' in args:
+                        n_args -= 1
+                        assert n_args <= len(reserved_fn_args)
+                        item = KGLambda(item, args=reserved_fn_args[:n_args], provide_klong=True)
+                    elif n_args <= len(reserved_fn_args):
+                        item = KGLambda(item, args=reserved_fn_args[:n_args])
     except Exception:
         if hasattr(item, "__class__") and hasattr(item.__class__, '__module__') and item.__class__.__module__ == "builtins":
             # LOOK AWAY. You didn't see this.
@@ -415,6 +470,17 @@ def _import_module(klong, x, from_set=None):
                 except Exception as e:
                     # TODO: this should be logged
                     print(f"failed to import function: {name}", e)
+
+            # For from_set imports, also check for lazy-loaded attributes not in __dict__
+            # (e.g., numpy.random in numpy 2.x)
+            if from_set is not None:
+                for name in from_set:
+                    if name not in export_items and hasattr(module, name):
+                        try:
+                            item = getattr(module, name)
+                            klong[name] = _handle_import(item)
+                        except Exception as e:
+                            print(f"failed to import function: {name}", e)
         finally:
             klong._context.push(ctx)
 
@@ -578,6 +644,49 @@ def eval_sys_python_from(klong, x, y):
     return _import_module(klong, x, from_set=set(y))
 
 
+def eval_sys_backend_fn(klong, x):
+    """
+
+        .bkf(x)                                           [Backend-Function]
+
+        Import functions from the current backend's array module.
+        This is similar to .pyf() but uses backend-aware functions that
+        work with both numpy and torch backends.
+
+        When using the torch backend, these functions preserve gradient
+        tracking for autograd.
+
+        Example:
+
+            .bkf("exp")
+            exp(1.0) --> 2.718...
+
+            .bkf(["exp";"sin";"cos"])
+            sin(1.0) --> 0.841...
+
+        Common functions available: exp, sin, cos, tan, tanh, sqrt, abs,
+        log, log10, floor, ceil, round
+
+    """
+    if isinstance(x, str):
+        x = [x]
+    if not (is_list(x) and all(map(lambda p: isinstance(p, str), x))):
+        raise RuntimeError("function name(s) must be a string or list of strings")
+
+    backend = klong._backend
+    ctx = klong._context.pop()
+    try:
+        for fn_name in x:
+            if hasattr(backend.np, fn_name):
+                fn = getattr(backend.np, fn_name)
+                klong[fn_name] = _handle_import(fn)
+            else:
+                raise RuntimeError(f"Backend does not have function: {fn_name}")
+    finally:
+        klong._context.push(ctx)
+    return None
+
+
 def eval_sys_random_number():
     """
 
@@ -586,7 +695,7 @@ def eval_sys_random_number():
         Return a random number x, such that 0 <= x < 1.
 
     """
-    return np.random.random()
+    return bknp.random.random()
 
 
 def eval_sys_read(klong):
@@ -607,7 +716,7 @@ def eval_sys_read(klong):
         f.at_eof = True
         return None
     else:
-        i,a = kg_read(r,i=0,module=klong.current_module())
+        i,a = kg_read_array(r, 0, klong._backend, module=klong.current_module())
         f.raw.seek(k+i,0)
         return a
 
@@ -642,7 +751,7 @@ def eval_sys_read_lines(klong):
     f = klong['.sys.cin']
     r = f.raw.readlines()
     f.at_eof = True
-    return kg_asarray(r)
+    return klong._backend.kg_asarray(r)
 
 
 def eval_sys_read_string(klong, x):
@@ -656,7 +765,8 @@ def eval_sys_read_string(klong, x):
         forms.
 
     """
-    return kg_read(x, i=0, module=klong.current_module(), read_neg=True)[1]
+    _, a = kg_read_array(x, 0, klong._backend, module=klong.current_module(), read_neg=True)
+    return a
 
 
 def eval_sys_system(x):
@@ -722,7 +832,7 @@ def eval_sys_write(klong, x):
         sequence. Use .p (Print) to do so.
 
     """
-    r = kg_write(x)
+    r = kg_write(x, klong._backend)
     klong['.sys.cout'].raw.write(r)
     return x
 
@@ -742,6 +852,45 @@ def eval_sys_exit(x):
     sys.exit(1)
 
 
+def eval_sys_strict(klong):
+    """
+
+        .strict()                                                 [Strict]
+
+        Enable strict mode for variable assignment (level 1).
+        In strict mode, functions cannot create new global variables
+        unless they are explicitly declared with the global: prefix.
+
+        Example:
+            .strict()
+            counter::0
+            increment::{counter::counter+1}  :" ERROR"
+            increment::{[global:counter];counter::counter+1}  :" OK"
+
+    """
+    klong._context._strict_mode = 1
+    return 1
+
+
+def eval_sys_unsafe(klong):
+    """
+
+        .unsafe()                                                 [Unsafe]
+
+        Disable strict mode for variable assignment (level 0).
+        In unsafe mode, functions can create new global variables
+        without declaration (legacy behavior).
+
+        Example:
+            .unsafe()
+            f::{newvar::42}  :" OK - creates global newvar"
+            f()
+
+    """
+    klong._context._strict_mode = 0
+    return 0
+
+
 def create_system_functions():
     def _get_name(s):
         i = s.index('.')