mplang-nightly 0.1.dev143__py3-none-any.whl → 0.1.dev144__py3-none-any.whl

mplang/backend/__init__.py

@@ -11,10 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-"""
-Backend module for mplang.
-
-This module contains handlers that execute serialized functions on individual
-parties in a multi-party computation system.
-"""

mplang/backend/base.py

@@ -12,12 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Flat backend kernel registry & per-participant runtime.
+"""Backend kernel registry & per-participant runtime (explicit op->kernel binding).
 
-Design revision:
-- Global, stateless kernel function catalog (fn_type -> callable).
-- BackendRuntime: per-rank state & cache; executes kernels.
-- Legacy global helpers removed after full migration to explicit runtimes.
+This version decouples *kernel implementation registration* from *operation binding*.
+
+Concepts:
+    * kernel_id: unique identifier of a concrete backend implementation.
+    * op_type: semantic operation name carried by ``PFunction.fn_type``.
+    * bind_op(op_type, kernel_id): performed by higher layer (see ``backend.context``)
+        to select which implementation handles an op. Runtime dispatch is now a 2-step:
+        pfunc.fn_type -> active kernel_id -> KernelSpec.fn
+
+The previous implicit "import == register+bind" coupling is removed. Kernel
+modules only call ``@kernel_def(kernel_id)``. Default bindings are established
+centrally (lazy) the first time a runtime executes a kernel.
 """
 
 from __future__ import annotations
@@ -27,22 +35,17 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from typing import Any
 
-from mplang.core.dtype import UINT8, DType
-from mplang.core.pfunc import PFunction
-from mplang.core.table import TableLike, TableType
-from mplang.core.tensor import TensorLike, TensorType
-
 __all__ = [
-    "BackendRuntime",
     "KernelContext",
-    "create_runtime",
+    "KernelSpec",
+    "bind_op",
     "cur_kctx",
-    "kernel_def",
-    "list_registered_kernels",
+    "get_kernel_for_op",
+    "list_kernels",
+    "list_ops",
+    "unbind_op",
 ]
 
-# ---------------- Context ----------------
-
 
 @dataclass
 class KernelContext:
@@ -99,189 +102,74 @@ def cur_kctx() -> KernelContext:
 
 # ---------------- Registry ----------------
 
-# Canonical kernel callable signature (new style): (pfunc, *args) -> Any | sequence
-# - No **kwargs (explicitly disallowed)
-# - Return normalization handled by BackendRuntime.run_kernel
+# Kernel callable signature: (pfunc, *args) -> Any | sequence (no **kwargs)
 KernelFn = Callable[..., Any]
 
-_KERNELS: dict[str, KernelFn] = {}
 
+@dataclass
+class KernelSpec:
+    kernel_id: str
+    fn: KernelFn
+    meta: dict[str, Any]
 
-def _validate_table_arg(
-    fn_type: str, arg_index: int, spec: TableType, value: Any
-) -> None:
-    if not isinstance(value, TableLike):
-        raise TypeError(
-            f"kernel {fn_type} input[{arg_index}] expects TableLike, got {type(value).__name__}"
-        )
-    if len(value.columns) != len(spec.columns):
-        raise ValueError(
-            f"kernel {fn_type} input[{arg_index}] column count mismatch: got {len(value.columns)}, expected {len(spec.columns)}"
-        )
 
+# All registered kernel implementations: kernel_id -> spec
+_KERNELS: dict[str, KernelSpec] = {}
+
+# Active op bindings: op_type -> kernel_id
+_BINDINGS: dict[str, str] = {}
 
-def _validate_tensor_arg(
-    fn_type: str, arg_index: int, spec: TensorType, value: Any
-) -> None:
-    # Backend-only handle sentinel (e.g., PHE keys) bypasses all structural checks
-    if tuple(spec.shape) == (-1, 0) and spec.dtype == UINT8:
-        return
 
-    if isinstance(value, (int, float, bool, complex)):
-        val_shape: tuple[Any, ...] = ()
-        duck_dtype: Any = type(value)
-    else:
-        if not isinstance(value, TensorLike):
-            raise TypeError(
-                f"kernel {fn_type} input[{arg_index}] expects TensorLike, got {type(value).__name__}"
-            )
-        val_shape = getattr(value, "shape", ())
-        duck_dtype = getattr(value, "dtype", None)
-
-    if len(spec.shape) != len(val_shape):
-        raise ValueError(
-            f"kernel {fn_type} input[{arg_index}] rank mismatch: got {val_shape}, expected {spec.shape}"
-        )
-
-    for dim_idx, (spec_dim, val_dim) in enumerate(
-        zip(spec.shape, val_shape, strict=True)
-    ):
-        if spec_dim >= 0 and spec_dim != val_dim:
-            raise ValueError(
-                f"kernel {fn_type} input[{arg_index}] shape mismatch at dim {dim_idx}: got {val_dim}, expected {spec_dim}"
-            )
-
-    try:
-        val_dtype = DType.from_any(duck_dtype)
-    except (ValueError, TypeError):  # pragma: no cover
-        raise TypeError(
-            f"kernel {fn_type} input[{arg_index}] has unsupported dtype object {duck_dtype!r}"
-        ) from None
-    if val_dtype != spec.dtype:
-        raise ValueError(
-            f"kernel {fn_type} input[{arg_index}] dtype mismatch: got {val_dtype}, expected {spec.dtype}"
-        )
-
-
-def kernel_def(fn_type: str) -> Callable[[KernelFn], KernelFn]:
-    """Decorator to register a backend kernel (new signature).
-
-    Expected Python signature form:
-
-        @kernel_def("namespace.op")
-        def _op(pfunc: PFunction, *args): ...
-
-    Rules:
-      * First parameter MUST be the PFunction object.
-      * Positional arguments correspond 1:1 to pfunc.ins_info order.
-      * **kwargs are NOT supported (will raise at call site if used).
-      * Return value forms accepted (n = len(pfunc.outs_info)):
-          - n == 0: return None / () / []
-          - n == 1: return scalar/object OR (value,) / [value]
-          - n > 1 : return tuple/list of length n
-        Anything else raises a ValueError.
+def kernel_def(kernel_id: str, /, **meta: Any) -> Callable[[KernelFn], KernelFn]:
+    """Decorator to register a concrete kernel implementation.
+
+    This ONLY registers the implementation (kernel_id -> fn). It does NOT bind
+    any op. Higher layer must call ``bind_op(op_type, kernel_id)`` explicitly.
     """
 
     def _decorator(fn: KernelFn) -> KernelFn:
-        if fn_type in _KERNELS:
-            raise ValueError(f"duplicate backend kernel fn_type={fn_type}")
-        _KERNELS[fn_type] = fn
+        if kernel_id in _KERNELS:
+            raise ValueError(f"duplicate kernel_id={kernel_id}")
+        _KERNELS[kernel_id] = KernelSpec(kernel_id=kernel_id, fn=fn, meta=dict(meta))
         return fn
 
     return _decorator
 
 
-def list_registered_kernels() -> list[str]:  # public API unchanged
-    return sorted(_KERNELS.keys())
+def bind_op(op_type: str, kernel_id: str, *, force: bool = True) -> None:
+    """Bind an op_type to a registered kernel implementation.
 
+    Args:
+        op_type: Semantic operation name.
+        kernel_id: Previously registered kernel identifier.
+        force: If False and op_type already bound, keep existing binding.
+               If True (default), overwrite.
+    """
+    if kernel_id not in _KERNELS:
+        raise KeyError(f"kernel_id {kernel_id} not registered")
+    if not force and op_type in _BINDINGS:
+        return
+    _BINDINGS[op_type] = kernel_id
 
-class BackendRuntime:
-    """Per-rank backend execution environment.
 
-    Holds mutable backend state (namespaced pockets) and a cache. Stateless
-    kernel implementations look up their state through cur_kctx().
-    """
+def unbind_op(op_type: str) -> None:
+    _BINDINGS.pop(op_type, None)
+
+
+def get_kernel_for_op(op_type: str) -> KernelSpec:
+    kid = _BINDINGS.get(op_type)
+    if kid is None:
+        # Tests expect NotImplementedError for unsupported operations
+        raise NotImplementedError(f"no backend kernel registered for op {op_type}")
+    spec = _KERNELS.get(kid)
+    if spec is None:  # inconsistent state
+        raise RuntimeError(f"active kernel_id {kid} missing spec")
+    return spec
+
+
+def list_kernels() -> list[str]:
+    return sorted(_KERNELS.keys())
+
 
-    def __init__(self, rank: int, world_size: int):
-        self.rank = rank
-        self.world_size = world_size
-        self.state: dict[str, dict[str, Any]] = {}
-        self.cache: dict[str, Any] = {}
-
-    # Main entry
-    def run_kernel(self, pfunc: PFunction, arg_list: list[Any]) -> list[Any]:
-        fn_type = pfunc.fn_type
-        fn = _KERNELS.get(fn_type)
-        if fn is None:
-            raise NotImplementedError(f"no backend kernel registered for {fn_type}")
-
-        # Strict positional arg count validation (no kernel-managed arity bypass)
-        if len(arg_list) != len(pfunc.ins_info):
-            raise ValueError(
-                f"kernel {fn_type} arg count mismatch: got {len(arg_list)}, expect {len(pfunc.ins_info)}"
-            )
-
-        for idx, (spec, val) in enumerate(zip(pfunc.ins_info, arg_list, strict=True)):
-            if isinstance(spec, TableType):
-                _validate_table_arg(fn_type, idx, spec, val)
-                continue
-
-            if isinstance(spec, TensorType):
-                _validate_tensor_arg(fn_type, idx, spec, val)
-                continue
-
-            # Unknown spec type: silently skip validation (legacy behavior)
-            continue
-
-        kctx = KernelContext(
-            rank=self.rank,
-            world_size=self.world_size,
-            state=self.state,
-            cache=self.cache,
-        )
-        token = _CTX_VAR.set(kctx)
-        try:
-            raw = fn(pfunc, *arg_list)
-        finally:
-            _CTX_VAR.reset(token)
-
-        # Normalize return values
-        expected = len(pfunc.outs_info)
-        if expected == 0:
-            if raw in (None, (), []):
-                return []
-            raise ValueError(
-                f"kernel {fn_type} should return no values; got {type(raw).__name__}"
-            )
-
-        # If multi-output expected, raw must be sequence of right length
-        if expected == 1:
-            if isinstance(raw, (tuple, list)):
-                if len(raw) != 1:
-                    raise ValueError(
-                        f"kernel {fn_type} produced {len(raw)} outputs, expected 1"
-                    )
-                return [raw[0]]
-            # Single object
-            return [raw]
-
-        # expected > 1
-        if not isinstance(raw, (tuple, list)):
-            raise TypeError(
-                f"kernel {fn_type} must return sequence (len={expected}), got {type(raw).__name__}"
-            )
-        if len(raw) != expected:
-            raise ValueError(
-                f"kernel {fn_type} produced {len(raw)} outputs, expected {expected}"
-            )
-        return list(raw)
-
-    # Optional helper
-    def reset(self) -> None:  # pragma: no cover - simple
-        self.state.clear()
-        self.cache.clear()
-
-
-def create_runtime(rank: int, world_size: int) -> BackendRuntime:
-    """Factory for BackendRuntime (allows future policy injection)."""
-    return BackendRuntime(rank, world_size)
+def list_ops() -> list[str]:
+    return sorted(_BINDINGS.keys())

mplang/backend/context.py

@@ -0,0 +1,255 @@
+# Copyright 2025 Ant Group Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from mplang.backend import base
+from mplang.backend.base import KernelContext, bind_op, get_kernel_for_op
+from mplang.core.dtype import UINT8, DType
+from mplang.core.pfunc import PFunction
+from mplang.core.table import TableLike, TableType
+from mplang.core.tensor import TensorLike, TensorType
+
+# Default bindings
+# Import kernel implementation modules explicitly so their @kernel_def entries
+# register at import time. Keep imports grouped; alias with leading underscore
+# to silence unused variable warnings without F401 pragmas.
+_IMPL_IMPORTED = False
+
+
+def _ensure_impl_imported() -> None:
+    global _IMPL_IMPORTED
+    if _IMPL_IMPORTED:
+        return
+    from mplang.backend import builtin as _impl_builtin  # noqa: F401
+    from mplang.backend import crypto as _impl_crypto  # noqa: F401
+    from mplang.backend import phe as _impl_phe  # noqa: F401
+    from mplang.backend import spu as _impl_spu  # noqa: F401
+    from mplang.backend import sql_duckdb as _impl_sql_duckdb  # noqa: F401
+    from mplang.backend import stablehlo as _impl_stablehlo  # noqa: F401
+    from mplang.backend import tee as _impl_tee  # noqa: F401
+
+    _IMPL_IMPORTED = True
+
+
+# imports consolidated above
+
+_DEFAULT_BINDINGS: dict[str, str] = {
+    # builtin
+    "builtin.identity": "builtin.identity",
+    "builtin.read": "builtin.read",
+    "builtin.write": "builtin.write",
+    "builtin.constant": "builtin.constant",
+    "builtin.rank": "builtin.rank",
+    "builtin.prand": "builtin.prand",
+    "builtin.table_to_tensor": "builtin.table_to_tensor",
+    "builtin.tensor_to_table": "builtin.tensor_to_table",
+    "builtin.debug_print": "builtin.debug_print",
+    "builtin.pack": "builtin.pack",
+    "builtin.unpack": "builtin.unpack",
+    # crypto
+    "crypto.keygen": "crypto.keygen",
+    "crypto.enc": "crypto.enc",
+    "crypto.dec": "crypto.dec",
+    "crypto.kem_keygen": "crypto.kem_keygen",
+    "crypto.kem_derive": "crypto.kem_derive",
+    "crypto.hkdf": "crypto.hkdf",
+    # phe
+    "phe.keygen": "phe.keygen",
+    "phe.encrypt": "phe.encrypt",
+    "phe.mul": "phe.mul",
+    "phe.add": "phe.add",
+    "phe.decrypt": "phe.decrypt",
+    "phe.dot": "phe.dot",
+    "phe.gather": "phe.gather",
+    "phe.scatter": "phe.scatter",
+    "phe.concat": "phe.concat",
+    "phe.reshape": "phe.reshape",
+    "phe.transpose": "phe.transpose",
+    # spu
+    "spu.seed_env": "spu.seed_env",
+    "spu.makeshares": "spu.makeshares",
+    "spu.reconstruct": "spu.reconstruct",
+    "spu.run_pphlo": "spu.run_pphlo",
+    # stablehlo
+    "mlir.stablehlo": "mlir.stablehlo",
+    # sql
+    # generic SQL op; backend-specific kernel id for duckdb
+    "sql.run": "duckdb.run_sql",
+    # tee
+    "tee.quote": "tee.quote",
+    "tee.attest": "tee.attest",
+}
+
+
+# --- RuntimeContext ---
+
+
+@dataclass
+class RuntimeContext:
+    rank: int
+    world_size: int
+    bindings: Mapping[str, str] | None = None  # optional overrides
+    state: dict[str, dict[str, Any]] = field(default_factory=dict)
+    cache: dict[str, Any] = field(default_factory=dict)
+    stats: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        _ensure_impl_imported()
+        if self.bindings is not None:
+            for op, kid in self.bindings.items():
+                bind_op(op, kid)
+        else:
+            for op, kid in _DEFAULT_BINDINGS.items():
+                bind_op(op, kid)
+        # Initialize stats pocket
+        self.stats.setdefault("op_calls", {})
+
+    def run_kernel(self, pfunc: PFunction, arg_list: list[Any]) -> list[Any]:
+        fn_type = pfunc.fn_type
+        spec = get_kernel_for_op(fn_type)
+        fn = spec.fn
+        if len(arg_list) != len(pfunc.ins_info):
+            raise ValueError(
+                f"kernel {fn_type} arg count mismatch: got {len(arg_list)}, expect {len(pfunc.ins_info)}"
+            )
+        for idx, (ins_spec, val) in enumerate(
+            zip(pfunc.ins_info, arg_list, strict=True)
+        ):
+            if isinstance(ins_spec, TableType):
+                _validate_table_arg(fn_type, idx, ins_spec, val)
+                continue
+            if isinstance(ins_spec, TensorType):
+                _validate_tensor_arg(fn_type, idx, ins_spec, val)
+                continue
+        # install kernel context
+        kctx = KernelContext(
+            rank=self.rank,
+            world_size=self.world_size,
+            state=self.state,
+            cache=self.cache,
+        )
+        token = base._CTX_VAR.set(kctx)  # type: ignore[attr-defined]
+        try:
+            raw = fn(pfunc, *arg_list)
+        finally:
+            base._CTX_VAR.reset(token)  # type: ignore[attr-defined]
+        # Stats (best effort)
+        try:
+            op_calls = self.stats.setdefault("op_calls", {})
+            op_calls[fn_type] = op_calls.get(fn_type, 0) + 1
+        except Exception:  # pragma: no cover - never raise due to stats
+            pass
+        expected = len(pfunc.outs_info)
+        if expected == 0:
+            if raw in (None, (), []):
+                return []
+            raise ValueError(
+                f"kernel {fn_type} should return no values; got {type(raw).__name__}"
+            )
+        if expected == 1:
+            if isinstance(raw, (tuple, list)):
+                if len(raw) != 1:
+                    raise ValueError(
+                        f"kernel {fn_type} produced {len(raw)} outputs, expected 1"
+                    )
+                return [raw[0]]
+            return [raw]
+        if not isinstance(raw, (tuple, list)):
+            raise TypeError(
+                f"kernel {fn_type} must return sequence (len={expected}), got {type(raw).__name__}"
+            )
+        if len(raw) != expected:
+            raise ValueError(
+                f"kernel {fn_type} produced {len(raw)} outputs, expected {expected}"
+            )
+        return list(raw)
+
+    def reset(self) -> None:
+        self.state.clear()
+        self.cache.clear()
+
+    # ---- explicit (re)binding API ----
+    def bind_op(self, op_type: str, kernel_id: str, *, force: bool = False) -> None:
+        """Bind an operation to a kernel at runtime.
+
+        force=False (default) preserves any existing binding to avoid accidental
+        silent overrides. Use ``rebind_op`` or ``force=True`` to intentionally
+        change a binding.
+        """
+        base.bind_op(op_type, kernel_id, force=force)
+
+    def rebind_op(self, op_type: str, kernel_id: str) -> None:
+        """Force rebind an operation to a different kernel (shorthand)."""
+        base.bind_op(op_type, kernel_id, force=True)
+
+
+def _validate_table_arg(
+    fn_type: str, arg_index: int, spec: TableType, value: Any
+) -> None:
+    if not isinstance(value, TableLike):
+        raise TypeError(
+            f"kernel {fn_type} input[{arg_index}] expects TableLike, got {type(value).__name__}"
+        )
+    if len(value.columns) != len(spec.columns):
+        raise ValueError(
+            f"kernel {fn_type} input[{arg_index}] column count mismatch: got {len(value.columns)}, expected {len(spec.columns)}"
+        )
+
+
+def _validate_tensor_arg(
+    fn_type: str, arg_index: int, spec: TensorType, value: Any
+) -> None:
+    # Backend-only handle sentinel (e.g., PHE keys) bypasses all structural checks
+    if tuple(spec.shape) == (-1, 0) and spec.dtype == UINT8:
+        return
+
+    if isinstance(value, (int, float, bool, complex)):
+        val_shape: tuple[Any, ...] = ()
+        duck_dtype: Any = type(value)
+    else:
+        if not isinstance(value, TensorLike):
+            raise TypeError(
+                f"kernel {fn_type} input[{arg_index}] expects TensorLike, got {type(value).__name__}"
+            )
+        val_shape = getattr(value, "shape", ())
+        duck_dtype = getattr(value, "dtype", None)
+
+    if len(spec.shape) != len(val_shape):
+        raise ValueError(
+            f"kernel {fn_type} input[{arg_index}] rank mismatch: got {val_shape}, expected {spec.shape}"
+        )
+
+    for dim_idx, (spec_dim, val_dim) in enumerate(
+        zip(spec.shape, val_shape, strict=True)
+    ):
+        if spec_dim >= 0 and spec_dim != val_dim:
+            raise ValueError(
+                f"kernel {fn_type} input[{arg_index}] shape mismatch at dim {dim_idx}: got {val_dim}, expected {spec_dim}"
+            )
+
+    try:
+        val_dtype = DType.from_any(duck_dtype)
+    except (ValueError, TypeError):  # pragma: no cover
+        raise TypeError(
+            f"kernel {fn_type} input[{arg_index}] has unsupported dtype object {duck_dtype!r}"
+        ) from None
+    if val_dtype != spec.dtype:
+        raise ValueError(
+            f"kernel {fn_type} input[{arg_index}] dtype mismatch: got {val_dtype}, expected {spec.dtype}"
+        )

mplang/backend/spu.py

@@ -186,16 +186,18 @@ def _spu_reconstruct(pfunc: PFunction, *args: Any) -> Any:
     return reconstructed
 
 
-@kernel_def("mlir.pphlo")
+@kernel_def("spu.run_pphlo")
 def _spu_run_mlir(pfunc: PFunction, *args: Any) -> Any:
-    """Execute compiled SPU function (mlir.pphlo) and return SpuValue outputs.
+    """Execute compiled SPU function (spu.run_pphlo) and return SpuValue outputs.
 
     Participation rule: a rank participates iff its entry in the stored
     link_ctx list is non-None. This allows us to allocate a world-sized list
     (indexed by global rank) and simply assign None for non-SPU parties.
     """
-    if pfunc.fn_type != "mlir.pphlo":
-        raise ValueError(f"Unsupported format: {pfunc.fn_type}. Expected 'mlir.pphlo'")
+    if pfunc.fn_type != "spu.run_pphlo":
+        raise ValueError(
+            f"Unsupported format: {pfunc.fn_type}. Expected 'spu.run_pphlo'"
+        )
 
     cfg, _ = _get_spu_config_and_world()
     pocket = _get_spu_pocket()

mplang/backend/sql_duckdb.py

@@ -20,7 +20,7 @@ from mplang.backend.base import kernel_def
 from mplang.core.pfunc import PFunction
 
 
-@kernel_def("sql[duckdb]")
+@kernel_def("duckdb.run_sql")
 def _duckdb_sql(pfunc: PFunction, *args: Any) -> Any:
     import duckdb
     import pandas as pd

mplang/core/expr/evaluator.py

@@ -27,7 +27,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import Any, Protocol
 
-from mplang.backend.base import BackendRuntime
+from mplang.backend.context import RuntimeContext
 from mplang.core.comm import ICommunicator
 from mplang.core.expr.ast import (
     AccessExpr,
@@ -56,7 +56,7 @@ class IEvaluator(Protocol):
     backend state via evaluator.runtime.run_kernel(...).
     """
 
-    runtime: BackendRuntime
+    runtime: RuntimeContext
 
     def evaluate(self, root: Expr, env: dict[str, Any] | None = None) -> list[Any]: ...
 
@@ -72,7 +72,7 @@ class EvalSemantic:
     rank: int
     env: dict[str, Any]
     comm: ICommunicator
-    runtime: BackendRuntime
+    runtime: RuntimeContext
 
     # ------------------------------ Shared helpers (semantics) ------------------------------
     def _should_run(self, rmask: Mask | None, args: list[Any]) -> bool:
@@ -205,7 +205,7 @@ class RecursiveEvaluator(EvalSemantic, ExprVisitor):
         rank: int,
         env: dict[str, Any],
         comm: ICommunicator,
-        runtime: BackendRuntime,
+        runtime: RuntimeContext,
     ) -> None:
         super().__init__(rank, env, comm, runtime)
         self._cache: dict[int, Any] = {}  # Cache based on expr id
@@ -380,7 +380,7 @@ class IterativeEvaluator(EvalSemantic):
         rank: int,
         env: dict[str, Any],
         comm: ICommunicator,
-        runtime: BackendRuntime,
+        runtime: RuntimeContext,
     ) -> None:
         super().__init__(rank, env, comm, runtime)
 
@@ -501,7 +501,7 @@ def create_evaluator(
     rank: int,
     env: dict[str, Any],
     comm: ICommunicator,
-    runtime: BackendRuntime,
+    runtime: RuntimeContext,
     kind: str | None = "iterative",
 ) -> IEvaluator:
     """Factory to create an evaluator engine.

mplang/frontend/base.py

@@ -129,7 +129,7 @@ class FeModule(ABC):
         - You need compilation/stateful behavior/dynamic routing, multiple PFunctions, or complex capture flows.
 
     Tips:
-    - Keep routing information in PFunction.fn_type (e.g., "builtin.read", "sql[duckdb]", "mlir.stablehlo").
+    - Keep routing information in PFunction.fn_type (e.g., "builtin.read", "sql.run", "mlir.stablehlo").
     - Avoid backend-specific logic in kernels; only validate and shape types.
     - Prefer keyword-only attributes in typed_op kernels for clarity (def op(x: MPObject, *, attr: int)).
     """

mplang/frontend/ibis_cc.py

@@ -57,8 +57,9 @@ def ibis2sql(
     outs_info = [_convert(expr.schema())]
 
     sql = ibis.to_sql(expr, dialect="duckdb")
+    # Emit generic sql.run op; runtime maps to backend-specific kernel.
     pfn = PFunction(
-        fn_type="sql[duckdb]",
+        fn_type="sql.run",
         fn_name=fn_name,
         fn_text=sql,
         ins_info=tuple(ins_info),

mplang/frontend/spu.py

@@ -94,9 +94,10 @@ def _compile_jax(
     *args: Any,
     **kwargs: Any,
 ) -> tuple[PFunction, list[MPObject], PyTreeDef]:
-    """Compile a JAX function into SPU pphlo MLIR representation."""
+    """Compile a JAX function into SPU pphlo MLIR and wrap as PFunction.
 
-    """Compile a JAX function into SPU pphlo MLIR representation."""
+    Resulting PFunction uses fn_type 'spu.run_pphlo'.
+    """
 
     def is_variable(arg: Any) -> bool:
         return isinstance(arg, MPObject)
@@ -132,7 +133,7 @@ def _compile_jax(
     executable_code = executable_code.decode("utf-8")
 
     pfunc = PFunction(
-        fn_type="mlir.pphlo",
+        fn_type="spu.run_pphlo",
         ins_info=tuple(TensorType.from_obj(x) for x in in_vars),
         outs_info=tuple(output_tensor_infos),
         fn_name=get_fn_name(fn),

mplang/runtime/resource.py

@@ -26,17 +26,7 @@ from urllib.parse import urlparse
 import cloudpickle as pickle
 import spu.libspu as libspu
 
-# Import backends (side-effect: kernel registration)
-from mplang.backend import (  # noqa: F401
-    builtin,
-    crypto,
-    phe,
-    spu,  # registers flat SPU kernels
-    sql_duckdb,
-    stablehlo,
-    tee,
-)
-from mplang.backend.base import BackendRuntime, create_runtime
+from mplang.backend.context import RuntimeContext
 from mplang.backend.spu import PFunction  # type: ignore
 from mplang.core.expr.ast import Expr
 from mplang.core.expr.evaluator import IEvaluator, create_evaluator
@@ -98,9 +88,6 @@ class Session:
 # Global session storage
 _sessions: dict[str, Session] = {}
 
-# Service-level global symbol table (process-local for this server)
-_global_symbols: dict[str, Symbol] = {}
-
 
 # Session Management
 def create_session(
@@ -147,6 +134,43 @@ def delete_session(name: str) -> bool:
     return False
 
 
+# Global symbol management (process-wide, not per-session)
+_global_symbols: dict[str, Symbol] = {}
+
+
+def create_global_symbol(name: str, mptype: dict[str, Any], data_b64: str) -> Symbol:
+    """Create or replace a global symbol.
+
+    Args:
+        name: Symbol identifier
+        mptype: Metadata dict (shape/dtype, etc.)
+        data_b64: Base64-encoded pickled data
+    """
+    try:
+        raw = base64.b64decode(data_b64)
+        data = pickle.loads(raw)
+    except Exception as e:  # pragma: no cover - defensive
+        raise InvalidRequestError(f"Failed to decode symbol payload: {e}") from e
+    sym = Symbol(name=name, mptype=mptype, data=data)
+    _global_symbols[name] = sym
+    return sym
+
+
+def get_global_symbol(name: str) -> Symbol:
+    sym = _global_symbols.get(name)
+    if sym is None:
+        raise ResourceNotFound(f"Global symbol '{name}' not found")
+    return sym
+
+
+def delete_global_symbol(name: str) -> bool:
+    return _global_symbols.pop(name, None) is not None
+
+
+def list_global_symbols() -> list[str]:  # pragma: no cover - trivial
+    return sorted(_global_symbols.keys())
+
+
 # Computation Management
 def create_computation(
     session_name: str, computation_name: str, expr: Expr
@@ -225,15 +249,7 @@ def execute_computation(
 
     # Build evaluator
     # Explicit per-rank backend runtime (deglobalized)
-    runtime = create_runtime(rank, session.communicator.world_size)
-    # Inject global symbol storage into backend runtime state so that
-    # symbols:// provider can access it during builtin.read/write.
-    if isinstance(runtime, BackendRuntime):
-        pocket = runtime.state.setdefault("resource.providers", {})
-        if "symbols" not in pocket:
-            pocket["symbols"] = _global_symbols
-        else:
-            raise RuntimeError("resource.providers.symbols already exists")
+    runtime = RuntimeContext(rank=rank, world_size=session.communicator.world_size)
     evaluator: IEvaluator = create_evaluator(
         rank=rank, env=bindings, comm=session.communicator, runtime=runtime
     )
@@ -285,45 +301,6 @@ def execute_computation(
             session.symbols[name] = Symbol(name=name, mptype={}, data=val)
 
 
-# Global symbol CRUD (service-level)
-def create_global_symbol(name: str, mptype: Any, data: str) -> Symbol:
-    """Create or update a global symbol in the service-level table.
-
-    WARNING: Uses Python pickle for arbitrary object deserialization. Deploy
-    only in trusted environments. Future work may replace this with a
-    restricted / structured serialization.
-
-    The `data` argument is a base64-encoded pickled Python object. Minimal
-    validation of `mptype` is performed for tensor metadata (shape/dtype)
-    when present to catch obvious mismatches.
-    """
-    try:
-        data_bytes = base64.b64decode(data)
-        obj = pickle.loads(data_bytes)
-    except Exception as e:
-        raise InvalidRequestError(f"Invalid global symbol data encoding: {e!s}") from e
-
-    sym = Symbol(name, mptype, obj)
-    _global_symbols[name] = sym
-    return sym
-
-
-def get_global_symbol(name: str) -> Symbol | None:
-    return _global_symbols.get(name)
-
-
-def list_global_symbols() -> list[str]:
-    return list(_global_symbols.keys())
-
-
-def delete_global_symbol(name: str) -> bool:
-    if name in _global_symbols:
-        del _global_symbols[name]
-        logging.info(f"Global symbol {name} deleted")
-        return True
-    return False
-
-
 # Symbol Management
 def create_symbol(session_name: str, name: str, mptype: Any, data: Any) -> Symbol:
     """Create a symbol in a session's symbol table.

mplang/runtime/simulation.py

@@ -24,17 +24,10 @@ from typing import Any, cast
 
 import spu.libspu as libspu
 
-# Import flat backends for kernel registration side-effects
-from mplang.backend import (
-    builtin,  # noqa: F401
-    crypto,  # noqa: F401
-    phe,  # noqa: F401
-    spu,  # noqa: F401  # ensure SPU kernels (spu.seed_env etc.) registered
-    sql_duckdb,  # noqa: F401
-    stablehlo,  # noqa: F401
-    tee,  # noqa: F401
-)
-from mplang.backend.base import create_runtime  # explicit per-rank backend runtime
+# New explicit binding model: we only need RuntimeContext which ensures
+# bindings via bind_all_ops() on creation; per-module side-effect imports
+# are no longer required here.
+from mplang.backend.context import RuntimeContext
 from mplang.core.cluster import ClusterSpec
 from mplang.core.comm import CollectiveMixin, CommunicatorBase
 from mplang.core.expr.ast import Expr
@@ -152,7 +145,7 @@ class Simulator(InterpContext):
 
         self._evaluators: list[IEvaluator] = []
         for rank in range(self.world_size()):
-            runtime = create_runtime(rank, self.world_size())
+            runtime = RuntimeContext(rank=rank, world_size=self.world_size())
             ev = create_evaluator(
                 rank,
                 {},  # the global environment for this rank
@@ -224,7 +217,7 @@ class Simulator(InterpContext):
         # Build per-rank evaluators with the per-party environment
         pts_evaluators: list[IEvaluator] = []
         for rank in range(self.world_size()):
-            runtime = create_runtime(rank, self.world_size())
+            runtime = RuntimeContext(rank=rank, world_size=self.world_size())
             ev = create_evaluator(
                 rank,
                 pts_env[rank],

{mplang_nightly-0.1.dev143.dist-info → mplang_nightly-0.1.dev144.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mplang-nightly
-Version: 0.1.dev143
+Version: 0.1.dev144
 Summary: Multi-Party Programming Language
 Author-email: SecretFlow Team <secretflow-contact@service.alipay.com>
 License:                                  Apache License

{mplang_nightly-0.1.dev143.dist-info → mplang_nightly-0.1.dev144.dist-info}/RECORD

@@ -3,13 +3,14 @@ mplang/api.py,sha256=ssmv0_CyZPFORhOUJ84Jo6NwRJSK7_Ono3n7ZjEg4sA,3058
 mplang/device.py,sha256=Iz_YFKkrbTFKtTQdGqkQZfc0NQH9dIxXP7-fUkIQOa4,12568
 mplang/analysis/__init__.py,sha256=CTHFvRsi-nFngojqjn08UaR3RY9i7CJ7T2UdR95kCrk,1056
 mplang/analysis/diagram.py,sha256=ffwgD12gL1_KH1uJ_EYkjmIlDrfxYJJkWj-wHl09_Xk,19520
-mplang/backend/__init__.py,sha256=Pn1MGW7FZ8ZQWcx_r2Io4Q1JbrMINCpefQt7yCNq_dc,741
-mplang/backend/base.py,sha256=zaofB1MPC9Af8FS-rx7q6utFxiu9Mppr2gWaKFZ70Os,9863
+mplang/backend/__init__.py,sha256=2WE4cmW96Xkzyq2yRRYNww4kZ5o6u6NbPV0BxqZG698,581
+mplang/backend/base.py,sha256=Rp4Ze1aF5dypdMOskXnQWE-rM5nc4vkdDogEoxQp7FU,5712
 mplang/backend/builtin.py,sha256=Mk1uUO2Vpw3meqZ0B7B0hG-wndea6cmFv2Uk1vM_uTg,7052
+mplang/backend/context.py,sha256=e22JzsHydi1DLkjzOEYwGBaAoVHyf8aEFYH_SuN7oR4,9351
 mplang/backend/crypto.py,sha256=H_s5HI7lUP7g0xz-a9qMbSn6dhJStUilKbn3-7SIh0I,3812
 mplang/backend/phe.py,sha256=uNqmrbDAbd97TWS_O6D5sopastHy6J20R7knFE4M4uc,65247
-mplang/backend/spu.py,sha256=i6Qqgeg-Anwpb5xX5Uz8GdmTWNkRy_pjp-xptIvlxl4,9273
-mplang/backend/sql_duckdb.py,sha256=WFEiJk3t68DMpjV9xZz5bJj3v8UMx4X5QWleLIE4Aq0,1499
+mplang/backend/spu.py,sha256=QT1q5uv-5P_nBGtTvtA_yI2h3h3zIqNSnvzGT7Shua4,9307
+mplang/backend/sql_duckdb.py,sha256=U_KzEUinxrBRDoUz2Vh597-N4I3hPOBT0RT3tX-ZqKE,1502
 mplang/backend/stablehlo.py,sha256=GOxy-qgOxyEdtBkt6LASKzaZnPewZhvHYSPOgFFXgIM,2612
 mplang/backend/tee.py,sha256=6kc7qTe8nWc3pr6iYtozEGLO8Umg-UBQLDiz6p3pdVg,1918
 mplang/core/__init__.py,sha256=lWxlEKfRwX7FNDzgyKZ1fiDMaCiqkyg0j5mKlZD_v7g,2244
@@ -29,20 +30,20 @@ mplang/core/tensor.py,sha256=86u6DogSZMoL0w5XjtTmQm2PhA_VjwybN1b6U4Zzphg,2361
 mplang/core/tracer.py,sha256=dVMfUeCMmPz4o6tLXewGMW1Kpy5gpZORvr9w4MhwDtM,14288
 mplang/core/expr/__init__.py,sha256=qwiSTUOcanFJLyK8HZ13_L1ZDrybqpPXIlTHAyeumE8,1988
 mplang/core/expr/ast.py,sha256=KE46KTtlH9RA2V_EzWVKCKolsycgTmt7SotUrOc8Qxs,20923
-mplang/core/expr/evaluator.py,sha256=0ZwDgRyzHh1zxmb-9XW85toVKpk2mk0_t-hY_10U_Q8,20786
+mplang/core/expr/evaluator.py,sha256=3rtfHVajmCUtkanzgiCWkepRRiTKehPcDF2i5EbcitI,20789
 mplang/core/expr/printer.py,sha256=VblKGnO0OUfzH7EBkszwRNxQUB8QyyC7BlJWJEUv9so,9546
 mplang/core/expr/transformer.py,sha256=TyL-8FjrVvDq_C9X7kAuKkiqt2XdZM-okjzVQj0A33s,4893
 mplang/core/expr/utils.py,sha256=VDTJ_-CsdHtVy9wDaGa7XdFxQ7o5lYYaeqcgsAhkbNI,2625
 mplang/core/expr/visitor.py,sha256=2Ge-I5N-wH8VVXy8d2WyNaEv8x6seiRx9peyH9S2BYU,2044
 mplang/core/expr/walk.py,sha256=lXkGJEEuvKGDqQihbxXPxfz2RfR1Q1zYUlt11iooQW0,11889
 mplang/frontend/__init__.py,sha256=3ZBFX_acM96tZ2mtJaxJm150n1cf0LnnCRmkrAc4uBw,1463
-mplang/frontend/base.py,sha256=I-Hhh5o6GVqBA1YySl9Nk3zkbMoVrQqLMRyX2JypsPI,18268
+mplang/frontend/base.py,sha256=rGtfBejcDh9mTRxOdJK5VUlG5vYiVJSir8X72X0Huvc,18264
 mplang/frontend/builtin.py,sha256=8qrlbe_SSy6QTXTnMG6_ADB8jSklVZGFBrkoR-p02FE,9368
 mplang/frontend/crypto.py,sha256=Nf8zT4Eko7MIs4R2tgZecKVd7d6Hvd_CGGmANhs3Ghs,3651
-mplang/frontend/ibis_cc.py,sha256=01joUdFS_Ja9--PkezBhEcW_a9mkDrLgOhu5320s_bQ,4167
+mplang/frontend/ibis_cc.py,sha256=CTTbPPZ9hFnHuFDDIfgJHie1EdNnHmi5Ha1KsX0iYh8,4235
 mplang/frontend/jax_cc.py,sha256=ssP6rCvyWQ5VAr80-7z9QZUE2mWXyozJCGpq1dYQYY8,6374
 mplang/frontend/phe.py,sha256=tDsCvStjVJ1Fs07yF3idkFnugUCA1zdFApPx7Uuulik,6795
-mplang/frontend/spu.py,sha256=yvnXH8HU55t7j_jaTpxa3Nbh0SqXufVDyZBTYCsRTK4,4994
+mplang/frontend/spu.py,sha256=7G6DaEfC5APSDhfeWSISTG_8tEcVbWth3XmjL8QUrVA,4994
 mplang/frontend/sql.py,sha256=DFdvjEPQX28VCRgUMeHYR0rwwOaoCH15bpvvlclLtHA,1999
 mplang/frontend/tee.py,sha256=EigmlbYDGvXkZCMHSYRAiOboSl9TG0ewoudbgl3_V6M,1393
 mplang/protos/v1alpha1/mpir_pb2.py,sha256=Bros37t-4LMJbuUYVSM65rImUYTtZDhNTIADGbZCKp0,7522
@@ -57,9 +58,9 @@ mplang/runtime/driver.py,sha256=Ok1jY301ctN1_KTb4jwSxOdB0lI_xhx9AwhtEGJ-VLQ,1130
 mplang/runtime/exceptions.py,sha256=c18U0xK20dRmgZo0ogTf5vXlkix9y3VAFuzkHxaXPEk,981
 mplang/runtime/http_api.md,sha256=-re1DhEqMplAkv_wnqEU-PSs8tTzf4-Ml0Gq0f3Go6s,4883
 mplang/runtime/link_comm.py,sha256=uNqTCGZVwWeuHAb7yXXQf0DUsMXLa8leHCkrcZdzYMU,4559
-mplang/runtime/resource.py,sha256=BkzpAjRrkjS-5FPawHHcVzxEE_htpMtS4JJCvcLrnU0,12564
+mplang/runtime/resource.py,sha256=-B9kSM7xhocc6mpXHmV9xTdpVR2duiUCepJKS7QuLqA,11688
 mplang/runtime/server.py,sha256=gTPqAux1EdefaBFnserYIXamoi7pbEsQrFX6cXbOjik,14716
-mplang/runtime/simulation.py,sha256=kj-RtvvypISO2xyYQGtm-N8yavnkVpD33KSLpvL-Vms,11107
+mplang/runtime/simulation.py,sha256=kuFXWuJLGcmy4OvLCBby4K5QbXaQZmKSb4qrCJ2stBY,10957
 mplang/simp/__init__.py,sha256=DmSMcKvHVXWS2pYsuHazEmwOWWpZeKOJQsNU6VxC10U,11614
 mplang/simp/mpi.py,sha256=Wv_Q16TQ3rdLam6OzqXiefIGSMmagGkso09ycyOkHEs,4774
 mplang/simp/random.py,sha256=7PVgWNL1j7Sf3MqT5PRiWplUu-0dyhF3Ub566iqX86M,3898
@@ -69,8 +70,8 @@ mplang/utils/crypto.py,sha256=rvPomBFtznRHc3RPi6Aip9lsU8zW2oxBqGv1K3vn7Rs,1052
 mplang/utils/func_utils.py,sha256=vCJcZmu0bEbqhOQKdpttV2_MBllIcPSN0b8U4WjNGGo,5164
 mplang/utils/spu_utils.py,sha256=S3L9RBkBe2AvSuMSQQ12cBY5Y1NPthubvErSX_7nj1A,4158
 mplang/utils/table_utils.py,sha256=aC-IZOKkSmFkpr3NZchLM0Wt0GOn-rg_xHBHREWBwAU,2202
-mplang_nightly-0.1.dev143.dist-info/METADATA,sha256=BvANiZEEU0N7uAY_k8Yd4p1rZKGPpdbrM5giBinXzdA,16547
-mplang_nightly-0.1.dev143.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-mplang_nightly-0.1.dev143.dist-info/entry_points.txt,sha256=mG1oJT-GAjQR834a62_QIWb7litzWPPyVnwFqm-rWuY,55
-mplang_nightly-0.1.dev143.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mplang_nightly-0.1.dev143.dist-info/RECORD,,
+mplang_nightly-0.1.dev144.dist-info/METADATA,sha256=i6ef5f8u6W_vm_UfZcAPsekVJ0j4a9wNYCdVDKJRBgw,16547
+mplang_nightly-0.1.dev144.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+mplang_nightly-0.1.dev144.dist-info/entry_points.txt,sha256=mG1oJT-GAjQR834a62_QIWb7litzWPPyVnwFqm-rWuY,55
+mplang_nightly-0.1.dev144.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mplang_nightly-0.1.dev144.dist-info/RECORD,,