numbox 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

numbox/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.5'
+__version__ = '0.2.7'

numbox/core/proxy/proxy.py

@@ -7,20 +7,13 @@ from numba.extending import intrinsic  # noqa: F401
 from types import FunctionType as PyFunctionType
 from typing import List, Optional, Tuple
 
+from numbox.utils.standard import make_params_strings
+
 
 def make_proxy_name(name):
     return f'__{name}'
 
 
-def make_params_strings(func):
-    func_params = inspect.signature(func).parameters
-    func_params_str = ', '.join(
-        [k if v.default == inspect._empty else f'{k}={v.default}' for k, v in func_params.items()]
-    )
-    func_names_params_str = ', '.join(func_params.keys())
-    return func_params_str, func_names_params_str
-
-
 def proxy(sig, jit_options: Optional[dict] = None):
     """ Create a proxy for the decorated function `func` with the given signature(s) `sig`.
 

numbox/core/work/builder.py

@@ -1,10 +1,11 @@
-from hashlib import md5
+from collections import namedtuple
+from hashlib import sha256
 from inspect import getfile, getmodule, getsource
 from io import StringIO
 from itertools import chain
 from numba import njit, typeof
 from numba.core.types import Type
-from typing import Any, Callable, NamedTuple, Optional, Sequence, Union
+from typing import Any, Callable, Dict, NamedTuple, Optional, Sequence, Union
 
 from numbox.core.configurations import default_jit_options
 from numbox.core.work.lowlevel_work_utils import ll_make_work
@@ -15,7 +16,7 @@ def _file_anchor():
     raise NotImplementedError
 
 
-_nodes_names = set()
+_specs_registry = dict()
 
 
 class _End(NamedTuple):
@@ -27,10 +28,11 @@ class _End(NamedTuple):
 def _new(cls, super_proxy, *args, **kwargs):
     name = kwargs.get("name")
     assert name, "`name` key-word argument has not been provided"
-    if name in _nodes_names:
+    if name in _specs_registry:
         raise ValueError(f"Node '{name}' has already been defined on this graph. Pick a different name.")
-    _nodes_names.add(name)
-    return super_proxy.__new__(cls, *args, **kwargs)
+    spec_ = super_proxy.__new__(cls, *args, **kwargs)
+    _specs_registry[name] = spec_
+    return spec_
 
 
 class End(_End):
@@ -100,7 +102,7 @@ def _derived_line(
     sources_ = sources_ + ", " if "," not in sources_ else sources_
     ty_ = get_ty(derived_)
     derive_func = derived_.derive
-    derive_hashes.append(md5(getsource(derive_func).encode("utf-8")).hexdigest())
+    derive_hashes.append(sha256(getsource(derive_func).encode("utf-8")).hexdigest())
     derive_ = _derived_cres(ty_, derived_.sources, derive_func, jit_options)
     derive_name = f"{name_}_derive"
     init_name = f"{name_}_init"
@@ -111,29 +113,34 @@ def _derived_line(
     return f"""{name_} = ll_make_work("{name_}", {init_name}, ({sources_}), {derive_name})"""
 
 
-def _verify_access_nodes(
-    all_inputs_: Sequence[End],
-    all_derived_: Sequence[Derived],
-    access_nodes: Sequence[SpecTy]
-):
-    for access_node in access_nodes:
-        assert access_node in all_inputs_ or access_node in all_derived_, f"{access_node} cannot be reached"
-
-
 def code_block_hash(code_txt: str):
     """ Re-compile and re-save cache when source code has changed. """
-    return md5(code_txt.encode("utf-8")).hexdigest()
+    return sha256(code_txt.encode("utf-8")).hexdigest()
 
 
-def make_graph(
-    all_inputs_: Sequence[End],
-    all_derived_: Sequence[Derived],
-    access_nodes: SpecTy | Sequence[SpecTy],
-    jit_options: Optional[dict] = None
-):
-    if isinstance(access_nodes, SpecTy):
-        access_nodes = (access_nodes,)
-    _verify_access_nodes(all_inputs_, all_derived_, access_nodes)
+def _infer_end_and_derived_nodes(spec: SpecTy, all_inputs_: Dict[str, Type], all_derived_: Dict[str, Type]):
+    if spec.name in all_inputs_ or spec.name in all_derived_:
+        return
+    if isinstance(spec, End):
+        all_inputs_[spec.name] = get_ty(spec)
+        return
+    for source in spec.sources:
+        _infer_end_and_derived_nodes(source, all_inputs_, all_derived_)
+    all_derived_[spec.name] = get_ty(spec)
+
+
+def infer_end_and_derived_nodes(access_nodes: SpecTy | Sequence[SpecTy]):
+    all_inputs_ = dict()
+    all_derived_ = dict()
+    for access_node in access_nodes:
+        _infer_end_and_derived_nodes(access_node, all_inputs_, all_derived_)
+    all_inputs_lst = [_specs_registry[name] for name in all_inputs_.keys()]
+    all_derived_lst = [_specs_registry[name] for name in all_derived_.keys()]
+    return all_inputs_lst, all_derived_lst
+
+
+def make_graph(*access_nodes: SpecTy | Sequence[SpecTy], jit_options: Optional[dict] = None):
+    all_inputs_, all_derived_ = infer_end_and_derived_nodes(access_nodes)
     if jit_options is None:
         jit_options = {}
     jit_options = {**default_jit_options, **jit_options}
@@ -144,26 +151,30 @@ def make_graph(
     _make_args = []
     code_txt = StringIO()
     initializers = {}
-    derive_hashes=[]
+    derive_hashes = []
     for input_ in all_inputs_:
         line_ = _input_line(input_, ns, initializers)
         code_txt.write(f"\n\t{line_}")
     for derived_ in all_derived_:
         line_ = _derived_line(derived_, ns, initializers, derive_hashes, _make_args, jit_options)
         code_txt.write(f"\n\t{line_}")
-    hash_str = f"code_block = {code_txt.getvalue()} initializers = {list(initializers.values())} derive_hashes = {derive_hashes}"
+    hash_str = f"code_block = {code_txt.getvalue()} initializers = {list(initializers.values())} derive_hashes = {derive_hashes}"  # noqa: E501
     hash_ = code_block_hash(hash_str)
-    code_txt.write(f"""\n\taccess_tuple = ({", ".join([n.name for n in access_nodes])})""")
-    code_txt.write(f"\n\treturn access_tuple")
+    access_nodes_names = [n.name for n in access_nodes]
+    tup_ = ", ".join(access_nodes_names)
+    tup_ = tup_ + ", " if ", " not in tup_ else tup_
+    code_txt.write(f"""\n\taccess_tuple = ({tup_})""")
+    code_txt.write("\n\treturn access_tuple")
     code_txt = code_txt.getvalue()
     make_params = ", ".join(chain(_make_args, initializers.keys()))
     make_name = f"_make_{hash_}"
     code_txt = f"""
 @njit(**jit_options)
 def {make_name}({make_params}):""" + code_txt + f"""
-return_node_ = {make_name}({make_params})
+access_tuple_ = {make_name}({make_params})
 """
     code = compile(code_txt, getfile(_file_anchor), mode="exec")
     exec(code, ns)
-    return_node_ = ns["return_node_"]
-    return return_node_
+    access_tuple_ = ns["access_tuple_"]
+    Access = namedtuple("Access", access_nodes_names)
+    return Access(*access_tuple_)

numbox/core/work/explain.py

@@ -30,6 +30,9 @@ def _explain(work: Work, derivation_: list, derived_: set):
 
 
 def explain(work: Work):
+    all_end_nodes = work.all_end_nodes()
     derivation = []
     _explain(work, derivation, set())
-    return "\n".join(derivation)
+    explain_txt = f"All required end nodes: {all_end_nodes}\n\n"
+    explain_txt += "\n".join(derivation)
+    return explain_txt

numbox/utils/highlevel.py

@@ -1,4 +1,7 @@
 import hashlib
+import re
+from inspect import getfile, getmodule, getsource
+from io import StringIO
 from numba import njit
 from numba.core.itanium_mangler import mangle_type_or_value
 from numba.core.types import Type
@@ -6,6 +9,17 @@ from numba.core.types.functions import Dispatcher
 from numba.core.types.function_type import CompileResultWAP
 from numba.core.typing.templates import Signature
 from numba.experimental.function_type import FunctionType
+from numba.experimental.structref import define_boxing, new, StructRefProxy
+from numba.extending import overload, overload_method
+from textwrap import dedent, indent
+from typing import Callable, Dict, Iterable, Optional
+
+from numbox.core.configurations import default_jit_options
+from numbox.utils.standard import make_params_strings
+
+
+def _file_anchor():
+    raise NotImplementedError
 
 
 def cres(sig, **kwargs):
@@ -32,7 +46,164 @@ def determine_field_index(struct_ty, field_name):
 
 def hash_type(ty: Type):
     mangled_ty = mangle_type_or_value(ty)
-    return hashlib.sha256(mangled_ty.encode('utf-8')).hexdigest()
+    return hashlib.sha256(mangled_ty.encode("utf-8")).hexdigest()
+
+
+def make_structref_code_txt(
+    struct_name: str,
+    struct_fields: Iterable[str] | Dict[str, Type],
+    struct_type_class: type | Type,
+    struct_methods: Optional[Dict[str, Callable]] = None,
+    jit_options: Optional[dict] = None
+):
+    if isinstance(struct_fields, dict):
+        struct_fields, fields_types = list(struct_fields.keys()), list(struct_fields.values())
+    else:
+        assert isinstance(struct_fields, (list, tuple)), struct_fields
+        fields_types = None
+    struct_fields_str = ", ".join([field for field in struct_fields])
+    make_name = f"make_{struct_name.lower()}"
+    new_returns = f"{make_name}({struct_fields_str})"
+    repr_str = f"f'{struct_name}(" + ", ".join([f"{field}={{self.{field}}}" for field in struct_fields]) + ")'"
+    code_txt = StringIO()
+    code_txt.write(f"""
+class {struct_name}(StructRefProxy):
+    def __new__(cls, {struct_fields_str}):
+        return {new_returns}
+
+    def __repr__(self):
+        return {repr_str}
+""")
+    for field in struct_fields:
+        code_txt.write(f"""
+    @property
+    @njit(**jit_options)
+    def {field}(self):
+        return self.{field}
+""")
+    methods_code_txt = StringIO()
+    if struct_methods is not None:
+        assert isinstance(struct_methods, dict), f"""
+    Expected dictionary of methods names to callable, got {struct_methods}"""
+        for method_name, method in struct_methods.items():
+            params_str, names_params_str = make_params_strings(method)
+            names_params_lst = names_params_str.split(", ")
+            self_name = names_params_lst[0]
+            names_params_str_wo_self = ", ".join(names_params_lst[1:])
+            method_source = dedent(getsource(method))
+            method_hash = hashlib.sha256(method_source.encode("utf-8")).hexdigest()
+            code_txt.write(f"""
+    def {method_name}({params_str}):
+        return {self_name}.{method_name}_{method_hash}({names_params_str_wo_self})
+    
+    @njit(**jit_options)
+    def {method_name}_{method_hash}({params_str}):
+        return {self_name}.{method_name}({names_params_str_wo_self})
+""")
+            method_header = re.findall(r"^\s*def\s+([a-zA-Z_]\w*)\s*\(([^)]*)\)\s*:[^\n]*", method_source, re.MULTILINE)
+            assert len(method_header) == 1, method_header
+            method_name, params_str_ = method_header[0]
+            assert params_str == params_str_, (params_str, params_str_)
+            method_source = re.sub(r"\bdef\s+([a-zA-Z_]\w*)\b", f"def _", method_source)
+            methods_code_txt.write(f"""
+@overload_method({struct_type_class.__name__}, "{method_name}", jit_options=jit_options)
+def ol_{method_name}({params_str}):
+{indent(method_source, "    ")}
+    return _
+""")
+    code_txt.write(f"""
+define_boxing({struct_type_class.__name__}, {struct_name})
+""")
+    struct_type_name = f"{struct_name}Type"
+    struct_fields_ty_str = ", ".join([f"{field}_ty" for field in struct_fields])
+    struct_type_code_block = ""
+    if fields_types is None:
+        struct_type_code_block = f"""fields_types = [{struct_fields_ty_str}]
+    fields_and_their_types = list(zip(fields, fields_types))    
+    {struct_name}Type = {struct_type_class.__name__}(fields_and_their_types)        
+"""
+    else:
+        code_txt.write(f"""
+fields_and_their_types = list(zip(fields, fields_types))    
+{struct_name}Type = {struct_type_class.__name__}(fields_and_their_types)
+""")
+    ctor_code_block = "\n".join([f"        struct_.{field} = {field}" for field in struct_fields])
+    code_txt.write(f"""
+@overload({struct_name}, strict=False, jit_options=jit_options)
+def ol_{struct_name.lower()}({struct_fields_ty_str}):
+    {struct_type_code_block}
+    def ctor({struct_fields_str}):
+        struct_ = new({struct_type_name})
+{ctor_code_block}
+        return struct_
+    return ctor
+""")
+    if fields_types is not None:
+        code_txt.write(f"""
+{make_name}_sig = {struct_name}Type(*fields_types)
+""")
+    else:
+        code_txt.write(f"""
+{make_name}_sig = None
+""")
+    code_txt.write(f"""
+@njit({make_name}_sig, **jit_options)
+def {make_name}({struct_fields_str}):
+    return {struct_name}({struct_fields_str})
+""")
+    code_txt = code_txt.getvalue() + methods_code_txt.getvalue()
+    return code_txt, fields_types
+
+
+def make_structref(
+    struct_name: str,
+    struct_fields: Iterable[str] | Dict[str, Type],
+    struct_type_class: type | Type,
+    *,
+    struct_methods: Optional[Dict[str, Callable]] = None,
+    jit_options: Optional[dict] = None
+):
+    """
+    Makes structure type with `struct_name` and `struct_fields` from the StructRef type class.
+
+    A unique `struct_type_class` for each structref needs to be provided.
+    If caching of code that will be using the created struct type is desired,
+    these type class(es) need/s to be defined in a python module that is *not* executed.
+    (Same requirement is also to observed even when the full definition of StructRef
+    is entirely hard-coded rather than created dynamically.)
+
+    In particular, that's why `struct_type_class` cannot be incorporated into
+    the dynamic compile / exec routine here.
+
+    Dictionary of methods to be bound to the created structref can be provided as well.
+    Struct methods will get inlined into the caller if numba deems it to be optimal
+    (even if `jit_options` says otherwise), therefore changing the methods code
+    without poking the jitted caller can result in a stale cache - when the latter is
+    cached. This is not an exclusive limitation of a dynamic structref creation via
+    this function and is equally true when the structref definition is coded explicitly.
+    """
+    code_txt, fields_types = make_structref_code_txt(
+        struct_name, struct_fields, struct_type_class, struct_methods, jit_options
+    )
+    if jit_options is None:
+        jit_options = default_jit_options
+    ns = {
+        **getmodule(_file_anchor).__dict__,
+        **{
+            "fields": struct_fields,
+            "fields_types": fields_types,
+            "define_boxing": define_boxing,
+            "jit_options": jit_options,
+            "new": new,
+            "njit": njit,
+            "overload_method": overload_method,
+            "StructRefProxy": StructRefProxy,
+            struct_type_class.__name__: struct_type_class
+        }
+    }
+    code = compile(code_txt, getfile(_file_anchor), mode="exec")
+    exec(code, ns)
+    return ns[struct_name]
 
 
 def prune_type(ty):

numbox/utils/standard.py

@@ -0,0 +1,17 @@
+import inspect
+
+
+def make_params_strings(func):
+    func_params = inspect.signature(func).parameters
+    func_params_str = ', '.join(
+        [k if v.default == inspect._empty else f'{k}={v.default}' for k, v in func_params.items()]
+    )
+    func_names_params_str = ', '.join(func_params.keys())
+    return func_params_str, func_names_params_str
+
+
+if __name__ == "__main__":
+    def aux(x, y, z=1):
+        pass
+    print(make_params_strings(aux))
+

{numbox-0.2.5.dist-info → numbox-0.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: numbox
-Version: 0.2.5
+Version: 0.2.7
 Author: Mikhail Goykhman
 License: MIT License (with Citation Clause)
         

{numbox-0.2.5.dist-info → numbox-0.2.7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-numbox/__init__.py,sha256=8f4ummWTPl6PtdE9DzSUgZhPjGioA_NDC-hDpqzB9qI,22
+numbox/__init__.py,sha256=e3cfQy_iVs9ILsymUalJnJu5D_dd7HuUVoVwKKurcL0,22
 numbox/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 numbox/core/configurations.py,sha256=0bCmxXL-QMwtvyIDhpXLeT-1KJMf_QpH0wLuEvYLGxQ,68
 numbox/core/any/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,11 +13,11 @@ numbox/core/bindings/call.py,sha256=LrSsp-b4Mz0Zjg7H3XkThEIOEqVcrcViEjI9yqhuMV4,
 numbox/core/bindings/signatures.py,sha256=OcSBDpJ422eoWkJXxHPEanMNbVB7bq9f5bRq5LGr86w,479
 numbox/core/bindings/utils.py,sha256=aRtN8oUYBk9vgoUGaUJosGx0Za-vvCNwwbZg_g_-LRs,460
 numbox/core/proxy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-numbox/core/proxy/proxy.py,sha256=kGYlEdLK40lxxu56e_S32s9YuQ6AuQnFegt5uQrUw5w,3889
+numbox/core/proxy/proxy.py,sha256=Wt7yzswDmeQXt0yjcTcnLi2coneowSHWXy_IFpZZJMU,3612
 numbox/core/work/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-numbox/core/work/builder.py,sha256=tfrizyZuBy21zsoILELzQIj-wtYAuPitDqtVg7V1atw,5362
+numbox/core/work/builder.py,sha256=U7hxopSwOc1ke1Hqup6xCliO7bqjq91zVX1SixYPxoM,6072
 numbox/core/work/combine_utils.py,sha256=qTVGke_ydzaTQ7o29DFjZWZzKjRNKb0L3yJMaR3TLII,2430
-numbox/core/work/explain.py,sha256=qEorYHRSe8P2MygodCYsz57UtWeRfaS27xHSw3hBgmo,1063
+numbox/core/work/explain.py,sha256=ESwvsTgfe0w7UnM13yyVpVDtfJyAK2A1sNdF3RNb-jU,1200
 numbox/core/work/loader_utils.py,sha256=g83mDWidZJ8oLWP3I3rK8aGISYOO2S-w6HDgtosCyck,1572
 numbox/core/work/lowlevel_work_utils.py,sha256=TgRRcNfks0oaOXGXXr3ptafd_Xv_lpmH8sjBrJ9bPuI,5748
 numbox/core/work/node.py,sha256=CMolyoRQjG2A-pTQqZQ0kxKOYTKipWRC0mu8RWHuTUI,5096
@@ -26,13 +26,14 @@ numbox/core/work/print_tree.py,sha256=y2u7xmbHvpcA57y8PrGSqOunLNCqhgNXdVtXHqvy1M
 numbox/core/work/work.py,sha256=V_pYW0sqdYHh45ago3aMz0fG4U_UfsNLIcpBejn_ttU,14725
 numbox/core/work/work_utils.py,sha256=3q_nnBdzuxWWcdFpbRL2H0T9ZNkUgx1J1uhiZkX3YG4,1039
 numbox/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-numbox/utils/highlevel.py,sha256=gXYzLsFPKHQfGqaz-Z4DWcCQddv0dS6SKwIsM-_xjYg,1487
+numbox/utils/highlevel.py,sha256=0sUVGOFAzFaGycKrXloGySLjp5EAPaf1B0AcOT1dfbw,8326
 numbox/utils/lowlevel.py,sha256=ACpf8_HyOIsobPlZ31bapkEyuCsV5dojW3AFrcKykrw,10712
 numbox/utils/meminfo.py,sha256=ykFi8Vt0WcHI3ztgMwvpn6NqaflDSQGL8tjI01jrzm0,1759
+numbox/utils/standard.py,sha256=2fPrMlSXe2TG3CIfjJOT8LQkHEH86oOOj1AvwQkYCfA,450
 numbox/utils/timer.py,sha256=KkAkWOHQ72WtPjyiAzt_tF1q0DcOnCDkITTb85DvkUM,553
 numbox/utils/void_type.py,sha256=IkZsjNeAIShYJtvWbvERdHnl_mbF1rCRWiM3gp6II8U,404
-numbox-0.2.5.dist-info/LICENSE,sha256=YYgNvjH_p6-1NsdrIqGJnr1GUbZzA_8DxsP6vVfM6nY,1446
-numbox-0.2.5.dist-info/METADATA,sha256=kHcNZstr-whJ0XoJ6IcoXACjnAn33yjvJgdn4Ksx6sk,2792
-numbox-0.2.5.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-numbox-0.2.5.dist-info/top_level.txt,sha256=A67jOkfqidCSYYm6ifjN_WZyIiR1B27fjxv6nNbPvjc,7
-numbox-0.2.5.dist-info/RECORD,,
+numbox-0.2.7.dist-info/LICENSE,sha256=YYgNvjH_p6-1NsdrIqGJnr1GUbZzA_8DxsP6vVfM6nY,1446
+numbox-0.2.7.dist-info/METADATA,sha256=J-Ash1idpJn-eUghPfVjNlVQhkbT0DHKa1D4K-A3ZTs,2792
+numbox-0.2.7.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+numbox-0.2.7.dist-info/top_level.txt,sha256=A67jOkfqidCSYYm6ifjN_WZyIiR1B27fjxv6nNbPvjc,7
+numbox-0.2.7.dist-info/RECORD,,