metafunctions 1.1.2__py3-none-any.whl

metafunctions/__init__.py

@@ -0,0 +1,27 @@
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
+
+from metafunctions.api import bind_call_state
+from metafunctions.api import concurrent
+from metafunctions.api import locate_error
+from metafunctions.api import mmap
+from metafunctions.api import node
+from metafunctions.api import recall
+from metafunctions.api import star
+from metafunctions.api import store
+
+try:
+    __version__ = version("metafunctions")
+except PackageNotFoundError:
+    __version__ = "unknown"
+
+__all__ = [
+    "bind_call_state",
+    "concurrent",
+    "locate_error",
+    "mmap",
+    "node",
+    "recall",
+    "star",
+    "store",
+]

metafunctions/api.py

@@ -0,0 +1,151 @@
+"""
+Utility functions for use in function pipelines.
+"""
+
+import functools
+import typing as tp
+
+from metafunctions import operators
+from metafunctions import util
+from metafunctions.core import CallState
+from metafunctions.core import FunctionMerge
+from metafunctions.core import MetaFunction
+from metafunctions.core import SimpleFunction
+from metafunctions.core.concurrent import ConcurrentMerge
+from metafunctions.map import MergeMap
+
+
+def node(_func=None, *, name=None):
+    """Turn the decorated function into a MetaFunction.
+
+    Args:
+        _func: Internal use. This will be the decorated function if node is used as a decorator
+        with no params.
+
+    Usage:
+
+    @node
+    def f(x):
+       <do something cool>
+    """
+
+    def decorator(function):
+        newfunc = SimpleFunction(function, name=name)
+        return newfunc
+
+    if not _func:
+        return decorator
+    return decorator(_func)
+
+
+def bind_call_state(func):
+    @functools.wraps(func)
+    def provides_call_state(*args, **kwargs):
+        call_state = kwargs.pop("call_state")
+        return func(call_state, *args, **kwargs)
+
+    provides_call_state._receives_call_state = True
+    return provides_call_state
+
+
+def star(meta_function: MetaFunction) -> MetaFunction:
+    """
+    star calls its Metafunction with *x instead of x.
+    """
+    fname = str(meta_function)
+
+    # This convoluted inline `if` just decides whether we should add brackets or not.
+    @node(name=f"star{fname}" if fname.startswith("(") else f"star({fname})")
+    @functools.wraps(meta_function)
+    def wrapper(args, **kwargs):
+        return meta_function(*args, **kwargs)
+
+    return wrapper
+
+
+def store(key):
+    """Store the received output in the meta data dictionary under the given key."""
+
+    @node(name=f"store('{key}')")
+    @bind_call_state
+    def storer(call_state, val):
+        call_state.data[key] = val
+        return val
+
+    return storer
+
+
+def recall(key, from_call_state: CallState = None):
+    """Retrieve the given key from the meta data dictionary. Optionally, use `from_call_state` to
+    specify a different call_state than the current one.
+    """
+
+    @node(name=f"recall('{key}')")
+    @bind_call_state
+    def recaller(call_state, *_):
+        if from_call_state:
+            return from_call_state.data[key]
+        return call_state.data[key]
+
+    return recaller
+
+
+def concurrent(function: FunctionMerge) -> ConcurrentMerge:
+    """
+    Upgrade the specified FunctionMerge object to a ConcurrentMerge, which runs each of its
+    component functions in separate processes. See ConcurrentMerge documentation for more
+    information.
+
+    Usage:
+
+    c = concurrent(long_running_function + other_long_running_function)
+    c(input_data) # The two functions run in parallel
+    """
+    return ConcurrentMerge(function)
+
+
+def mmap(function: tp.Callable, operator: tp.Callable = operators.concat) -> MergeMap:
+    """
+    Upgrade the specified function to a MergeMap, which calls its single function once per input,
+    as per the builtin `map` (https://docs.python.org/3.6/library/functions.html#map).
+
+    Consider the name 'mmap' to be a placeholder for now.
+    """
+    return MergeMap(MetaFunction.make_meta(function), operator)
+
+
+def locate_error(
+    meta_function: MetaFunction,
+    use_color=util.system_supports_color(),  # noqa: B008
+) -> SimpleFunction:
+    """
+    Wrap the given MetaFunction with an error handler that adds location information to any
+    exception raised therein.
+
+    Usage:
+        cmp = locate_error(a | b | c)
+        cmp()
+    """
+
+    def with_location(*args, call_state, **kwargs):
+        new_e = None
+        try:
+            return meta_function(*args, call_state=call_state, **kwargs)
+        except Exception as e:  # noqa: BLE001
+            if hasattr(e, "location") and e.location:
+                # If the exception has location info attached
+                location = e.location
+            else:
+                location = call_state.highlight_active_function()
+
+            if use_color:
+                location = util.color_highlights(location)
+            detailed_message = (
+                f"{e!s} \n\nOccured in the following function: {location}"
+            )
+            new_e = type(e)(detailed_message).with_traceback(e.__traceback__)
+            new_e.__cause__ = e.__cause__
+        raise new_e
+
+    with_location._receives_call_state = True
+    return node(with_location, name=str(meta_function))

metafunctions/core/__init__.py

@@ -0,0 +1,19 @@
+from .base import DeferredValue
+from .base import FunctionChain
+from .base import FunctionMerge
+from .base import MetaFunction
+from .base import SimpleFunction
+from .call_state import CallState
+from .concurrent import ConcurrentMerge
+from .decorators import manage_call_state
+
+__all__ = [
+    "CallState",
+    "ConcurrentMerge",
+    "DeferredValue",
+    "FunctionChain",
+    "FunctionMerge",
+    "MetaFunction",
+    "SimpleFunction",
+    "manage_call_state",
+]

metafunctions/core/base.py

@@ -0,0 +1,294 @@
+import abc
+import functools
+import itertools
+import typing as tp
+
+from metafunctions import exceptions
+from metafunctions import operators
+from metafunctions.core.call_state import CallState
+from metafunctions.core.decorators import binary_operation
+from metafunctions.core.decorators import manage_call_state
+
+
+class MetaFunction(metaclass=abc.ABCMeta):
+    # Metafunctions will pass call state to any function with this attribute set to true
+    _receives_call_state = True
+    _function_join_str = ""
+
+    @abc.abstractmethod
+    def __init__(self, *args, **kwargs):
+        """A MetaFunction is a function that contains other functions. When executed, it calls the
+        functions it contains.
+        """
+        self._functions = []
+
+    @abc.abstractmethod
+    def __call__(self, *args, call_state=None, **kwargs):
+        """Call the functions contained in this MetaFunction"""
+
+    def __str__(self):
+        return "({})".format(
+            f" {self._function_join_str} ".join(str(f) for f in self.functions)
+        )
+
+    @property
+    def functions(self):
+        return self._functions
+
+    @staticmethod
+    def make_meta(function):
+        """Wrap the given function in a metafunction, unless it's already a metafunction"""
+        if not isinstance(function, MetaFunction):
+            return SimpleFunction(function)
+        return function
+
+    @staticmethod
+    def defer_value(value):
+        """Wrap the given value in a DeferredValue object, (which returns its value when called)."""
+        return DeferredValue(value)
+
+    @staticmethod
+    def new_call_state():
+        return CallState()
+
+    ### Operator overloads ###
+    @binary_operation
+    def __or__(self, other):
+        return FunctionChain.combine(self, other)
+
+    @binary_operation
+    def __ror__(self, other):
+        return FunctionChain.combine(other, self)
+
+    @binary_operation
+    def __and__(self, other):
+        return FunctionMerge.combine(operators.concat, self, other)
+
+    @binary_operation
+    def __rand__(self, other):
+        return FunctionMerge.combine(operators.concat, other, self)
+
+    @binary_operation
+    def __add__(self, other):
+        return FunctionMerge(operators.add, (self, other))
+
+    @binary_operation
+    def __radd__(self, other):
+        return FunctionMerge(operators.add, (other, self))
+
+    @binary_operation
+    def __sub__(self, other):
+        return FunctionMerge(operators.sub, (self, other))
+
+    @binary_operation
+    def __rsub__(self, other):
+        return FunctionMerge(operators.sub, (other, self))
+
+    @binary_operation
+    def __mul__(self, other):
+        return FunctionMerge(operators.mul, (self, other))
+
+    @binary_operation
+    def __rmul__(self, other):
+        return FunctionMerge(operators.mul, (other, self))
+
+    @binary_operation
+    def __truediv__(self, other):
+        return FunctionMerge(operators.truediv, (self, other))
+
+    @binary_operation
+    def __rtruediv__(self, other):
+        return FunctionMerge(operators.truediv, (other, self))
+
+    @binary_operation
+    def __matmul__(self, other):
+        from metafunctions.api import star
+
+        return FunctionChain.combine(self, star(other))
+
+    @binary_operation
+    def __rmatmul__(self, other):
+        from metafunctions.api import star
+
+        return FunctionChain.combine(other, star(self))
+
+
+class FunctionChain(MetaFunction):
+    _function_join_str = "|"
+
+    def __init__(self, *functions):
+        """A FunctionChain is a metafunction that calls its functions in sequence, passing the
+        results of the first function subsequent functions.
+        """
+        super().__init__()
+        self._functions = functions
+
+    @manage_call_state
+    def __call__(self, *args, **kwargs):
+        f_iter = iter(self._functions)
+        result = next(f_iter)(*args, **kwargs)
+        for f in f_iter:
+            result = f(result, **kwargs)
+        return result
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}{self.functions}"
+
+    @classmethod
+    def combine(cls, *funcs):
+        """Merge chains; i.e., combine all FunctionChains in `funcs` into a single FunctionChain."""
+        new_funcs = []
+        for f in funcs:
+            if type(f) is cls:
+                new_funcs.extend(f.functions)
+            else:
+                new_funcs.append(f)
+        return cls(*new_funcs)
+
+
+class FunctionMerge(MetaFunction):
+    _character_to_operator: tp.ClassVar = {
+        "+": operators.add,
+        "-": operators.sub,
+        "*": operators.mul,
+        "/": operators.truediv,
+        "&": operators.concat,
+    }
+    _operator_to_character: tp.ClassVar = {
+        v: k for k, v in _character_to_operator.items()
+    }
+
+    def __init__(self, merge_func: tp.Callable, functions: tuple, function_join_str=""):
+        """
+        A FunctionMerge merges its functions by executing all of them and passing their results to
+        `merge_func`.
+
+        Behaviour of __call__:
+
+        FunctionMerge does not pass all positional arguments to all of its functions. Rather, given
+        `f=FunctionMerge()` when f is called with `f(*args)`,
+
+        * if len(args) == 1, each component function is called with args[0]
+        * if len(args) > 1 <= len(functions), function n is called with arg n. Any remaining
+        functions after all args have been exhausted are called with no args.
+        * if len(args) < len(functions), a MetaFunction CallError is raised.
+
+        Args:
+            function_join_str: If you're using a `merge_func` that is not one of the standard operator
+            functions, use this argument to provide a custom character to use in string formatting. If
+            not provided, we default to using str(merge_func).
+        """
+        super().__init__()
+        self._merge_func = merge_func
+        self._functions = functions
+        self._function_join_str = function_join_str or self._operator_to_character.get(
+            merge_func, str(merge_func)
+        )
+
+    @manage_call_state
+    def __call__(self, *args, **kwargs):
+        args_iter, func_iter = self._get_call_iterators(args)
+
+        results = []
+        # Note that args_iter appears first in the zip. This is because I know its len is <=
+        # len(func_iter) (I asserted so above). In zip, if the first iterator is longer than the
+        # second, the first will be advanced one extra time, because zip has already called next()
+        # on the first iterator before discovering that the second has been exhausted.
+        for arg, f in zip(args_iter, func_iter):
+            results.append(self._call_function(f, (arg,), kwargs))
+
+        # Any extra functions are called with no input
+        results.extend([self._call_function(f, (), kwargs) for f in func_iter])
+        return self._merge_func(*results)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._merge_func}, {self.functions})"
+
+    @classmethod
+    def combine(cls, merge_func: tp.Callable, *funcs, function_join_str=None):
+        """Combine FunctionMerges. If consecutive FunctionMerges have the same merge_funcs, combine
+        them into a single FunctionMerge.
+
+        NOTE: combine does not check to make sure the merge_func can accept the new number of
+        arguments, or that combining is appropriate for the operator. (e.g., it is inappropriate to
+        combine FunctionMerges where order of operations matter. 5 / 2 / 3 != 5 / (2 / 3))
+        """
+        new_funcs = []
+        for f in funcs:
+            if isinstance(f, cls) and f._merge_func is merge_func:
+                new_funcs.extend(f.functions)
+            else:
+                new_funcs.append(f)
+        return cls(merge_func, tuple(new_funcs), function_join_str=function_join_str)
+
+    def _get_call_iterators(self, args):
+        """Do length checking and return (`args_iter`, `call_iter`), iterables of arguments and
+        self.functions. Call them using zip. Note that len(args) can be less than
+        len(self.functions), and remaining functions should be called with no argument.
+        """
+        args_iter = iter(args)
+        func_iter = iter(self.functions)
+        if len(args) > len(self.functions):
+            raise exceptions.CallError(
+                f"{self} takes 1 or <= {len(self.functions)} arguments, but {len(args)} were given"
+            )
+        if len(args) == 1:
+            args_iter = itertools.repeat(next(args_iter))
+
+        return args_iter, func_iter
+
+    def _call_function(self, f, args: tuple, kwargs: dict):
+        """This function receives one function, and the args and kwargs that should be used to call
+        that function. It returns the result of the function call. This gets its own method so that
+        subclasses can customize its behaviour.
+        """
+        return f(*args, **kwargs)
+
+
+class SimpleFunction(MetaFunction):
+    def __init__(self, function, name=None):
+        """A MetaFunction-aware wrapper around a single function
+        The `bind` parameter causes us to pass a meta object as the first argument to our inherited function, but it is only respected if the wrapped function is not another metafunction.
+        """
+        # An interesting side effect of wraps: it causes simplefunctions to collapse into each
+        # other. Because calling wraps on a function copies all that function's attributes to the
+        # new function, we copy _function, etc from the wrapped function. Essentially
+        # absorbing it. I'm not sure if that's good or bad.
+        functools.wraps(function)(self)
+
+        super().__init__()
+        self._function = function
+        self._name = name or getattr(function, "__name__", False) or str(function)
+
+    @manage_call_state
+    def __call__(self, *args, call_state, **kwargs):
+        if getattr(self._function, "_receives_call_state", False):
+            kwargs["call_state"] = call_state
+        return self._function(*args, **kwargs)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.functions[0]!r})"
+
+    def __str__(self):
+        return self._name
+
+    @property
+    def functions(self):
+        return (self._function,)
+
+
+class DeferredValue(SimpleFunction):
+    def __init__(self, value):
+        """A simple Deferred Value. Returns `value` when called. Equivalent to lambda x: x."""
+        self._value = value
+        self._name = repr(value)
+
+    def __call__(self, *args, **kwargs):
+        return self._value
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._value!r})"
+
+    @property
+    def functions(self):
+        return (self,)

metafunctions/core/call_state.py

@@ -0,0 +1,97 @@
+from collections import OrderedDict
+from collections import defaultdict
+from collections import namedtuple
+
+from metafunctions import util
+
+
+class CallState:
+    Node = namedtuple("Node", "function insert_index")
+
+    def __init__(self):
+        """
+        A call tree keeps track of function execution order in metafunctions. This is used to
+        accurately determine the location of the currently active function, and to identify
+        exception locations. It can be thought of as a metafunction aware call stack.
+        """
+        # Meta entry is conceptually the root node of the call tree
+        self._meta_entry = None
+        self.active_node = None
+        self._nodes_visited = 0
+
+        # A dictionary of {child: parent} functions
+        self._parents = OrderedDict()
+
+        # A dictionary of {parent: [children]}, in call order
+        self._children = defaultdict(list)
+        self.data = {}
+
+    def push(self, f):
+        """
+        Push a function onto the tree
+        """
+        if self._meta_entry is None:
+            node = self.Node(f, self._nodes_visited)
+            self._meta_entry = node
+        else:
+            node = self.Node(f, self._nodes_visited)
+            self._parents[node] = self.active_node
+            self._children[self.active_node].append(node)
+        self._nodes_visited += 1
+        self.active_node = node
+
+    def pop(self):
+        """Remove last inserted f from the call tree."""
+        try:
+            node, parent = self._parents.popitem()
+        except KeyError:
+            m = self._meta_entry
+            self._meta_entry = None
+            self.active_node = None
+            return m
+        self._children.pop(node, None)
+        self.active_node = parent
+        return node[0]
+
+    def iter_parent_nodes(self, node):
+        """
+        Return an iterator over all parents of this node in the tree, ending with the meta_entry
+        point.
+        """
+        # if node isn't in _parents, it must be the meta entry
+        parent = self._parents.get(node, self._meta_entry)
+        yield parent
+        if parent is not self._meta_entry:
+            yield from self.iter_parent_nodes(parent)
+
+    def highlight_active_function(self):
+        """
+        Return a formatted string showing the location of the most recently called function in
+        call_state.
+
+        Consider this a 'you are here' when called from within a function pipeline.
+        """
+        current_function = self.active_node.function
+        current_name = str(current_function)
+        new_name = util.highlight(current_name)
+
+        # rename active function in parent (if active function isn't in parent, active function
+        # becomes parent)
+        for parent in self.iter_parent_nodes(self.active_node):
+            parent_name = str(parent.function)
+
+            # Note we count occurences of current name in functions we've called so far. We do this
+            # because it's possible previous functions contain the name of this function.
+            name_count = sum(
+                str(n.function).count(current_name) for n in self._children[parent]
+            )
+
+            new_name = util.replace_nth(parent_name, current_name, name_count, new_name)
+            current_function = parent.function
+            current_name = parent_name
+
+            # if new parent name hasn't changed (meaning it didn't contain the name we're
+            # highlighting), highlight the parent name
+            if new_name == parent_name:
+                new_name = util.highlight(new_name)
+        return new_name