fred-oss 0.62.0__tar.gz → 0.64.0__tar.gz

{fred_oss-0.62.0/src/main/fred_oss.egg-info → fred_oss-0.64.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.62.0
+Version: 0.64.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development
@@ -10,6 +10,7 @@ Description-Content-Type: text/markdown
 License-File: NOTICE.txt
 Requires-Dist: fire==0.7.1
 Requires-Dist: psutil==7.0.0
+Requires-Dist: dill==0.4.0
 Requires-Dist: redis==6.4.0
 Requires-Dist: requests==2.32.5
 Requires-Dist: fastapi==0.116.2

{fred_oss-0.62.0 → fred_oss-0.64.0}/requirements.txt

@@ -2,6 +2,8 @@
 fire==0.7.1
 # Minimal requirements
 psutil==7.0.0
+# Fred Core Implementation
+dill==0.4.0
 # Fred Worker/Runner Infra
 redis==6.4.0
 requests==2.32.5

fred_oss-0.64.0/src/main/fred/edag/__init__.py

@@ -0,0 +1,15 @@
+from fred.maturity import Maturity, MaturityLevel
+from fred.edag.decorator import NodeDecorator
+
+
+module_maturity = Maturity(
+    level=MaturityLevel.ALPHA,
+    reference=__name__,
+    message=(
+        "Fred-eDAG implementation is in early development "
+        "and therefore currently with incomplete and unstable features."
+    )
+)
+
+
+node = NodeDecorator

fred_oss-0.64.0/src/main/fred/edag/comp/_group.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+
+from fred.edag.comp.interface import ComponentInterface
+from fred.edag.comp._node import Node
+
+
+@dataclass(frozen=True, slots=True, unsafe_hash=True)
+class Group(ComponentInterface):
+    nodes: list[Node]  # TODO: let's make this a frozenset instead of a list to ensure immutability

fred_oss-0.64.0/src/main/fred/edag/comp/_node.py

@@ -0,0 +1,198 @@
+import uuid
+from inspect import Signature, signature, Parameter
+from dataclasses import dataclass, field, asdict
+from typing import Callable, Optional
+
+from fred.settings import logger_manager
+from fred.edag.comp.interface import ComponentInterface
+
+logger = logger_manager.get_logger(__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class NodeFun:
+    fname: str
+    inner: Callable
+    signature: Signature
+    
+
+    @classmethod
+    def auto(cls, function: Callable, name: Optional[str] = None) -> "NodeFun":
+        fname = name or getattr(function, "__name__", "undefined")
+        return cls(
+            fname=fname,
+            inner=function,
+            signature=signature(function),
+        )
+
+    def validate_parameter_compliance(self, *args, **kwargs) -> dict:
+        # Determine if the signature accepts **kwargs
+        var_kwargs = any(
+            p.kind == Parameter.VAR_KEYWORD
+            for p in self.signature.parameters.values()
+        )
+        # Validate keywords against signature if not accepting **kwargs
+        clean_kwargs = {
+            k: v
+            for k, v in kwargs.items()
+            if k in self.signature.parameters
+        } if not var_kwargs else kwargs
+        # Bind arguments to signature
+        bound = self.signature.bind_partial(*args, **clean_kwargs)
+        bound.apply_defaults()
+        # Return the bound arguments as a dictionary
+        return {
+            "args": bound.args,
+            "kwargs": bound.kwargs,
+        }
+
+    def __call__(self, *args, **kwargs):
+        params = self.validate_parameter_compliance(*args, **kwargs)
+        return self.inner(*params["args"], **params["kwargs"])
+    
+    def __name__(self):
+        return self.fname
+
+    def __hash__(self):
+        return hash((
+            self.inner,
+            tuple(self.signature.parameters.items()),
+        ))
+
+
+@dataclass(frozen=True, slots=True)
+class Node(ComponentInterface):
+    name: str
+    key: str  # Output key
+    nfun: NodeFun
+    # TODO: let's make the 'params' a frozenset (i.e., frozenparams) instead of a dict to ensure immutability
+    params: dict = field(default_factory=dict)
+    nid: str = field(default_factory=lambda: str(uuid.uuid4()))
+    _inplace: bool = False
+    _explode: bool = False  # Whether this node's output should be exploded when used as input to another node
+
+    def __hash__(self):
+        obj = asdict(self)
+        obj["nfun"] = self.nfun.__hash__()
+        obj["params"] = frozenset((obj.get("params") or {}).keys())  # only hash keys to avoid unhashable values
+        return hash(frozenset(obj.items()))
+
+    def __getitem__(self, key) -> "Node":
+        if key is Ellipsis:  # node[...] syntax
+            return self.clone(key="*")
+        logger.warning("Node indexing is reserved for syntax like node[...] to indicate iterator mode.")
+        raise KeyError(f"Node does not support indexing with key: {key}")
+
+    def clone(self, **kwargs) -> "Node":
+        # Verify if 'inplace' is set via '_inplace' or 'inplace' keys; otherwise, keep current value
+        for key in ("inplace", "_inplace"):
+            value = kwargs.pop(key, None)
+            if isinstance(value, bool):
+                kwargs["_inplace"] = value
+                break
+        else:
+            kwargs["_inplace"] = self._inplace
+        # Verify if 'explode' is set via '_explode' or 'explode' keys; otherwise, keep current value
+        for key in ("explode", "_explode"):
+            value = kwargs.pop(key, None)
+            if isinstance(value, bool):
+                kwargs["_explode"] = value
+                break
+        else:
+            kwargs["_explode"] = self._explode
+        # Create a new Node with updated attributes
+        return self.__class__(
+            **{
+                "name": self.name,
+                "key": self.key,
+                "nfun": self.nfun,
+                "params": self.params,
+                **kwargs,
+            },
+            nid=str(uuid.uuid4()),  # Must have a new ID
+        )
+
+    def wrap(self, function: Callable) -> "Node":
+        fname = getattr(function, "__name__", "undef_wrapper_function")
+        return self.clone(
+            nfun=NodeFun.auto(
+                name=fname,
+                function=lambda *args, **kwargs: function(self.fun(*args, **kwargs))
+            ),
+        )
+
+    @classmethod
+    def auto(
+            cls,
+            function: Callable,
+            inplace: bool = False,
+            explode: bool = False,
+            fname: Optional[str] = None,
+            name: Optional[str] = None,
+            key: Optional[str] = None,
+            **params,
+    ):
+        name = name or getattr(function, "__name__", "undefined")
+        return cls(
+            name=name,
+            key=key or name,
+            nfun=NodeFun.auto(function=function, name=fname),
+            _inplace=inplace,
+            _explode=explode,
+            params=params,
+        )
+
+    @property
+    def fun(self) -> Callable:
+        return self.nfun
+
+    def inplace(self) -> "Node":
+        return self.clone(_inplace=True)
+    
+    def explode(self) -> "Node":
+        return self.clone(_explode=True)
+
+    @property
+    def E(self) -> "Node":
+        # Shortcut to set explode=True
+        return self.explode()
+
+    def __invert__(self) -> "Node":
+        # Unary ~ operator to set explode=True
+        return self.explode()
+
+    def with_output(self, key: str) -> "Node":
+        return self.with_alias(alias=self.name, key=key, keep_key=False)
+
+    def with_alias(self, alias: str, key: Optional[str] = None, keep_key: bool = False) -> "Node":
+        return self.__class__(
+            name=alias,
+            key=key or (self.key if keep_key else alias),
+            nfun=self.nfun,
+            params=self.params,
+            _inplace=self._inplace,
+            _explode=self._explode,
+        )
+
+    def with_params(self, update_key: Optional[str] = None, **params) -> "Node":
+        return self.__class__(
+            name=self.name,
+            nfun=self.nfun,
+            key=update_key or self.key,
+            params={
+                **self.params,
+                **params,
+            },
+            _inplace=self._inplace,
+            _explode=self._explode,
+        )
+
+    def execute(self, *args, **kwargs):
+        params = {
+            **self.params,
+            **kwargs
+        }
+        if self._inplace:
+            return self.fun(*args, **params)
+        from fred.future.impl import Future
+        return Future(self.fun, **params)

fred_oss-0.64.0/src/main/fred/edag/comp/catalog.py

@@ -0,0 +1,26 @@
+import enum
+
+from fred.edag.comp.interface import ComponentInterface
+from fred.edag.comp._group import Group
+from fred.edag.comp._node import Node
+from fred.edag.plan import Plan
+
+
+class PlanCompositionMixin:
+
+    def __rshift__(self, other: ComponentInterface | Plan) -> Plan:
+        left = Plan.as_plan(self)
+        right = Plan.as_plan(other)
+        return left >> right
+
+
+class CompCatalog(enum.Enum):
+    NODE = type("Node", (Node, PlanCompositionMixin), {})
+    GROUP = type("Group", (Group, PlanCompositionMixin), {})
+
+    @property
+    def ref(self) -> type[ComponentInterface]:
+        return self.value
+
+    def __call__(self, *args, **kwargs) -> ComponentInterface:
+        return self.value(*args, **kwargs)

fred_oss-0.64.0/src/main/fred/edag/comp/interface.py

@@ -0,0 +1,20 @@
+
+class ComponentInterface:
+    
+    def __or__(self, other: 'ComponentInterface') -> 'ComponentInterface':
+        # union: d1 | d2 | d3  (build a group)
+        # allow Node | Group and Group | Node chaining
+        from fred.edag.comp._group import Group
+        from fred.edag.comp._node import Node
+
+        match self, other:
+            case Node(), Node():
+                return Group(nodes=[self, other])
+            case Node(), Group():
+                return Group(nodes=[self, *other.nodes])
+            case Group(), Node():
+                return Group(nodes=[*self.nodes, other])
+            case Group(), Group():
+                return Group(nodes=[*self.nodes, *other.nodes])
+            case _:
+                raise TypeError("| expects Node or Group")

fred_oss-0.64.0/src/main/fred/edag/conn/catalog.py

@@ -0,0 +1,21 @@
+import enum
+
+from fred.edag.comp.catalog import CompCatalog
+
+
+class ConnCatalog(enum.Enum):
+    PASS = CompCatalog.NODE.ref.auto(
+        name="conn:passthrough",
+        function=lambda **kwargs: kwargs,
+        explode=True,
+        inplace=True,
+    )
+    PASS_PRINT = CompCatalog.NODE.ref.auto(
+        name="conn:passthrough_print",
+        function=lambda **kwargs: print(kwargs) or kwargs,
+        explode=True,
+        inplace=True,
+    )
+
+    def __call__(self, *args, **kwargs) -> CompCatalog.NODE.ref:
+        return self.value.clone(*args, **kwargs)

fred_oss-0.64.0/src/main/fred/edag/decorator.py

@@ -0,0 +1,48 @@
+from typing import Callable, Optional
+
+from fred.edag.comp.catalog import CompCatalog
+
+
+class NodeDecorator:
+    """Decorator to create Node instances from functions.
+    Can be used with or without parameters.
+
+    Example usage:
+
+    @NodeDecorator
+    def my_function(...):
+        ...
+    or
+    @NodeDecorator(name="example", param1=value1, param2=value2)
+        def my_function(...):
+    """
+
+    def __new__(cls, func: Optional[Callable] = None, **kwargs):
+        if not func:
+            # Create an instance of NodeDecorator to hold kwargs until __call__ is invoked
+            instance = super(NodeDecorator, cls).__new__(cls)
+            instance.kwargs = kwargs
+            return instance
+        # Return a Node instance directly if func is provided
+        return CompCatalog.NODE.ref.auto(
+            function=func,
+            **kwargs,
+        )
+
+    def __init__(self, func: Optional[Callable] = None, **kwargs):
+        self.func = func
+        self.kwargs = kwargs
+
+    def __call__(self, func: Callable, **kwargs) -> CompCatalog.NODE.ref:
+        self.func = func
+        self.kwargs = {**self.kwargs, **kwargs}
+        return self.get_node()
+
+    def get_node(self, **kwargs) -> CompCatalog.NODE.ref:
+        return CompCatalog.NODE.ref.auto(
+            function=self.func,
+            **{
+                **self.kwargs,
+                **kwargs,
+            },
+        )

fred_oss-0.64.0/src/main/fred/edag/executor.py

@@ -0,0 +1,144 @@
+import uuid
+from graphlib import TopologicalSorter
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from fred.future.impl import Future
+from fred.settings import logger_manager
+from fred.edag.comp.catalog import CompCatalog
+from fred.edag.plan import Plan
+
+
+logger = logger_manager.get_logger(__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class Executor:
+    predmap: dict[CompCatalog.NODE.ref, set[CompCatalog.NODE.ref]]
+    results: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+    @classmethod
+    def from_plan(cls, plan: Plan, **kwargs) -> "Executor":
+        return cls(predmap=plan.as_predmap(**kwargs))
+    
+    def get_tsort(self) -> TopologicalSorter:
+        return TopologicalSorter(self.predmap)
+        
+    def loop(
+            self,
+            run_id: str,
+            tsort: TopologicalSorter,
+            prev_layer: list[list[str]],
+            start_with: Optional[dict] = None,
+            unrestricted: bool = False,
+            non_destructive_node_explosion: bool = False,
+    ) -> list[list[str]]:
+        start_with = start_with or {}
+        if not (nodes := tsort.get_ready()):
+            return prev_layer
+        # You can only get access to results of previous layers unless unrestricted is requested.
+        prev_layer_results = self.results[run_id] if unrestricted else {
+            key: val
+            for key, val in self.results[run_id].items()
+            if key in prev_layer[-1]
+        }
+        curr_layer = []
+        for node in nodes:
+            parents = [
+                parent.name
+                for parent in self.predmap.get(node, [])
+            ]
+            accessible_results = prev_layer_results if unrestricted else {
+                key: val
+                for key, val in prev_layer_results.items()
+                if key in parents
+            }
+            kwargs = {
+                **start_with,
+                **{
+                    arg: val
+                    for node_key, node_out in accessible_results.items()
+                    for arg, val in node_out.items()
+                }
+            }
+            # Handle iterator mode; if '*' is provided in kwargs, execute node for each item in the iterator
+            # and collect results in a list.
+            # We should 'pop' to simulate 'consuming' the iterator input and avoid passing it to other nodes.
+            if (iterator := kwargs.pop("*", None)):
+                logger.debug(f"Executor mapping functionality detected: Node '{node.name}' executing in iterator mode.")
+                # TODO: https://github.com/fahera-mx/fred-oss/issues/179
+                # TODO: Can we consider exploiting the item components?
+                node_output = [
+                    node.execute(item)
+                    for item in iterator
+                ]
+            else:
+                node_output = node.execute(**kwargs)
+            # Execute node function
+            match node_output:
+                case Future() as future:
+                    # Can't we just build the whole graph in the future and 'wait_and_resolve' only at the end?
+                    # Or at least per layer/generation?
+                    output = {node.key: future.wait_and_resolve()}
+                case present:
+                    output = {node.key: present}
+            # We can only explode if  requested and the output result is a dict
+            if node._explode and isinstance(output.get(node.key), dict):
+                output = {
+                    # Keep original output if non-destrictive-explode is requested;
+                    # The original key can be overwritten if key collides during explosion.
+                    **(output if non_destructive_node_explosion else {}),
+                    # Explode keys into the output dict
+                    **output[node.key],
+                }
+            # Store output in results
+            self.results[run_id][node.name] = {
+                #**self.results[run_id].get(node.name, {}),
+                **output,
+            }
+            # Mark node as done
+            tsort.done(node)
+            curr_layer.append(node.name)
+        prev_layer.append(curr_layer)
+        return self.loop(
+            run_id=run_id,
+            tsort=tsort,
+            prev_layer=prev_layer,
+            unrestricted=unrestricted,
+            start_with={},  # Only availabe during the first layer call
+            non_destructive_node_explosion=non_destructive_node_explosion,
+        )
+
+    def execute(
+            self,
+            keep: bool = False,
+            unrestricted: bool = False,
+            start_with: Optional[dict] = None,
+            non_destructive_node_explosion: bool = False,
+        ) -> dict:
+        from fred.utils.dateops import datetime_utcnow
+
+        run_id = str(uuid.uuid4())
+        run_start = datetime_utcnow()
+        # Initialize in-memory result storage for this run
+        # TODO: Swap the result-store to our fred-keyval implementation
+        self.results[run_id] = {}
+        # Prepare TopologicalSorter
+        tsort = self.get_tsort()
+        tsort.prepare()
+        # Execute nodes in topological order
+        layers = self.loop(
+            run_id=run_id,
+            tsort=tsort,
+            prev_layer=[[]],
+            unrestricted=unrestricted,
+            start_with=start_with or {},
+            non_destructive_node_explosion=non_destructive_node_explosion,
+        )
+        return {
+            "run_id": run_id,
+            "run_start": run_start,
+            "run_end": datetime_utcnow(),
+            "results": self.results[run_id] if keep else self.results.pop(run_id),
+            "layers": layers,
+        }

fred_oss-0.64.0/src/main/fred/edag/plan.py

@@ -0,0 +1,66 @@
+from dataclasses import dataclass, field
+from typing import FrozenSet, Tuple, Union
+
+from fred.edag.comp._node import Node
+from fred.edag.comp._group import Group
+from fred.edag.comp.interface import ComponentInterface
+
+
+@dataclass(frozen=True, slots=True)
+class Plan:
+    nodes: list[Node] = field(default_factory=list)
+    edges: list[Tuple[Node, Node]] = field(default_factory=list)  # directed edges: (from/src, to/dst)
+    heads: list[Node] = field(default_factory=list)  # current entry set of chaining
+    tails: list[Node] = field(default_factory=list)  # current exit set of chaining
+
+    @staticmethod
+    def empty() -> 'Plan':
+        return Plan()
+
+    @classmethod
+    def as_plan(cls, other: Union[ComponentInterface, 'Plan']) -> 'Plan':
+        match other:
+            case Plan() as plan:
+                return plan
+            case Node():
+                return Plan(
+                    nodes=[other],
+                    edges=[],
+                    heads=[other],
+                    tails=[other],
+                )
+            case Group():
+                return Plan(
+                    nodes=[*other.nodes],
+                    edges=[],
+                    heads=[*other.nodes],
+                    tails=[*other.nodes],
+                )
+            case _:
+                raise TypeError("as_plan expects Node, Group, or Plan")
+
+    def __rshift__(self, other: Union[ComponentInterface, 'Plan']) -> "Plan":
+        other_plan = self.as_plan(other)
+        # combine edges, nodes, and link tails -> heads (i.e., connect current tails to new heads)
+        new_edges = [
+            *self.edges,
+            *other_plan.edges,
+            *(
+                (src, dst)
+                for src in self.tails
+                for dst in other_plan.heads
+            ),
+        ]
+        new_nodes = [*self.nodes, *other_plan.nodes]
+        return Plan(
+            nodes=new_nodes,
+            edges=new_edges,
+            heads=self.heads,
+            tails=other_plan.tails,
+        )
+
+    def as_predmap(self) -> dict[Node, set[Node]]:
+        predmap = {n: set() for n in self.nodes}  # Ensure all nodes are keys
+        for src, dst in self.edges:
+            predmap[dst].add(src)
+        return predmap

{fred_oss-0.62.0 → fred_oss-0.64.0}/src/main/fred/future/result.py

@@ -133,15 +133,15 @@ class FutureResult(Generic[A], FutureBackend.infer_backend()):
         return self._get_obj_key(future_id=self.future_id)
 
     def stringify(self) -> str:
-        import pickle
+        import dill
         import base64
-        return base64.b64encode(pickle.dumps(self)).decode("ascii")
+        return base64.b64encode(dill.dumps(self)).decode("ascii")
 
     @classmethod
     def from_string(cls, payload: str) -> 'FutureResult[A]':
-        import pickle
+        import dill
         import base64
-        return pickle.loads(base64.b64decode(payload))
+        return dill.loads(base64.b64decode(payload))
     
     def _from_backend(self) -> Optional['FutureResult[A]']:
         payload = self.obj.get()

{fred_oss-0.62.0 → fred_oss-0.64.0}/src/main/fred/rest/router/endpoint.py

@@ -31,6 +31,7 @@ class RouterEndpointConfig:
 class RouterEndpoint:
     function: Callable
     configs: RouterEndpointConfig
+    _og: Optional[Callable] = None  # Original function before any binding... this is mainly for testing purposes.
 
     def __call__(self, *args, **kwargs):
         return self.function(*args, **kwargs)
@@ -93,5 +94,6 @@ class RouterEndpointAnnotation(RouterEndpoint):
             return self.function(other_class, **params)
         return RouterEndpoint(
             function=closure,
-            configs=self.configs
+            configs=self.configs,
+            _og=self.function,
         )

fred_oss-0.64.0/src/main/fred/version

@@ -0,0 +1 @@
+0.64.0

{fred_oss-0.62.0 → fred_oss-0.64.0}/src/main/fred/worker/runner/rest/router/_runner.py

@@ -109,11 +109,19 @@ class RunnerRouterMixin(RouterInterfaceMixin):
         request.dispatch(
             request_queue=self.runner_backend.queue(f"req:{queue_slug}")
         )
+        # Starting the runner to process the request if requested; this should always be BEFORE placing the request in the queue
+        # to avoid race conditions where a blocking runner is spawned before the request is enqueued.
+        # TODO: Let's optimize this by identifying if there's already an on-going runner for the same spec.
+        # If so, we can potentially reuse it instead of starting a new one each time.
+        # TODO: Let's remove the '_og' reference and find a cleaner way to handle this...
+        runner_start_output = self.runner_start._og(self, **start_configs) \
+            if (start_configs := kwargs.pop("start_configs", {})) else {}
         return {
             "item_id": item.item_id,
             "request_id": request.request_id,
             "queue_slug": queue_slug,
             "dispatched_at": datetime_utcnow().isoformat(),
+            "runner_start_output": runner_start_output,
         }
 
     @RouterEndpointAnnotation.set(

{fred_oss-0.62.0 → fred_oss-0.64.0/src/main/fred_oss.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.62.0
+Version: 0.64.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development
@@ -10,6 +10,7 @@ Description-Content-Type: text/markdown
 License-File: NOTICE.txt
 Requires-Dist: fire==0.7.1
 Requires-Dist: psutil==7.0.0
+Requires-Dist: dill==0.4.0
 Requires-Dist: redis==6.4.0
 Requires-Dist: requests==2.32.5
 Requires-Dist: fastapi==0.116.2

{fred_oss-0.62.0 → fred_oss-0.64.0}/src/main/fred_oss.egg-info/SOURCES.txt

@@ -33,6 +33,17 @@ src/main/fred/dao/service/_minio/policy/catalog.py
 src/main/fred/dao/service/_minio/policy/loader.py
 src/main/fred/dao/service/_minio/policy/templates/public_ro.json
 src/main/fred/dao/service/_minio/policy/templates/public_rw.json
+src/main/fred/edag/__init__.py
+src/main/fred/edag/decorator.py
+src/main/fred/edag/executor.py
+src/main/fred/edag/plan.py
+src/main/fred/edag/comp/__init__.py
+src/main/fred/edag/comp/_group.py
+src/main/fred/edag/comp/_node.py
+src/main/fred/edag/comp/catalog.py
+src/main/fred/edag/comp/interface.py
+src/main/fred/edag/conn/__init__.py
+src/main/fred/edag/conn/catalog.py
 src/main/fred/future/__init__.py
 src/main/fred/future/impl.py
 src/main/fred/future/result.py

{fred_oss-0.62.0 → fred_oss-0.64.0}/src/main/fred_oss.egg-info/requires.txt

@@ -1,5 +1,6 @@
 fire==0.7.1
 psutil==7.0.0
+dill==0.4.0
 redis==6.4.0
 requests==2.32.5
 fastapi==0.116.2

fred_oss-0.62.0/src/main/fred/version

@@ -1 +0,0 @@
-0.62.0