dv-flow-mgr 0.0.1.14013608039a1__py3-none-any.whl → 0.0.1.14097297609a1__py3-none-any.whl

dv_flow/mgr/task_node_ctor.py

@@ -0,0 +1,118 @@
+#****************************************************************************
+#* task_node_ctor.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import enum
+import os
+import sys
+import dataclasses as dc
+import pydantic.dataclasses as pdc
+import logging
+import toposort
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_node import TaskNode
+from .param import Param
+
+@dc.dataclass
+class TaskNodeCtor(object):
+    """
+    Factory for a specific task type
+    - Produces a task parameters object, applying value-setting instructions
+    - Produces a TaskNode
+    """
+    name : str
+    srcdir : str
+    paramT : Any
+    passthrough : bool
+    consumes : List[Any]
+    needs : List[str]
+
+    def __call__(self, 
+                 builder=None,
+                 name=None,
+                 srcdir=None,
+                 params=None,
+                 needs=None,
+                 passthrough=None,
+                 consumes=None,
+                 **kwargs):
+        """Convenience method for direct creation of tasks"""
+        if params is None:
+            params = self.mkTaskParams(kwargs)
+        
+        node = self.mkTaskNode(
+            builder=builder,
+            srcdir=srcdir, 
+            params=params, 
+            name=name, 
+            needs=needs)
+        if passthrough is not None:
+            node.passthrough = passthrough
+        else:
+            node.passthrough = self.passthrough
+        if consumes is not None:
+            if node.consumes is None:
+                node.consumes = consumes
+            else:
+                node.consumes.extend(consumes)
+        else:
+            if node.consumes is None:
+                node.consumes = self.consumes
+
+        return node
+
+    def getNeeds(self) -> List[str]:
+        return []
+
+    def mkTaskNode(self,
+                   builder,
+                   params,
+                   srcdir=None,
+                   name=None,
+                   needs=None) -> TaskNode:
+        raise NotImplementedError("mkTaskNode in type %s" % str(type(self)))
+
+    def mkTaskParams(self, params : Dict = None) -> Any:
+        obj = self.paramT()
+
+        # Apply user-specified params
+        if params is not None:
+            for key,value in params.items():
+                if not hasattr(obj, key):
+                    raise Exception("Parameters class %s does not contain field %s" % (
+                        str(type(obj)),
+                        key))
+                else:
+                    if isinstance(value, Param):
+                        if value.append is not None:
+                            ex_value = getattr(obj, key, [])
+                            ex_value.extend(value.append)
+                            setattr(obj, key, ex_value)
+                        elif value.prepend is not None:
+                            ex_value = getattr(obj, key, [])
+                            value = value.copy()
+                            value.extend(ex_value)
+                            setattr(obj, key, value)
+                            pass
+                        else:
+                            raise Exception("Unhandled value spec: %s" % str(value))
+                    else:
+                        setattr(obj, key, value)
+        return obj

dv_flow/mgr/task_node_ctor_compound.py

@@ -0,0 +1,117 @@
+#****************************************************************************
+#* task_node_ctor_compound.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import os
+import json
+import dataclasses as dc
+import logging
+from pydantic import BaseModel
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_def import TaskDef
+from .task_data import TaskDataOutput, TaskDataResult
+from .task_node import TaskNode
+from .task_node_ctor import TaskNodeCtor
+from .task_node_compound import TaskNodeCompound
+
+@dc.dataclass
+class TaskNodeCtorCompound(TaskNodeCtor):
+    task_def : TaskDef
+    tasks : List[TaskNodeCtor] = dc.field(default_factory=list)
+
+    _log : ClassVar = logging.getLogger("TaskNodeCtorCompound")
+
+    def mkTaskNode(self, builder, params, srcdir=None, name=None, needs=None) -> 'TaskNode':
+        """Creates a task object without a base task"""
+        self._log.debug("--> mkTaskNode %s (%d)" % (name, len(self.tasks)))
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        node = TaskNodeCompound(
+            name=name, 
+            srcdir=srcdir,
+            params=params,
+            needs=needs)
+
+
+        builder.enter_compound(node)
+        builder.addTask("in", node.input)
+
+        self._buildSubGraph(builder, node)
+
+        builder.leave_compound(node)
+
+        self._log.debug("<-- mkTaskNode %s (%d)" % (name, len(node.needs)))
+        return node
+
+    def _buildSubGraph(self, builder, node):
+        nodes = []
+
+        for t in self.tasks:
+            # Need to get the parent name
+            needs = []
+            for n in t.needs:
+                need_name = "%s.%s" % (builder.package().name, n)
+                task = builder.findTask(n)
+                if task is None:
+                    task = builder.findTask(need_name)
+
+                if task is None:
+                    raise Exception("Failed to find task %s (%s)" % (n, need_name))
+                self._log.debug("Add %s as dependency of %s" % (
+                    task.name, t.name
+                ))
+                needs.append(task)
+            sn = t.mkTaskNode(
+                builder=builder, 
+                params=t.mkTaskParams(),
+                name=t.name,
+                needs=needs)
+            nodes.append(sn)
+            builder.addTask(t.name, sn)
+        in_t = builder.findTask("in")
+
+        
+        for n in nodes:
+
+            # If this node references one of the others, then 
+            # it takes input from that node, and not the 'in' node
+            has_ref = False
+            for nt in n.needs:
+                self._log.debug("nt: %s %s" % (nt[0].name, str(n.needs)))
+                if nt[0] in nodes or nt[0] is in_t:
+                    has_ref = True
+                    break
+            if not has_ref:
+                n.needs.append([builder.findTask("in"), False])
+
+            # Only add a dependency on the node if no other node references it
+            is_ref = False
+            for nt in nodes:
+                for nn in nt.needs:
+                    if nn[0] == n:
+                        is_ref = True
+                        break
+            if not is_ref:
+                node.needs.append([n, False])
+        
+        self._log.debug("nodes: %d (%d %d)" % (len(nodes), len(self.tasks), len(node.needs)))
+
+        pass

dv_flow/mgr/task_node_ctor_compound_proxy.py

@@ -0,0 +1,65 @@
+import os
+import json
+import dataclasses as dc
+import logging
+from pydantic import BaseModel
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_data import TaskDataOutput, TaskDataResult
+from .task_node import TaskNode
+from .task_node_compound import TaskNodeCompound
+from .task_node_ctor import TaskNodeCtor
+from .task_node_ctor_compound import TaskNodeCtorCompound
+
+# TaskParamsCtor accepts an evaluation context and returns a task parameter object
+TaskParamsCtor = Callable[[object], Any]
+
+@dc.dataclass
+class TaskNodeCtorCompoundProxy(TaskNodeCtorCompound):
+    """Task has a 'uses' clause, so we delegate creation of the node"""
+    uses : TaskNodeCtor = dc.field(default=None)
+
+    _log : ClassVar = logging.getLogger("TaskNodeCtorCompoundProxy")
+
+    def mkTaskNode(self, builder, params, srcdir=None, name=None, needs=None) -> 'TaskNode':
+        """Creates a task object without a base task"""
+        self._log.debug("--> mkTaskNode: %s", name)
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        is_compound_uses = builder.is_compound_uses()
+
+        if not is_compound_uses:
+            # We're at the leaf level
+            node = TaskNodeCompound(
+                name=name, 
+                srcdir=srcdir,
+                params=params)
+
+            builder.enter_compound(node)
+            builder.addTask("in", node.input)
+        else:
+            node = None
+
+        builder.enter_compound_uses()
+
+        # Construct the base-task node
+        base_node = self.uses.mkTaskNode(
+            builder=builder, 
+            params=params, 
+            srcdir=srcdir, 
+            name=name + ".super", 
+            needs=[node.input])
+        builder.addTask("super", base_node)
+
+        # Build 'uses' node
+        self._buildSubGraph(builder, node)
+
+        builder.leave_compound_uses()
+
+        if not is_compound_uses:
+            builder.leave_compound(node)
+
+
+        self._log.debug("<-- mkTaskNode: %s", name)
+        return node
+    

dv_flow/mgr/task_node_ctor_def_base.py

@@ -0,0 +1,47 @@
+#****************************************************************************
+#* task_node_ctor_def_base.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import enum
+import os
+import sys
+import dataclasses as dc
+import pydantic.dataclasses as pdc
+import logging
+import toposort
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_params_ctor import TaskParamsCtor
+from .param_ref_eval import ParamRefEval
+from .param import Param
+from .task_node_ctor import TaskNodeCtor
+
+@dc.dataclass
+class TaskNodeCtorDefBase(TaskNodeCtor):
+    """Task defines its own needs, that will need to be filled in"""
+    needs : List['str']
+
+    def __post_init__(self):
+        if self.needs is None:
+            self.needs = []
+
+    def getNeeds(self) -> List[str]:
+        return self.needs
+

dv_flow/mgr/task_node_ctor_proxy.py

@@ -0,0 +1,56 @@
+#****************************************************************************
+#* task_node_ctor_def_base.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import enum
+import os
+import sys
+import dataclasses as dc
+import pydantic.dataclasses as pdc
+import logging
+import toposort
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_params_ctor import TaskParamsCtor
+from .param_ref_eval import ParamRefEval
+from .param import Param
+from .task_node import TaskNode
+from .task_node_ctor import TaskNodeCtor
+from .task_node_ctor_def_base import TaskNodeCtorDefBase
+
+@dc.dataclass
+class TaskNodeCtorProxy(TaskNodeCtorDefBase):
+    """Task has a 'uses' clause, so we delegate creation of the node"""
+    uses : TaskNodeCtor
+
+    def mkTaskNode(self, builder, params, srcdir=None, name=None, needs=None) -> TaskNode:
+        if srcdir is None:
+            srcdir = self.srcdir
+        builder.enter_uses()
+        node = self.uses.mkTaskNode(
+            builder=builder, params=params, srcdir=srcdir, name=name, needs=needs)
+        node.passthrough = self.passthrough
+        node.consumes = self.consumes
+        builder.leave_uses()
+
+        if not builder.in_uses():
+            builder.addTask(name, node)
+        return node
+    

dv_flow/mgr/task_node_ctor_task.py

@@ -0,0 +1,64 @@
+#****************************************************************************
+#* task_node_ctor_task.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import enum
+import os
+import sys
+import dataclasses as dc
+import pydantic.dataclasses as pdc
+import logging
+import toposort
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_params_ctor import TaskParamsCtor
+from .param_ref_eval import ParamRefEval
+from .param import Param
+from .task_node import TaskNode
+from .task_node_leaf import TaskNodeLeaf
+from .task_node_ctor import TaskNodeCtor
+from .task_node_ctor_def_base import TaskNodeCtorDefBase
+
+@dc.dataclass
+class TaskNodeCtorTask(TaskNodeCtorDefBase):
+    task : Callable[['TaskRunner','TaskDataInput'],'TaskDataResult']
+
+    _log : ClassVar[logging.Logger] = logging.getLogger("TaskNodeCtorTask")
+
+    def mkTaskNode(self, builder, params, srcdir=None, name=None, needs=None) -> TaskNode:
+        self._log.debug("--> mkTaskNode needs=%d" % (
+            (len(needs) if needs is not None else -1),
+        ))
+        if srcdir is None:
+            srcdir = self.srcdir
+
+        node = TaskNodeLeaf(
+            name=name, 
+            srcdir=srcdir, 
+            params=params, 
+            task=self.task,
+            needs=needs)
+        node.passthrough = self.passthrough
+        node.consumes = self.consumes
+        node.task = self.task
+        builder.addTask(name, node)
+
+        self._log.debug("<-- mkTaskNode")
+        return node

dv_flow/mgr/task_node_ctor_wrapper.py

@@ -0,0 +1,96 @@
+#****************************************************************************
+#* task_node_ctor_task.py
+#*
+#* Copyright 2023-2025 Matthew Ballance and Contributors
+#*
+#* Licensed under the Apache License, Version 2.0 (the "License"); you may 
+#* not use this file except in compliance with the License.  
+#* You may obtain a copy of the License at:
+#*  
+#*   http://www.apache.org/licenses/LICENSE-2.0
+#*  
+#* Unless required by applicable law or agreed to in writing, software 
+#* distributed under the License is distributed on an "AS IS" BASIS, 
+#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  
+#* See the License for the specific language governing permissions and 
+#* limitations under the License.
+#*
+#* Created on:
+#*     Author: 
+#*
+#****************************************************************************
+import enum
+import os
+import sys
+import dataclasses as dc
+import pydantic.dataclasses as pdc
+import logging
+import toposort
+from typing import Any, Callable, ClassVar, Dict, List, Tuple
+from .task_data import TaskDataInput, TaskDataOutput, TaskDataResult
+from .task_params_ctor import TaskParamsCtor
+from .param_ref_eval import ParamRefEval
+from .param import Param
+from .task_node import TaskNode
+from .task_node_leaf import TaskNodeLeaf
+from .task_node_ctor import TaskNodeCtor
+from .task_node_ctor_def_base import TaskNodeCtorDefBase
+
+@dc.dataclass
+class TaskNodeCtorWrapper(TaskNodeCtor):
+    T : Any
+
+    def mkTaskNode(self, builder, params, srcdir=None, name=None, needs=None) -> TaskNode:
+        node = TaskNodeLeaf(
+            name=name, 
+            srcdir=srcdir, 
+            params=params, 
+            task=self.T, 
+            needs=needs)
+        node.passthrough = self.passthrough
+        node.consumes = self.consumes
+        return node
+
+    def mkTaskParams(self, params : Dict = None) -> Any:
+        obj = self.paramT()
+
+        # Apply user-specified params
+        for key,value in params.items():
+            if not hasattr(obj, key):
+                raise Exception("Parameters class %s does not contain field %s" % (
+                    str(type(obj)),
+                    key))
+            else:
+                if isinstance(value, Param):
+                    if value.append is not None:
+                        ex_value = getattr(obj, key, [])
+                        ex_value.extend(value.append)
+                        setattr(obj, key, ex_value)
+                    elif value.prepend is not None:
+                        ex_value = getattr(obj, key, [])
+                        value = value.copy()
+                        value.extend(ex_value)
+                        setattr(obj, key, value)
+                        pass
+                    else:
+                        raise Exception("Unhandled value spec: %s" % str(value))
+                else:
+                    setattr(obj, key, value)
+        return obj
+    
+def task(paramT,passthrough=False,consumes=None):
+    """Decorator to wrap a task method as a TaskNodeCtor"""
+    def wrapper(T):
+        task_mname = T.__module__
+        task_module = sys.modules[task_mname]
+        ctor = TaskNodeCtorWrapper(
+            name=T.__name__, 
+            srcdir=os.path.dirname(os.path.abspath(task_module.__file__)), 
+            paramT=paramT,
+            passthrough=passthrough,
+            consumes=consumes,
+            needs=[],
+            T=T)
+        return ctor
+    return wrapper
+