runnable 0.12.3__tar.gz → 0.13.0__tar.gz

{runnable-0.12.3 → runnable-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runnable
-Version: 0.12.3
+Version: 0.13.0
 Summary: A Compute agnostic pipelining software
 Home-page: https://github.com/vijayvammi/runnable
 License: Apache-2.0

{runnable-0.12.3 → runnable-0.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "runnable"
-version = "0.12.3"
+version = "0.13.0"
 description = "A Compute agnostic pipelining software"
 authors = ["Vijay Vammi <mesanthu@gmail.com>"]
 license = "Apache-2.0"
@@ -82,7 +82,7 @@ pyflame = "^0.3.1"
 
 # Plugins for Executors
 [tool.poetry.plugins."executor"]
-"local" = "runnable.extensions.executor.local.implementation:LocalExecutor"
+"local" = "runnable.extensions.executor.local:LocalExecutor"
 "local-container" = "runnable.extensions.executor.local_container.implementation:LocalContainerExecutor"
 "argo" = "runnable.extensions.executor.argo.implementation:ArgoExecutor"
 "mocked" = "runnable.extensions.executor.mocked.implementation:MockedExecutor"

runnable-0.12.3/runnable/extensions/executor/local/implementation.py → runnable-0.13.0/runnable/extensions/executor/local.py

@@ -19,8 +19,6 @@ class LocalExecutor(GenericExecutor):
     Example config:
     execution:
       type: local
-      config:
-        enable_parallel: True or False to enable parallel.
 
     """
 

{runnable-0.12.3 → runnable-0.13.0}/runnable/extensions/nodes.py

@@ -797,12 +797,27 @@ class DagNode(CompositeNode):
 class StubNode(ExecutableNode):
     """
     Stub is a convenience design node.
-
     It always returns success in the attempt log and does nothing.
 
     This node is very similar to pass state in Step functions.
 
     This node type could be handy when designing the pipeline and stubbing functions
+    --8<-- [start:stub_reference]
+    An stub execution node of the pipeline.
+    Please refer to define pipeline/tasks/stub for more information.
+
+    As part of the dag definition, a stub task is defined as follows:
+
+    dag:
+      steps:
+        stub_task: # The name of the node
+        type: stub
+        on_failure: The name of the step to traverse in case of failure
+        next: The next node to execute after this task, use "success" to terminate the pipeline successfully
+          or "fail" to terminate the pipeline with an error.
+
+    It can take arbritary number of parameters, which is handy to temporarily silence a task node.
+    --8<-- [end:stub_reference]
     """
 
     node_type: str = Field(default="stub", serialization_alias="type")

{runnable-0.12.3 → runnable-0.13.0}/runnable/tasks.py

@@ -188,7 +188,56 @@ def task_return_to_parameter(task_return: TaskReturns, value: Any) -> Parameter:
 
 
 class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
-    """The task class for python command."""
+    """
+    --8<-- [start:python_reference]
+    An execution node of the pipeline of python functions.
+    Please refer to define pipeline/tasks/python for more information.
+
+    As part of the dag definition, a python task is defined as follows:
+
+    dag:
+      steps:
+        python_task: # The name of the node
+          type: task
+          command_type: python # this is default
+          command: my_module.my_function # the dotted path to the function. Please refer to the yaml section of
+            define pipeline/tasks/python for concrete details.
+          returns:
+            - name: # The name to assign the return value
+              kind: json # the default value is json,
+                can be object for python objects and metric for metrics
+          secrets:
+            - my_secret_key # A list of secrets to expose by secrets manager
+          catalog:
+            get:
+              - A list of glob patterns to get from the catalog to the local file system
+            put:
+              - A list of glob patterns to put to the catalog from the local file system
+          on_failure: The name of the step to traverse in case of failure
+          overrides:
+            Individual tasks can override the global configuration config by referring to the
+            specific override.
+
+            For example,
+            #Global configuration
+            executor:
+            type: local-container
+            config:
+              docker_image: "runnable/runnable:latest"
+              overrides:
+              custom_docker_image:
+                docker_image: "runnable/runnable:custom"
+
+            ## In the node definition
+            overrides:
+            local-container:
+              docker_image: "runnable/runnable:custom"
+
+            This instruction will override the docker image for the local-container executor.
+          next: The next node to execute after this task, use "success" to terminate the pipeline successfully
+            or "fail" to terminate the pipeline with an error.
+    --8<-- [end:python_reference]
+    """
 
     task_type: str = Field(default="python", serialization_alias="command_type")
     command: str
@@ -277,7 +326,56 @@ class PythonTaskType(BaseTaskType):  # pylint: disable=too-few-public-methods
 
 
 class NotebookTaskType(BaseTaskType):
-    """The task class for Notebook based execution."""
+    """
+    --8<-- [start:notebook_reference]
+    An execution node of the pipeline of notebook execution.
+    Please refer to define pipeline/tasks/notebook for more information.
+
+    As part of the dag definition, a notebook task is defined as follows:
+
+    dag:
+      steps:
+        notebook_task: # The name of the node
+          type: task
+          command_type: notebook
+          command: the path to the notebook relative to project root.
+          optional_ploomber_args: a dictionary of arguments to be passed to ploomber engine
+          returns:
+            - name: # The name to assign the return value
+              kind: json # the default value is json,
+                can be object for python objects and metric for metrics
+          secrets:
+            - my_secret_key # A list of secrets to expose by secrets manager
+          catalog:
+            get:
+              - A list of glob patterns to get from the catalog to the local file system
+            put:
+              - A list of glob patterns to put to the catalog from the local file system
+          on_failure: The name of the step to traverse in case of failure
+          overrides:
+            Individual tasks can override the global configuration config by referring to the
+            specific override.
+
+            For example,
+            #Global configuration
+            executor:
+            type: local-container
+            config:
+              docker_image: "runnable/runnable:latest"
+              overrides:
+                custom_docker_image:
+                  docker_image: "runnable/runnable:custom"
+
+            ## In the node definition
+            overrides:
+              local-container:
+                docker_image: "runnable/runnable:custom"
+
+            This instruction will override the docker image for the local-container executor.
+          next: The next node to execute after this task, use "success" to terminate the pipeline successfully
+            or "fail" to terminate the pipeline with an error.
+    --8<-- [end:notebook_reference]
+    """
 
     task_type: str = Field(default="notebook", serialization_alias="command_type")
     command: str
@@ -410,7 +508,54 @@ class NotebookTaskType(BaseTaskType):
 
 class ShellTaskType(BaseTaskType):
     """
-    The task class for shell based commands.
+    --8<-- [start:shell_reference]
+    An execution node of the pipeline of shell execution.
+    Please refer to define pipeline/tasks/shell for more information.
+
+    As part of the dag definition, a shell task is defined as follows:
+
+    dag:
+      steps:
+        shell_task: # The name of the node
+          type: task
+          command_type: shell
+          command: The command to execute, it could be multiline
+          optional_ploomber_args: a dictionary of arguments to be passed to ploomber engine
+          returns:
+            - name: # The name to assign the return value
+            kind: json # the default value is json,
+                can be object for python objects and metric for metrics
+          secrets:
+            - my_secret_key # A list of secrets to expose by secrets manager
+          catalog:
+            get:
+              - A list of glob patterns to get from the catalog to the local file system
+            put:
+              - A list of glob patterns to put to the catalog from the local file system
+          on_failure: The name of the step to traverse in case of failure
+          overrides:
+            Individual tasks can override the global configuration config by referring to the
+            specific override.
+
+            For example,
+            #Global configuration
+            executor:
+            type: local-container
+            config:
+              docker_image: "runnable/runnable:latest"
+              overrides:
+                custom_docker_image:
+                  docker_image: "runnable/runnable:custom"
+
+            ## In the node definition
+            overrides:
+              local-container:
+                docker_image: "runnable/runnable:custom"
+
+            This instruction will override the docker image for the local-container executor.
+          next: The next node to execute after this task, use "success" to terminate the pipeline successfully
+            or "fail" to terminate the pipeline with an error.
+    --8<-- [end:shell_reference]
     """
 
     task_type: str = Field(default="shell", serialization_alias="command_type")