runnable 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

runnable/extensions/executor/argo/implementation.py

@@ -292,7 +292,7 @@ class ContainerTemplate(BaseModel):
 
 class DagTemplate(BaseModel):
     # These are used for parallel, map nodes dag definition
-    name: str = "magnus-dag"
+    name: str = "runnable-dag"
     tasks: List[DagTaskTemplate] = Field(default=[], exclude=True)
     inputs: Optional[List[Parameter]] = Field(default=None, serialization_alias="inputs")
     parallelism: Optional[int] = None
@@ -561,7 +561,7 @@ def get_renderer(node):
 
 
 class MetaData(BaseModel):
-    generate_name: str = Field(default="magnus-dag-", serialization_alias="generateName")
+    generate_name: str = Field(default="runnable-dag-", serialization_alias="generateName")
     annotations: Optional[Dict[str, str]] = Field(default_factory=dict)
     labels: Optional[Dict[str, str]] = Field(default_factory=dict)
     namespace: Optional[str] = Field(default=None)
@@ -569,7 +569,7 @@ class MetaData(BaseModel):
 
 class Spec(BaseModel):
     active_deadline_seconds: int = Field(serialization_alias="activeDeadlineSeconds")
-    entrypoint: str = Field(default="magnus-dag")
+    entrypoint: str = Field(default="runnable-dag")
     node_selector: Optional[Dict[str, str]] = Field(default_factory=dict, serialization_alias="nodeSelector")
     tolerations: Optional[List[Toleration]] = Field(default=None, serialization_alias="tolerations")
     parallelism: Optional[int] = Field(default=None, serialization_alias="parallelism")
@@ -665,6 +665,7 @@ class Override(BaseModel):
 
 class ArgoExecutor(GenericExecutor):
     service_name: str = "argo"
+    _local: bool = False
 
     model_config = ConfigDict(extra="forbid")
 
@@ -674,7 +675,7 @@ class ArgoExecutor(GenericExecutor):
     output_file: str = "argo-pipeline.yaml"
 
     # Metadata related fields
-    name: str = Field(default="magnus-dag-", description="Used as an identifier for the workflow")
+    name: str = Field(default="runnable-dag-", description="Used as an identifier for the workflow")
     annotations: Dict[str, str] = Field(default_factory=dict)
     labels: Dict[str, str] = Field(default_factory=dict)
 
@@ -994,7 +995,7 @@ class ArgoExecutor(GenericExecutor):
         return DagTaskTemplate(name=f"{clean_name}-fan-in", template=f"{clean_name}-fan-in")
 
     def _gather_task_templates_of_dag(
-        self, dag: Graph, dag_name="magnus-dag", list_of_iter_values: Optional[List] = None
+        self, dag: Graph, dag_name="runnable-dag", list_of_iter_values: Optional[List] = None
     ):
         current_node = dag.start_at
         previous_node = None

runnable/extensions/executor/argo/specification.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
-  generateName: magnus-dag
+  generateName: runnable-dag
 spec:
   activeDeadlineSeconds: int # max run time of the workflow
   entrypoint: str

runnable/extensions/executor/k8s_job/implementation_FF.py

@@ -4,10 +4,10 @@
 
 # from pydantic import BaseModel
 
-# from magnus import defaults, integration, utils
-# from magnus.executor import BaseExecutor
-# from magnus.graph import Graph
-# from magnus.nodes import BaseNode
+# from runnable import defaults, integration, utils
+# from runnable.executor import BaseExecutor
+# from runnable.graph import Graph
+# from runnable.nodes import BaseNode
 
 # logger = logging.getLogger(defaults.NAME)
 

runnable/extensions/executor/local/implementation.py

@@ -25,6 +25,7 @@ class LocalExecutor(GenericExecutor):
     """
 
     service_name: str = "local"
+    _local: bool = True
 
     def trigger_job(self, node: BaseNode, map_variable: TypeMapVariable = None, **kwargs):
         """

runnable/extensions/executor/local_container/implementation.py

@@ -55,6 +55,8 @@ class LocalContainerExecutor(GenericExecutor):
     run_in_local: bool = False
     environment: Dict[str, str] = Field(default_factory=dict)
 
+    _local: bool = False
+
     _container_log_location = "/tmp/run_logs/"
     _container_catalog_location = "/tmp/catalog/"
     _container_secrets_location = "/tmp/dotenv"
@@ -131,7 +133,7 @@ class LocalContainerExecutor(GenericExecutor):
 
 
         If the config has "run_in_local: True", we compute it on local system instead of container.
-        In local container execution, we just spin the container to execute magnus execute_single_node.
+        In local container execution, we just spin the container to execute runnable execute_single_node.
 
         Args:
             node (BaseNode): The node we are currently executing
@@ -198,6 +200,7 @@ class LocalContainerExecutor(GenericExecutor):
 
         try:
             logger.info(f"Running the command {command}")
+            print(command)
             #  Overrides global config with local
             executor_config = self._resolve_executor_config(node)
 
@@ -256,15 +259,6 @@ class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
     service_type = "run_log_store"  # One of secret, catalog, datastore
     service_provider = "file-system"  # The actual implementation of the service
 
-    def validate(self, **kwargs):
-        if self.executor._is_parallel_execution():  # pragma: no branch
-            msg = (
-                "Run log generated by file-system run log store are not thread safe. "
-                "Inconsistent results are possible because of race conditions to write to the same file.\n"
-                "Consider using partitioned run log store like database for consistent results."
-            )
-            logger.warning(msg)
-
     def configure_for_traversal(self, **kwargs):
         from runnable.extensions.run_log_store.file_system.implementation import FileSystemRunLogstore
 

runnable/extensions/executor/mocked/implementation.py

@@ -8,7 +8,6 @@ from runnable import context, defaults
 from runnable.defaults import TypeMapVariable
 from runnable.extensions.executor import GenericExecutor
 from runnable.extensions.nodes import TaskNode
-from runnable.integration import BaseIntegration
 from runnable.nodes import BaseNode
 from runnable.tasks import BaseTaskType
 
@@ -25,8 +24,7 @@ def create_executable(params: Dict[str, Any], model: Type[BaseTaskType], node_na
 
 class MockedExecutor(GenericExecutor):
     service_name: str = "mocked"
-
-    enable_parallel: bool = defaults.ENABLE_PARALLEL
+    _local_executor: bool = True
 
     patches: Dict[str, Any] = Field(default_factory=dict)
 
@@ -119,6 +117,7 @@ class MockedExecutor(GenericExecutor):
         self.prepare_for_node_execution()
         self.execute_node(node=node, map_variable=map_variable, **kwargs)
 
+    # TODO: This needs to go away
     def _is_step_eligible_for_rerun(self, node: BaseNode, map_variable: TypeMapVariable = None):
         """
         In case of a re-run, this method checks to see if the previous run step status to determine if a re-run is
@@ -188,33 +187,3 @@ class MockedExecutor(GenericExecutor):
             map_variable (dict[str, str], optional): _description_. Defaults to None.
         """
         self._execute_node(node=node, map_variable=map_variable, **kwargs)
-
-
-class LocalContainerComputeFileSystemRunLogstore(BaseIntegration):
-    """
-    Integration between local container and file system run log store
-    """
-
-    executor_type = "local-container"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "file-system"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        if self.executor._is_parallel_execution():  # pragma: no branch
-            msg = "Mocked executor does not support parallel execution. "
-            logger.warning(msg)
-
-
-class LocalContainerComputeChunkedFSRunLogstore(BaseIntegration):
-    """
-    Integration between local container and file system run log store
-    """
-
-    executor_type = "local-container"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "chunked-fs"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        if self.executor._is_parallel_execution():  # pragma: no branch
-            msg = "Mocked executor does not support parallel execution. "
-            logger.warning(msg)

runnable/extensions/nodes.py

@@ -1,10 +1,9 @@
-import json
+import copy
 import logging
-import multiprocessing
 from collections import OrderedDict
 from copy import deepcopy
 from datetime import datetime
-from typing import Any, Dict, cast
+from typing import Any, Dict, Optional, cast
 
 from pydantic import ConfigDict, Field, ValidationInfo, field_serializer, field_validator
 from typing_extensions import Annotated
@@ -44,9 +43,15 @@ class TaskNode(ExecutableNode):
         executable = create_task(task_config)
         return cls(executable=executable, **node_config, **task_config)
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
-        All that we do in magnus is to come to this point where we actually execute the command.
+        All that we do in runnable is to come to this point where we actually execute the command.
 
         Args:
             executor (_type_): The executor class
@@ -62,9 +67,11 @@ class TaskNode(ExecutableNode):
         try:
             attempt_log.start_time = str(datetime.now())
             attempt_log.status = defaults.SUCCESS
+            attempt_log.input_parameters = copy.deepcopy(params)
             if not mock:
                 # Do not run if we are mocking the execution, could be useful for caching and dry runs
-                self.executable.execute_command(map_variable=map_variable)
+                output_parameters = self.executable.execute_command(map_variable=map_variable, params=params)
+                attempt_log.output_parameters = output_parameters
         except Exception as _e:  # pylint: disable=W0703
             logger.exception("Task failed")
             attempt_log.status = defaults.FAIL
@@ -88,7 +95,13 @@ class FailNode(TerminalNode):
     def parse_from_config(cls, config: Dict[str, Any]) -> "FailNode":
         return cast("FailNode", super().parse_from_config(config))
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         Execute the failure node.
         Set the run or branch log status to failure.
@@ -105,6 +118,7 @@ class FailNode(TerminalNode):
         try:
             attempt_log.start_time = str(datetime.now())
             attempt_log.status = defaults.SUCCESS
+            attempt_log.input_parameters = params
             #  could be a branch or run log
             run_or_branch_log = self._context.run_log_store.get_branch_log(
                 self._get_branch_log_name(map_variable), self._context.run_id
@@ -133,7 +147,13 @@ class SuccessNode(TerminalNode):
     def parse_from_config(cls, config: Dict[str, Any]) -> "SuccessNode":
         return cast("SuccessNode", super().parse_from_config(config))
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         Execute the success node.
         Set the run or branch log status to success.
@@ -150,6 +170,7 @@ class SuccessNode(TerminalNode):
         try:
             attempt_log.start_time = str(datetime.now())
             attempt_log.status = defaults.SUCCESS
+            attempt_log.input_parameters = params
             #  could be a branch or run log
             run_or_branch_log = self._context.run_log_store.get_branch_log(
                 self._get_branch_log_name(map_variable), self._context.run_id
@@ -257,35 +278,11 @@ class ParallelNode(CompositeNode):
             executor (Executor): The Executor as per the use config
             **kwargs: Optional kwargs passed around
         """
-        from runnable import entrypoints
 
         self.fan_out(map_variable=map_variable, **kwargs)
 
-        jobs = []
-        # Given that we can have nesting and complex graphs, controlling the number of processes is hard.
-        # A better way is to actually submit the job to some process scheduler which does resource management
-        for internal_branch_name, branch in self.branches.items():
-            if self._context.executor._is_parallel_execution():
-                # Trigger parallel jobs
-                action = entrypoints.execute_single_brach
-                kwargs = {
-                    "configuration_file": self._context.configuration_file,
-                    "pipeline_file": self._context.pipeline_file,
-                    "branch_name": internal_branch_name.replace(" ", defaults.COMMAND_FRIENDLY_CHARACTER),
-                    "run_id": self._context.run_id,
-                    "map_variable": json.dumps(map_variable),
-                    "tag": self._context.tag,
-                }
-                process = multiprocessing.Process(target=action, kwargs=kwargs)
-                jobs.append(process)
-                process.start()
-
-            else:
-                # If parallel is not enabled, execute them sequentially
-                self._context.executor.execute_graph(branch, map_variable=map_variable, **kwargs)
-
-        for job in jobs:
-            job.join()  # Find status of the branches
+        for _, branch in self.branches.items():
+            self._context.executor.execute_graph(branch, map_variable=map_variable, **kwargs)
 
         self.fan_in(map_variable=map_variable, **kwargs)
 
@@ -418,7 +415,6 @@ class MapNode(CompositeNode):
             map_variable (dict): The map variables the graph belongs to
             **kwargs: Optional kwargs passed around
         """
-        from runnable import entrypoints
 
         iterate_on = None
         try:
@@ -433,34 +429,11 @@ class MapNode(CompositeNode):
 
         self.fan_out(map_variable=map_variable, **kwargs)
 
-        jobs = []
-        # Given that we can have nesting and complex graphs, controlling the number of processess is hard.
-        # A better way is to actually submit the job to some process scheduler which does resource management
         for iter_variable in iterate_on:
             effective_map_variable = map_variable or OrderedDict()
             effective_map_variable[self.iterate_as] = iter_variable
 
-            if self._context.executor._is_parallel_execution():
-                # Trigger parallel jobs
-                action = entrypoints.execute_single_brach
-                kwargs = {
-                    "configuration_file": self._context.configuration_file,
-                    "pipeline_file": self._context.pipeline_file,
-                    "branch_name": self.branch.internal_branch_name.replace(" ", defaults.COMMAND_FRIENDLY_CHARACTER),
-                    "run_id": self._context.run_id,
-                    "map_variable": json.dumps(effective_map_variable),
-                    "tag": self._context.tag,
-                }
-                process = multiprocessing.Process(target=action, kwargs=kwargs)
-                jobs.append(process)
-                process.start()
-
-            else:
-                # If parallel is not enabled, execute them sequentially
-                self._context.executor.execute_graph(self.branch, map_variable=effective_map_variable, **kwargs)
-
-        for job in jobs:
-            job.join()
+            self._context.executor.execute_graph(self.branch, map_variable=effective_map_variable, **kwargs)
 
         self.fan_in(map_variable=map_variable, **kwargs)
 
@@ -652,7 +625,13 @@ class StubNode(ExecutableNode):
     def parse_from_config(cls, config: Dict[str, Any]) -> "StubNode":
         return cls(**config)
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         Do Nothing node.
         We just send an success attempt log back to the caller
@@ -666,6 +645,7 @@ class StubNode(ExecutableNode):
             [type]: [description]
         """
         attempt_log = self._context.run_log_store.create_attempt_log()
+        attempt_log.input_parameters = params
 
         attempt_log.start_time = str(datetime.now())
         attempt_log.status = defaults.SUCCESS  # This is a dummy node and always will be success

runnable/integration.py

@@ -84,7 +84,7 @@ def get_integration_handler(executor: "BaseExecutor", service: object) -> BaseIn
             logger.info(f"Identified an integration pattern {kls.obj}")
             integrations.append(kls.obj)
 
-    # Get all the implementations defined by the magnus package
+    # Get all the implementations defined by the runnable package
     for kls in BaseIntegration.__subclasses__():
         # Match the exact service type
         if kls.service_type == service_type and kls.service_provider == service_name:
@@ -95,7 +95,7 @@ def get_integration_handler(executor: "BaseExecutor", service: object) -> BaseIn
     if len(integrations) > 1:
         msg = (
             f"Multiple integrations between {executor.service_name} and {service_name} of type {service_type} found. "
-            "If you defined an integration pattern, please ensure it is specific and does not conflict with magnus "
+            "If you defined an integration pattern, please ensure it is specific and does not conflict with runnable "
             " implementations."
         )
         logger.exception(msg)

runnable/interaction.py

@@ -58,6 +58,11 @@ def track_this(step: int = 0, **kwargs):
         os.environ[prefix + key + f"{defaults.STEP_INDICATOR}{step}"] = json.dumps(value)
 
 
+# TODO: Do we need the API for parameters?
+# If we still want them, what takes precedence? API or returns?
+# Once we decide that, collect the parameters and update them in tasks
+
+
 @check_context
 def set_parameter(**kwargs) -> None:
     """
@@ -279,7 +284,7 @@ def get_run_id() -> str:
     """
     Returns the run_id of the current run.
 
-    You can also access this from the environment variable `MAGNUS_RUN_ID`.
+    You can also access this from the environment variable `runnable_RUN_ID`.
     """
     return context.run_context.run_id
 
@@ -321,14 +326,14 @@ def get_experiment_tracker_context() -> ContextManager:
 
 def start_interactive_session(run_id: str = "", config_file: str = "", tag: str = "", parameters_file: str = ""):
     """
-    During interactive python coding, either via notebooks or ipython, you can start a magnus session by calling
+    During interactive python coding, either via notebooks or ipython, you can start a runnable session by calling
     this function. The executor would always be local executor as its interactive.
 
     If this was called during a pipeline/function/notebook execution, it will be ignored.
 
     Args:
         run_id (str, optional): The run id to use. Defaults to "" and would be created if not provided.
-        config_file (str, optional): The configuration file to use. Defaults to "" and magnus defaults.
+        config_file (str, optional): The configuration file to use. Defaults to "" and runnable defaults.
         tag (str, optional): The tag to attach to the run. Defaults to "".
         parameters_file (str, optional): The parameters file to use. Defaults to "".
     """
@@ -350,7 +355,7 @@ def start_interactive_session(run_id: str = "", config_file: str = "", tag: str
 
     executor = context.run_context.executor
 
-    utils.set_magnus_environment_variables(run_id=run_id, configuration_file=config_file, tag=tag)
+    utils.set_runnable_environment_variables(run_id=run_id, configuration_file=config_file, tag=tag)
 
     context.run_context.execution_plan = defaults.EXECUTION_PLAN.INTERACTIVE.value
     executor.prepare_for_graph_execution()

runnable/nodes.py

@@ -64,7 +64,7 @@ class BaseNode(ABC, BaseModel):
     @classmethod
     def _get_internal_name_from_command_name(cls, command_name: str) -> str:
         """
-        Replace Magnus specific character (%) with whitespace.
+        Replace runnable specific character (%) with whitespace.
         The opposite of _command_friendly_name.
 
         Args:
@@ -274,7 +274,13 @@ class BaseNode(ABC, BaseModel):
         ...
 
     @abstractmethod
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         """
         The actual function that does the execution of the command in the config.
 
@@ -282,7 +288,7 @@ class BaseNode(ABC, BaseModel):
         composite nodes.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor class
+            executor (runnable.executor.BaseExecutor): The executor class
             mock (bool, optional): Don't run, just pretend. Defaults to False.
             map_variable (str, optional): The value of the map iteration variable, if part of a map node.
                 Defaults to ''.
@@ -301,7 +307,7 @@ class BaseNode(ABC, BaseModel):
         Function should only be implemented for composite nodes like dag, map, parallel.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor.
+            executor (runnable.executor.BaseExecutor): The executor.
 
         Raises:
             NotImplementedError: Base class, hence not implemented.
@@ -317,7 +323,7 @@ class BaseNode(ABC, BaseModel):
         Function should only be implemented for composite nodes like dag, map, parallel.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor.
+            executor (runnable.executor.BaseExecutor): The executor.
             map_variable (str, optional): The value of the map iteration variable, if part of a map node.
 
         Raises:
@@ -334,7 +340,7 @@ class BaseNode(ABC, BaseModel):
         Function should only be implemented for composite nodes like dag, map, parallel.
 
         Args:
-            executor (magnus.executor.BaseExecutor): The executor.
+            executor (runnable.executor.BaseExecutor): The executor.
             map_variable (str, optional): The value of the map iteration variable, if part of a map node.
 
         Raises:
@@ -449,7 +455,13 @@ class CompositeNode(TraversalNode):
     def _get_max_attempts(self) -> int:
         raise Exception("This is a composite node and does not have a max_attempts")
 
-    def execute(self, mock=False, map_variable: TypeMapVariable = None, **kwargs) -> StepAttempt:
+    def execute(
+        self,
+        mock=False,
+        params: Optional[Dict[str, Any]] = None,
+        map_variable: TypeMapVariable = None,
+        **kwargs,
+    ) -> StepAttempt:
         raise Exception("This is a composite node and does not have an execute function")
 
 

runnable/parameters.py

@@ -16,7 +16,7 @@ logger = logging.getLogger(defaults.LOGGER_NAME)
 
 def get_user_set_parameters(remove: bool = False) -> Dict[str, Any]:
     """
-    Scans the environment variables for any user returned parameters that have a prefix MAGNUS_PRM_.
+    Scans the environment variables for any user returned parameters that have a prefix runnable_PRM_.
 
     This function does not deal with any type conversion of the parameters.
     It just deserializes the parameters and returns them as a dictionary.