runnable 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

extensions/pipeline_executor/local_container.py

@@ -2,12 +2,12 @@ import logging
 from pathlib import Path
 from typing import Dict
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from extensions.pipeline_executor import GenericPipelineExecutor
-from runnable import console, defaults, task_console, utils
+from runnable import defaults, utils
 from runnable.datastore import StepLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -70,7 +70,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
     auto_remove_container: bool = True
     environment: Dict[str, str] = Field(default_factory=dict)
 
-    _is_local: bool = False
+    _should_setup_run_log_at_traversal: bool = PrivateAttr(default=True)
 
     _container_log_location = "/tmp/run_logs/"
     _container_catalog_location = "/tmp/catalog/"
@@ -104,7 +104,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
             code_id.code_identifier_url = "local docker host"
             step_log.code_identities.append(code_id)
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         We are already in the container, we just execute the node.
         The node is already prepared for execution.
@@ -112,69 +112,8 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         self._use_volumes()
         return self._execute_node(node, map_variable)
 
-    def execute_from_graph(
-        self,
-        node: BaseNode,
-        map_variable: TypeMapVariable = None,
-    ):
-        """
-        This is the entry point to from the graph execution.
-
-        While the self.execute_graph is responsible for traversing the graph, this function is responsible for
-        actual execution of the node.
-
-        If the node type is:
-            * task : We can delegate to _execute_node after checking the eligibility for re-run in cases of a re-run
-            * success: We can delegate to _execute_node
-            * fail: We can delegate to _execute_node
-
-        For nodes that are internally graphs:
-            * parallel: Delegate the responsibility of execution to the node.execute_as_graph()
-            * dag: Delegate the responsibility of execution to the node.execute_as_graph()
-            * map: Delegate the responsibility of execution to the node.execute_as_graph()
-
-        Transpilers will NEVER use this method and will NEVER call ths method.
-        This method should only be used by interactive executors.
-
-        Args:
-            node (Node): The node to execute
-            map_variable (dict, optional): If the node if of a map state, this corresponds to the value of iterable.
-                    Defaults to None.
-        """
-        step_log = self._context.run_log_store.create_step_log(
-            node.name, node._get_step_log_name(map_variable)
-        )
-
-        self.add_code_identities(node=node, step_log=step_log)
-
-        step_log.step_type = node.node_type
-        step_log.status = defaults.PROCESSING
-
-        self._context.run_log_store.add_step_log(step_log, self._context.run_id)
-
-        logger.info(f"Executing node: {node.get_summary()}")
-
-        # Add the step log to the database as per the situation.
-        # If its a terminal node, complete it now
-        if node.node_type in ["success", "fail"]:
-            self._execute_node(node, map_variable=map_variable)
-            return
-
-        # We call an internal function to iterate the sub graphs and execute them
-        if node.is_composite:
-            node.execute_as_graph(map_variable=map_variable)
-            return
-
-        task_console.export_text(clear=True)
-
-        task_name = node._resolve_map_placeholders(node.internal_name, map_variable)
-        console.print(
-            f":runner: Executing the node {task_name} ... ", style="bold color(208)"
-        )
-        self.trigger_node_execution(node=node, map_variable=map_variable)
-
     def trigger_node_execution(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         We come into this step via execute from graph, use trigger job to spin up the container.
@@ -192,7 +131,9 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         logger.debug("Here is the resolved executor config")
         logger.debug(executor_config)
 
-        command = utils.get_node_execution_command(node, map_variable=map_variable)
+        command = self._context.get_node_callable_command(
+            node, map_variable=map_variable
+        )
 
         self._spin_container(
             node=node,
@@ -218,7 +159,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         self,
         node: BaseNode,
         command: str,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         auto_remove_container: bool = True,
     ):
         """
@@ -294,6 +235,7 @@ class LocalContainerExecutor(GenericPipelineExecutor):
         """
         Mount the volumes for the container
         """
+        # TODO: There should be an abstraction on top of service providers
         match self._context.run_log_store.service_name:
             case "file-system":
                 write_to = self._context.run_log_store.log_folder
@@ -308,17 +250,17 @@ class LocalContainerExecutor(GenericPipelineExecutor):
                     "mode": "rw",
                 }
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                catalog_location = self._context.catalog_handler.catalog_location
+                catalog_location = self._context.catalog.catalog_location
                 self._volumes[str(Path(catalog_location).resolve())] = {
                     "bind": f"{self._container_catalog_location}",
                     "mode": "rw",
                 }
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                secrets_location = self._context.secrets_handler.location
+                secrets_location = self._context.secrets.location
                 self._volumes[str(Path(secrets_location).resolve())] = {
                     "bind": f"{self._container_secrets_location}",
                     "mode": "ro",
@@ -331,14 +273,12 @@ class LocalContainerExecutor(GenericPipelineExecutor):
             case "chunked-fs":
                 self._context.run_log_store.log_folder = self._container_log_location
 
-        match self._context.catalog_handler.service_name:
+        match self._context.catalog.service_name:
             case "file-system":
-                self._context.catalog_handler.catalog_location = (
+                self._context.catalog.catalog_location = (
                     self._container_catalog_location
                 )
 
-        match self._context.secrets_handler.service_name:
+        match self._context.secrets.service_name:
             case "dotenv":
-                self._context.secrets_handler.location = (
-                    self._container_secrets_location
-                )
+                self._context.secrets.location = self._container_secrets_location

extensions/pipeline_executor/mocked.py

@@ -4,10 +4,10 @@ from typing import Any, Dict, Type, cast
 
 from pydantic import ConfigDict, Field
 
-from extensions.nodes.nodes import TaskNode
+from extensions.nodes.task import TaskNode
 from extensions.pipeline_executor import GenericPipelineExecutor
 from runnable import context, defaults
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 from runnable.tasks import BaseTaskType
 
@@ -36,7 +36,7 @@ class MockedExecutor(GenericPipelineExecutor):
     def _context(self):
         return context.run_context
 
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This is the entry point to from the graph execution.
 
@@ -140,7 +140,7 @@ class MockedExecutor(GenericPipelineExecutor):
 
         return effective_node_config
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         The entry point for all executors apart from local.
         We have already prepared for node execution.

extensions/pipeline_executor/retry.py

@@ -3,9 +3,9 @@ from functools import cached_property
 from typing import Any, Dict, Optional
 
 from extensions.pipeline_executor import GenericPipelineExecutor
-from runnable import context, defaults, exceptions
+from runnable import defaults, exceptions
 from runnable.datastore import RunLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.nodes import BaseNode
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -33,10 +33,6 @@ class RetryExecutor(GenericPipelineExecutor):
     _original_run_log: Optional[RunLog] = None
     _restart_initiated: bool = False
 
-    @property
-    def _context(self):
-        return context.run_context
-
     @cached_property
     def original_run_log(self):
         return self._context.run_log_store.get_run_log_by_id(
@@ -46,7 +42,7 @@ class RetryExecutor(GenericPipelineExecutor):
 
     def _set_up_for_re_run(self, params: Dict[str, Any]) -> None:
         # Sync the previous run log catalog to this one.
-        self._context.catalog_handler.sync_between_runs(
+        self._context.catalog.sync_between_runs(
             previous_run_id=self.run_id, run_id=self._context.run_id
         )
 
@@ -63,7 +59,7 @@ class RetryExecutor(GenericPipelineExecutor):
         # Should the parameters be copied from previous execution
         # self._set_up_for_re_run(params=params)
 
-    def execute_from_graph(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_from_graph(self, node: BaseNode, map_variable: MapVariableType = None):
         """
         This is the entry point to from the graph execution.
 
@@ -124,7 +120,7 @@ class RetryExecutor(GenericPipelineExecutor):
         self.execute_node(node=node, map_variable=map_variable)
 
     def _is_step_eligible_for_rerun(
-        self, node: BaseNode, map_variable: TypeMapVariable = None
+        self, node: BaseNode, map_variable: MapVariableType = None
     ):
         """
         In case of a re-run, this method checks to see if the previous run step status to determine if a re-run is
@@ -172,5 +168,5 @@ class RetryExecutor(GenericPipelineExecutor):
         self._restart_initiated = True
         return True
 
-    def execute_node(self, node: BaseNode, map_variable: TypeMapVariable = None):
+    def execute_node(self, node: BaseNode, map_variable: MapVariableType = None):
         self._execute_node(node, map_variable=map_variable)

extensions/tasks/torch.py

@@ -81,7 +81,7 @@ class TorchTaskType(BaseTaskType, TorchConfig):
 
     def execute_command(
         self,
-        map_variable: defaults.TypeMapVariable = None,
+        map_variable: defaults.MapVariableType = None,
     ):
         assert map_variable is None, "map_variable is not supported for torch"
 

runnable/__init__.py

@@ -1,17 +1,9 @@
 # ruff: noqa
 
-
-import logging
 import os
-from logging.config import dictConfig
 
 from rich.console import Console
 
-from runnable import defaults
-
-dictConfig(defaults.LOGGING_CONFIG)
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
 console = Console(record=True)
 console.print(":runner: Lets go!!")
 

runnable/catalog.py

@@ -1,6 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -11,26 +11,6 @@ from runnable.datastore import DataCatalog
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-def is_catalog_out_of_sync(
-    catalog, synced_catalogs=Optional[List[DataCatalog]]
-) -> bool:
-    """
-    Check if the catalog items are out of sync from already cataloged objects.
-    If they are, return False.
-    If the object does not exist or synced catalog does not exist, return True
-    """
-    if not synced_catalogs:
-        return True  # If nothing has been synced in the past
-
-    for synced_catalog in synced_catalogs:
-        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
-            if synced_catalog.data_hash == catalog.data_hash:
-                return False
-            return True
-
-    return True  # The object does not exist, sync it
-
-
 # --8<-- [start:docs]
 
 

runnable/cli.py

@@ -223,56 +223,6 @@ def fan(
     )
 
 
-@app.command()
-def submit_job(
-    job_definition_file: Annotated[
-        str,
-        typer.Argument(
-            help=("The yaml file containing the job definition"),
-        ),
-    ],
-    config_file: Annotated[
-        str,
-        typer.Option(
-            "--config", "-c", help="The configuration file specifying the services"
-        ),
-    ] = "",
-    parameters_file: Annotated[
-        str,
-        typer.Option(
-            "--parameters",
-            "-p",
-            help="Parameters, in yaml,  accessible by the application",
-        ),
-    ] = "",
-    log_level: Annotated[
-        LogLevel,
-        typer.Option(
-            "--log-level",
-            help="The log level",
-            show_default=True,
-            case_sensitive=False,
-        ),
-    ] = LogLevel.WARNING,
-    tag: Annotated[str, typer.Option(help="A tag attached to the run")] = "",
-    run_id: Annotated[
-        str,
-        typer.Option(
-            help="An optional run_id, one would be generated if its not provided"
-        ),
-    ] = "",
-):
-    logger.setLevel(log_level.value)
-
-    entrypoints.execute_job_yaml_spec(
-        configuration_file=config_file,
-        job_definition_file=job_definition_file,
-        tag=tag,
-        run_id=run_id,
-        parameters_file=parameters_file,
-    )
-
-
 @app.command(hidden=True)
 def execute_job(
     job_definition_file: Annotated[
@@ -302,14 +252,6 @@ def execute_job(
             help="Parameters, in yaml,  accessible by the application",
         ),
     ] = "",
-    mode: Annotated[
-        ExecutionMode,
-        typer.Option(
-            "--mode",
-            "-m",
-            help="spec in yaml or python sdk",
-        ),
-    ] = ExecutionMode.YAML,
     log_level: Annotated[
         LogLevel,
         typer.Option(
@@ -326,7 +268,6 @@ def execute_job(
     entrypoints.execute_job_non_local(
         configuration_file=config_file,
         job_definition_file=job_definition_file,
-        mode=mode,
         tag=tag,
         run_id=run_id,
         parameters_file=parameters_file,