runnable 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/utils.py

@@ -21,7 +21,6 @@ from runnable import defaults, names
 from runnable.defaults import TypeMapVariable
 
 if TYPE_CHECKING:  # pragma: no cover
-    from extensions.nodes.nodes import TaskNode
     from runnable.nodes import BaseNode
 
 
@@ -428,25 +427,26 @@ def get_node_execution_command(
     log_level = log_level or logging.getLevelName(logger.getEffectiveLevel())
 
     action = (
-        f"runnable execute_single_node {run_id} "
-        f"{node._command_friendly_name()}"
-        f" --log-level {log_level}"
+        f"runnable execute-single-node {run_id} "
+        f"{context.run_context.pipeline_file} "
+        f"{node._command_friendly_name()} "
+        f"--log-level {log_level} "
     )
 
-    if context.run_context.pipeline_file:
-        action = action + f" --file {context.run_context.pipeline_file}"
+    if context.run_context.from_sdk:
+        action = action + "--mode python "
 
     if map_variable:
-        action = action + f" --map-variable '{json.dumps(map_variable)}'"
+        action = action + f"--map-variable '{json.dumps(map_variable)}' "
 
     if context.run_context.configuration_file:
-        action = action + f" --config-file {context.run_context.configuration_file}"
+        action = action + f"--config {context.run_context.configuration_file} "
 
     if context.run_context.parameters_file:
-        action = action + f" --parameters-file {context.run_context.parameters_file}"
+        action = action + f"--parameters-file {context.run_context.parameters_file} "
 
     if context.run_context.tag:
-        action = action + f" --tag {context.run_context.tag}"
+        action = action + f"--tag {context.run_context.tag}"
 
     return action
 
@@ -476,8 +476,8 @@ def get_fan_command(
     action = (
         f"runnable fan {run_id} "
         f"{node._command_friendly_name()} "
+        f"{context.run_context.pipeline_file} "
         f"--mode {mode} "
-        f"--file {context.run_context.pipeline_file} "
         f"--log-level {log_level} "
     )
     if context.run_context.configuration_file:
@@ -496,18 +496,10 @@ def get_fan_command(
 
 
 # TODO: This is not the right place for this.
-def get_job_execution_command(node: TaskNode, over_write_run_id: str = "") -> str:
+def get_job_execution_command(over_write_run_id: str = "") -> str:
     """Get the execution command to run a job via command line.
 
     This function should be used by all executors to submit jobs in remote environment
-
-    Args:
-        executor (BaseExecutor): The executor class.
-        node (BaseNode): The node being executed.
-        over_write_run_id (str, optional): If the node is part of a map step. Defaults to ''.
-
-    Returns:
-        str: The execution command to run a job via command line.
     """
 
     run_id = context.run_context.run_id
@@ -517,24 +509,23 @@ def get_job_execution_command(node: TaskNode, over_write_run_id: str = "") -> st
 
     log_level = logging.getLevelName(logger.getEffectiveLevel())
 
-    cli_command, cli_options = node.executable.get_cli_options()
-
-    action = f"runnable execute_{cli_command} {run_id} " f" --log-level {log_level}"
-
-    action = action + f" --entrypoint {defaults.ENTRYPOINT.SYSTEM.value}"
+    action = (
+        f"runnable execute-job /app/{context.run_context.job_definition_file} {run_id} "
+        f" --log-level {log_level}"
+    )
 
     if context.run_context.configuration_file:
-        action = action + f" --config-file {context.run_context.configuration_file}"
+        action = action + f" --config {context.run_context.configuration_file}"
 
     if context.run_context.parameters_file:
-        action = action + f" --parameters-file {context.run_context.parameters_file}"
+        action = action + f" --parameters {context.run_context.parameters_file}"
+
+    if context.run_context.from_sdk:
+        action = action + " --mode python "
 
     if context.run_context.tag:
         action = action + f" --tag {context.run_context.tag}"
 
-    for key, value in cli_options.items():
-        action = action + f" --{key} {value}"
-
     return action
 
 
@@ -555,6 +546,7 @@ def get_provider_by_name_and_type(
     Returns:
         object: A service object
     """
+
     namespace = service_type
 
     service_name = service_details["type"]

{runnable-0.14.0.dist-info → runnable-0.16.0.dist-info}/METADATA

@@ -1,16 +1,17 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.14.0
+Version: 0.16.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: catalog
 Requires-Dist: click-plugins>=1.1.1
 Requires-Dist: click<=8.1.3
 Requires-Dist: dill>=0.3.9
-Requires-Dist: executor
+Requires-Dist: job-executor
 Requires-Dist: nodes
+Requires-Dist: pipeline-executor
 Requires-Dist: pydantic>=2.10.3
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: rich>=13.9.4
@@ -24,6 +25,8 @@ Provides-Extra: docker
 Requires-Dist: docker>=7.1.0; extra == 'docker'
 Provides-Extra: examples
 Requires-Dist: pandas>=2.2.3; extra == 'examples'
+Provides-Extra: k8s
+Requires-Dist: kubernetes>=31.0.0; extra == 'k8s'
 Provides-Extra: notebook
 Requires-Dist: ploomber-engine>=0.0.33; extra == 'notebook'
 Description-Content-Type: text/markdown

runnable-0.16.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+runnable/__init__.py,sha256=KqpLDTD1CfdEj2aDyEkSn2KW-_83qyrRrrWLc5lZVM4,624
+runnable/catalog.py,sha256=MiEmb-18liAKmgeMdDF41VVn0ZEAVLP8hR33oacQ1zs,4930
+runnable/cli.py,sha256=01zmzOdynEmLI4vWDtSHQ6y1od_Jlc8G1RF69fi2L8g,8446
+runnable/context.py,sha256=pLw_n_5U-FM8-9-41YnkzETX94KrBAZWjxPgjm0O7hk,1305
+runnable/datastore.py,sha256=a1pT_P8TNcqQB-di2_uga7y-zS3TqUCb7sFhdxmVKGY,31907
+runnable/defaults.py,sha256=3o9IVGryyCE6PoQTOoaIaHHTbJGEzmdXMcwzOhwAYoI,3518
+runnable/entrypoints.py,sha256=67gPBiIIS4Kd9g6LdoGCraRJPda8K1i7Lp7XcD2iY5k,18913
+runnable/exceptions.py,sha256=LFbp0-Qxg2PAMLEVt7w2whhBxSG-5pzUEv5qN-Rc4_c,3003
+runnable/executor.py,sha256=cS30EC2Pfz8OzzEVcUYrVIyvGboKVUw5jKG2l72-UfM,15606
+runnable/graph.py,sha256=jVjikRLR-so3b2ufmNKpEQ_Ny68qN4bcGDAdXBRKiCY,16574
+runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
+runnable/nodes.py,sha256=YU9u7r1ESzui1uVtJ1dgwdv1ozyJnF2k-MCFieT8CLI,17519
+runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
+runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
+runnable/sdk.py,sha256=hwdk2dLmJsOTs2GnOlayw8WfliyeZFpA6Tcnp3tgblg,33370
+runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
+runnable/tasks.py,sha256=JnIIYQf3YUidHXIN6hiUIfDnegc7_rJMNXuHW4WS9ig,29378
+runnable/utils.py,sha256=wqyN7lMW56cBqyE59iDE6_i2HXPkvEUCQ-66UQnIwTA,19993
+runnable-0.16.0.dist-info/METADATA,sha256=JMu8mSsxMWr_wF246saRXV80D8_8cXMOPZrK6Pr9X6k,10102
+runnable-0.16.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.16.0.dist-info/entry_points.txt,sha256=I92DYldRrCb9HCsoum8GjC2UsQrWpuw2kawXTZpkIz4,1559
+runnable-0.16.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.16.0.dist-info/RECORD,,

{runnable-0.14.0.dist-info → runnable-0.16.0.dist-info}/entry_points.txt

@@ -1,16 +1,14 @@
 [console_scripts]
-runnable = runnable.cli:cli
+runnable = runnable.cli:app
 
 [catalog]
 do-nothing = runnable.catalog:DoNothingCatalog
 file-system = extensions.catalog.file_system:FileSystemCatalog
 
-[executor]
-argo = extensions.executor.argo:ArgoExecutor
-local = extensions.executor.local:LocalExecutor
-local-container = extensions.executor.local_container:LocalContainerExecutor
-mocked = extensions.executor.mocked:MockedExecutor
-retry = extensions.executor.retry:RetryExecutor
+[job_executor]
+k8s-job = extensions.job_executor.k8s:K8sJobExecutor
+local = extensions.job_executor.local:LocalJobExecutor
+local-container = extensions.job_executor.local_container:LocalContainerJobExecutor
 
 [nodes]
 dag = extensions.nodes.nodes:DagNode
@@ -24,6 +22,13 @@ task = extensions.nodes.nodes:TaskNode
 [pickler]
 pickle = runnable.pickler:NativePickler
 
+[pipeline_executor]
+argo = extensions.pipeline_executor.argo:ArgoExecutor
+local = extensions.pipeline_executor.local:LocalExecutor
+local-container = extensions.pipeline_executor.local_container:LocalContainerExecutor
+mocked = extensions.pipeline_executor.mocked:MockedExecutor
+retry = extensions.pipeline_executor.retry:RetryExecutor
+
 [run_log_store]
 buffered = runnable.datastore:BufferRunLogstore
 chunked-fs = extensions.run_log_store.chunked_fs:ChunkedFileSystemRunLogStore

runnable/integration.py

@@ -1,197 +0,0 @@
-import logging
-
-from stevedore import extension
-
-from runnable import defaults
-from runnable.executor import BaseExecutor
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-logging.getLogger("stevedore").setLevel(logging.CRITICAL)
-
-# --8<-- [start:docs]
-
-
-class BaseIntegration:
-    """
-    Base class for handling integration between Executor and one of Catalog, Secrets, RunLogStore.
-    """
-
-    executor_type = ""
-    service_type = ""  # One of secret, catalog, datastore, experiment tracker
-    service_provider = ""  # The actual implementation of the service
-
-    def __init__(self, executor: "BaseExecutor", integration_service: object):
-        self.executor = executor
-        self.service = integration_service
-
-    def validate(self, **kwargs):
-        """
-        Raise an exception if the executor_type is not compatible with service provider.
-
-        By default, it is considered as compatible.
-        """
-
-    def configure_for_traversal(self, **kwargs):
-        """
-        Do any changes needed to both executor and service provider during traversal of the graph.
-
-        By default, no change is required.
-        """
-
-    def configure_for_execution(self, **kwargs):
-        """
-        Do any changes needed to both executor and service provider during execution of a node.
-
-        By default, no change is required.
-        """
-
-
-# --8<-- [end:docs]
-
-
-def get_integration_handler(
-    executor: "BaseExecutor", service: object
-) -> BaseIntegration:
-    """
-    Return the integration handler between executor and the service.
-
-    If none found to be implemented, return the BaseIntegration which does nothing.
-
-    Args:
-        executor (BaseExecutor): The executor
-        service (object): The service provider
-
-    Returns:
-        [BaseIntegration]: The implemented integration handler or BaseIntegration if none found
-
-    Raises:
-        Exception: If multiple integrations are found for the executor and service
-    """
-    service_type = service.service_type  # type: ignore
-    service_name = getattr(service, "service_name")
-    integrations = []
-
-    # Get all the integrations defined by the 3rd party in their pyproject.toml
-    mgr = extension.ExtensionManager(
-        namespace="integration",
-        invoke_on_load=True,
-        invoke_kwds={"executor": executor, "integration_service": service},
-    )
-    for _, kls in mgr.items():
-        if (
-            kls.obj.executor_type == executor.service_name
-            and kls.obj.service_type == service_type
-            and kls.obj.service_provider == service_name
-        ):
-            logger.info(f"Identified an integration pattern {kls.obj}")
-            integrations.append(kls.obj)
-
-    # Get all the implementations defined by the runnable package
-    for kls in BaseIntegration.__subclasses__():
-        # Match the exact service type
-        if kls.service_type == service_type and kls.service_provider == service_name:
-            # Match either all executor or specific ones provided
-            if kls.executor_type == "" or kls.executor_type == executor.service_name:
-                integrations.append(kls(executor=executor, integration_service=service))
-
-    if len(integrations) > 1:
-        msg = (
-            f"Multiple integrations between {executor.service_name} and {service_name} of type {service_type} found. "
-            "If you defined an integration pattern, please ensure it is specific and does not conflict with runnable "
-            " implementations."
-        )
-        logger.exception(msg)
-        raise Exception(msg)
-
-    if not integrations:
-        logger.info(
-            f"Could not find an integration pattern for {executor.service_name} and {service_name} for {service_type}."
-            " This implies that there is no need to change the configurations."
-        )
-        return BaseIntegration(executor, service)
-
-    return integrations[0]
-
-
-def validate(executor: "BaseExecutor", service: object, **kwargs):
-    """
-    Helper function to resolve the Integration class and validate the compatibility between executor and service
-
-    Args:
-        executor (BaseExecutor) : The executor
-        service (object): The service provider
-    """
-    integration_handler = get_integration_handler(executor, service)
-    integration_handler.validate(**kwargs)
-
-
-def configure_for_traversal(executor: "BaseExecutor", service: object, **kwargs):
-    """
-    Helper function to resolve the Integration class and configure the executor and service for graph traversal
-
-    Args:
-        executor (BaseExecutor) : The executor
-        service (object): The service provider
-    """
-    integration_handler = get_integration_handler(executor, service)
-    integration_handler.configure_for_traversal(**kwargs)
-
-
-def configure_for_execution(executor: "BaseExecutor", service: object, **kwargs):
-    """
-    Helper function to resolve the Integration class and configure the executor and service for execution
-
-    Args:
-        executor (BaseExecutor) : The executor
-        service (object): The service provider
-    """
-    integration_handler = get_integration_handler(executor, service)
-    integration_handler.configure_for_execution(**kwargs)
-
-
-# TODO: Move all of these to the proper locations
-class BufferedRunLogStore(BaseIntegration):
-    """
-    Integration between any executor and buffered run log store
-    """
-
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "buffered"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        if not self.executor.service_name == "local":
-            raise Exception(
-                "Buffered run log store is only supported for local executor"
-            )
-
-        msg = (
-            "Run log generated by buffered run log store are not persisted. "
-            "Re-running this run, in case of a failure, is not possible"
-        )
-        logger.info(msg)
-
-
-class DoNothingCatalog(BaseIntegration):
-    """
-    Integration between any executor and do nothing catalog
-    """
-
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "do-nothing"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "A do-nothing catalog does not hold any data and therefore cannot pass data between nodes."
-        logger.info(msg)
-
-
-class DoNothingSecrets(BaseIntegration):
-    """
-    Integration between any executor and do nothing secrets
-    """
-
-    service_type = "secrets"  # One of secret, catalog, datastore
-    service_provider = "do-nothing"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "A do-nothing secrets does not hold any secrets and therefore cannot return you any secrets."
-        logger.info(msg)

runnable-0.14.0.dist-info/RECORD

@@ -1,24 +0,0 @@
-runnable/__init__.py,sha256=WuJwXEBxjiz2E1jBapkOkYpIaCAPZ1Udyep0dnN4bkE,666
-runnable/catalog.py,sha256=5eTYwZWqfVBXIIn8WbweTMqiXZ9ccvtJBnAiIxSQ3Vk,4835
-runnable/cli.py,sha256=rBTvkNDetN6psHmDLa0kko8IHGvND5xMuO30hU_gcvY,9931
-runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
-runnable/datastore.py,sha256=Q_KKb4PNP2IXnUlR2bjOclDFAsAVJ_oNiCd5x0vB5jc,28127
-runnable/defaults.py,sha256=HYkXNI2hg0Y-SsXySjliwdc-3FUGJvJV3TnarmMIFFs,4656
-runnable/entrypoints.py,sha256=gMywHyoUheSAXCyqLMJ0QWK4IxiFVgEYwRDuZWsk-uI,18612
-runnable/exceptions.py,sha256=3gyN2bhqYvaZF_bo8hA7I09u8aQCAeh8NclBp5lCH8w,2574
-runnable/executor.py,sha256=Y-yCw4ZIz88nHn47QzCXvXm7VjByTIyBWzsqsaIpNP8,14653
-runnable/graph.py,sha256=EuH0210DcbEFlc6J-aSvfXJOb0SqORUiTpgFYyb_KPM,16602
-runnable/integration.py,sha256=IXBH20QKpFYW7pQwwbTI0qQvrg4kJseM0KMacQKli74,6791
-runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
-runnable/nodes.py,sha256=I9C65nj3kAHHXJSwn5QYximFjV7tbjBiTk0ayEgrmK4,16526
-runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
-runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
-runnable/sdk.py,sha256=tEwTwcfm1KVfnEql3G_yJpgymDWOqoIIA4q3RzKmHp0,30365
-runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
-runnable/tasks.py,sha256=QPCgH_D7YkN2oAi7-w6Ipt9IZ397SayjhAl_PPyVto8,29822
-runnable/utils.py,sha256=THMHnWVrUhNKdIvUbeZdDiXnP1WEOuee9e9OB8zzW5M,20441
-runnable-0.14.0.dist-info/METADATA,sha256=n3mrLZadanBuHaUz4h5WgtsfuXmkjaHdtsfOCSAmEEk,9994
-runnable-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.14.0.dist-info/entry_points.txt,sha256=8yBeduXOnO3SUnafZQwzXiE8rQMPXGDbqueyL7G9euM,1297
-runnable-0.14.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.14.0.dist-info/RECORD,,