runnable 0.14.0__tar.gz → 0.17.0__tar.gz

{runnable-0.14.0 → runnable-0.17.0}/.dockerignore

@@ -6,3 +6,5 @@ docs/
 .tox/
 .scripts/
 .tests/
+.venv/
+minikube/

{runnable-0.14.0 → runnable-0.17.0}/.github/workflows/release.yaml

@@ -6,7 +6,6 @@ on:
       - "examples/**"
     branches:
       - "main"
-      - "release-fix*"
 
 
 jobs:
@@ -22,6 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
+
       - name: "Set up Python"
         run: uv python install
       - run: |

{runnable-0.14.0 → runnable-0.17.0}/.gitignore

@@ -155,3 +155,5 @@ cov.xml
 .DS_Store
 
 data/
+
+minikube/

runnable-0.17.0/.python-version

@@ -0,0 +1 @@
+3.10

runnable-0.17.0/Dockerfile

@@ -0,0 +1,22 @@
+# Python 3.8 Image without Dependecies
+FROM ubuntu:24.04
+
+LABEL maintainer="vijay.vammi@astrazeneca.com"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    ca-certificates \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD https://astral.sh/uv/0.5.12/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin/:$PATH"
+
+COPY . /app
+WORKDIR /app
+
+RUN uv python install && \
+    uv sync --index https://artifactory.astrazeneca.net/api/pypi/pypi-virtual/simple/ --frozen --all-extras
+
+ENV PATH="/app/.venv/bin:$PATH"

{runnable-0.14.0 → runnable-0.17.0}/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.14.0
+Version: 0.17.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: catalog
 Requires-Dist: click-plugins>=1.1.1
 Requires-Dist: click<=8.1.3
 Requires-Dist: dill>=0.3.9
-Requires-Dist: executor
+Requires-Dist: job-executor
 Requires-Dist: nodes
+Requires-Dist: pipeline-executor
 Requires-Dist: pydantic>=2.10.3
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: rich>=13.9.4
@@ -24,6 +25,8 @@ Provides-Extra: docker
 Requires-Dist: docker>=7.1.0; extra == 'docker'
 Provides-Extra: examples
 Requires-Dist: pandas>=2.2.3; extra == 'examples'
+Provides-Extra: k8s
+Requires-Dist: kubernetes>=31.0.0; extra == 'k8s'
 Provides-Extra: notebook
 Requires-Dist: ploomber-engine>=0.0.33; extra == 'notebook'
 Description-Content-Type: text/markdown

{runnable-0.14.0 → runnable-0.17.0}/examples/01-tasks/python_tasks.py

@@ -36,7 +36,6 @@ def main():
     pipeline = Pipeline(steps=[hello_task])
 
     pipeline.execute()
-
     return pipeline
 
 

runnable-0.17.0/examples/11-jobs/catalog.py

@@ -0,0 +1,23 @@
+from examples.common.functions import write_files
+from runnable import Catalog, Job, PythonTask
+
+print("Running catalog.py")
+
+
+def main():
+    write_catalog = Catalog(put=["df.csv", "data_folder/data.txt"])
+    generate_data = PythonTask(
+        name="generate_data",
+        function=write_files,
+        catalog=write_catalog,
+    )
+
+    job = Job(name="catalog", task=generate_data)
+
+    _ = job.execute()
+
+    return job
+
+
+if __name__ == "__main__":
+    main()

runnable-0.17.0/examples/11-jobs/catalog.yaml

@@ -0,0 +1,5 @@
+type: python
+command: examples.common.functions.write_files
+catalog:
+    - "*"
+    - data_folder/data.txt

runnable-0.17.0/examples/11-jobs/k8s-job.yaml

@@ -0,0 +1,41 @@
+job-executor:
+  type: "k8s-job"
+  config:
+    jobSpec:
+      # activeDeadlineSeconds: Optional[int]
+      # selector: Optional[LabelSelector]
+      # ttlSecondsAfterFinished: Optional[int]
+      template:
+        # metadata:
+          # annotations: Optional[Dict[str, str]]
+          # generate_name: Optional[str] = run_id
+          # namespace: Optional[str] = "default"
+        spec:
+          # activeDeadlineSeconds: Optional[int]
+          # nodeSelector: Optional[Dict[str, str]]
+          # tolerations: Optional[List[Toleration]]
+          # volumes:
+          #   - name: str
+          #     hostPath:
+          #       path: str
+          # serviceAccountName: Optional[str]
+          # restartPolicy: Optional[str] = Choose from [Always, OnFailure, Never]
+          container:
+            # command: List[str]
+            # env:
+            #     - name: str
+            #       value: str
+            image: runnable-m1
+            # imagePullPolicy: Optional[str] = choose from [Always, Never, IfNotPresent]
+            # resources:
+            #   limits:
+            #     cpu: str
+            #     memory: str
+            #     gpu: str
+            #   requests:
+            #     cpu: str
+            #     memory: str
+            #     gpu: str
+            # volumeMounts:
+            #   - name: str
+            #     mountPath: str

runnable-0.17.0/examples/11-jobs/local-container.yaml

@@ -0,0 +1,4 @@
+job-executor:
+  type: "local-container" # (1)
+  config:
+    docker_image: runnable-m1:latest # (2)

runnable-0.17.0/examples/11-jobs/notebook.yaml

@@ -0,0 +1,3 @@
+# name: notebook job
+type: notebook
+command: examples/common/simple_notebook.ipynb # The path is relative to the root of the project.

runnable-0.17.0/examples/11-jobs/passing_parameters_python.py

@@ -0,0 +1,47 @@
+"""
+The below example shows how to set/get parameters in python
+tasks of the pipeline.
+
+The function, set_parameter, returns
+    - JSON serializable types
+    - pydantic models
+    - pandas dataframe, any "object" type
+
+pydantic models are implicitly handled by runnable
+but "object" types should be marked as "pickled".
+
+Use pickled even for python data types is advised for
+reasonably large collections.
+
+Run the below example as:
+    python examples/03-parameters/passing_parameters_python.py
+
+"""
+
+from examples.common.functions import write_parameter
+from runnable import Job, PythonTask, metric, pickled
+
+
+def main():
+    write_parameters = PythonTask(
+        function=write_parameter,
+        returns=[
+            pickled("df"),
+            "integer",
+            "floater",
+            "stringer",
+            "pydantic_param",
+            metric("score"),
+        ],
+        name="set_parameter",
+    )
+
+    job = Job(name="set_parameters", task=write_parameters)
+
+    job.execute()
+
+    return job
+
+
+if __name__ == "__main__":
+    main()

runnable-0.17.0/examples/11-jobs/python_parameters.yaml

@@ -0,0 +1,11 @@
+type: python
+command: examples.common.functions.write_parameter
+returns:
+  - name: df
+    kind: object
+  - name: integer
+  - name: floater
+  - name: stringer
+  - name: pydantic_param
+  - name: score
+    kind: metric

runnable-0.17.0/examples/11-jobs/python_tasks.py

@@ -0,0 +1,44 @@
+"""
+You can execute this pipeline by:
+
+    python examples/01-tasks/python_tasks.py
+
+The stdout of "Hello World!" would be captured as execution
+log and stored in the catalog.
+
+An example of the catalog structure:
+
+.catalog
+└── baked-heyrovsky-0602
+    └── hello.execution.log
+
+2 directories, 1 file
+
+
+The hello.execution.log has the captured stdout of "Hello World!".
+"""
+
+from examples.common.functions import hello
+from runnable import Job, PythonTask
+
+
+def main():
+    # Create a tasks which calls the function "hello"
+    # If this step executes successfully,
+    # the pipeline will terminate with success
+    hello_task = PythonTask(
+        name="hello",
+        function=hello,
+        terminate_with_success=True,
+    )
+
+    # The pipeline has only one step.
+    job = Job(name="hello", task=hello_task)
+
+    job.execute()
+
+    return job
+
+
+if __name__ == "__main__":
+    main()

runnable-0.17.0/examples/11-jobs/python_tasks.yaml

@@ -0,0 +1,2 @@
+type: python
+command: examples.common.functions.hello # dotted path to the function.

runnable-0.17.0/examples/11-jobs/scripts.yaml

@@ -0,0 +1,20 @@
+dag:
+  description: |
+    This is a sample pipeline with one step that
+    executes a shell command.
+
+    You can run this pipeline by:
+    runnable execute -f examples/01-tasks/scripts.yaml
+
+    For example:
+    .catalog
+    └── seasoned-perlman-1355
+        └── hello.execution.log
+
+  start_at: shell
+  steps:
+    shell:
+      type: task
+      command_type: shell
+      command: echo "hello world!!" # The path is relative to the root of the project.
+      next: success

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/argo-config.yaml

@@ -1,4 +1,4 @@
-executor:
+pipeline-executor:
   type: "argo" # (1)
   config:
     image: harbor.csis.astrazeneca.net/mlops/runnable:latest # (2)
@@ -7,12 +7,12 @@ executor:
       - name: magnus-volume
         mount_path: /mnt
 
-run_log_store: # (4)
+run-log-store: # (4)
   type: chunked-fs
-  config:
-    log_folder: /mnt/run_log_store
+  # config:
+  #   log_folder: /mnt/run_log_store
 
 catalog:
   type: file-system
-  config:
-   catalog_location: /mnt/catalog
+  # config:
+  #  catalog_location: /mnt/catalog

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/chunked-fs-run_log.yaml

@@ -1,2 +1,2 @@
-run_log_store:
+run-log-store:
   type: chunked-fs

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/default.yaml

@@ -1,7 +1,7 @@
-executor:
+pipeline-executor:
   type: local # (1)
 
-run_log_store:
+run-log-store:
   type: buffered # (2)
 
 catalog:

runnable-0.17.0/examples/configs/in-memory.yaml

@@ -0,0 +1,14 @@
+pipeline-executor:
+  type: local # (1)
+  config:
+    object_serialisation: false
+
+
+run-log-store:
+  type: buffered # (2)
+
+catalog:
+  type: do-nothing # (3)
+
+secrets:
+  type: do-nothing # (4)

runnable-0.17.0/examples/configs/local-container.yaml

@@ -0,0 +1,7 @@
+pipeline-executor:
+  type: "local-container" # (1)
+  config:
+    docker_image: runnable-m1:latest # (2)
+
+run-log-store: # (4)
+  type: chunked-fs

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/mocked-config-debug.yaml

@@ -1,10 +1,10 @@
 catalog:
   type: file-system # (1)
 
-run_log_store:
+run-log-store:
   type: file-system # (1)
 
-executor:
+pipeline-executor:
   type: mocked
   config:
     patches:

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/mocked-config-simple.yaml

@@ -1,8 +1,8 @@
 catalog:
   type: file-system # (1)
 
-run_log_store:
+run-log-store:
   type: file-system # (1)
 
-executor:
+pipeline-executor:
   type: mocked

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/mocked-config-unittest.yaml

@@ -1,10 +1,10 @@
 catalog:
   type: file-system # (1)
 
-run_log_store:
+run-log-store:
   type: file-system # (1)
 
-executor:
+pipeline-executor:
   type: mocked
   config:
     patches:

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/mocked-config.yaml

@@ -1,10 +1,10 @@
 catalog:
   type: file-system # (1)
 
-run_log_store:
+run-log-store:
   type: file-system # (1)
 
-executor:
+pipeline-executor:
   type: mocked
   config:
     patches:

{runnable-0.14.0 → runnable-0.17.0}/examples/configs/retry-config.yaml

@@ -1,4 +1,4 @@
-executor:
+pipeline-executor:
   type: retry
   config:
     run_id: parallel-fail
@@ -6,5 +6,5 @@ executor:
 catalog:
   type: file-system # (1)
 
-run_log_store:
+run-log-store:
   type: file-system # (1)

runnable-0.17.0/extensions/job_executor/__init__.py

@@ -0,0 +1,160 @@
+import logging
+import os
+from typing import Dict, List, Optional
+
+from runnable import context, defaults, exceptions, parameters, utils
+from runnable.datastore import DataCatalog, JobLog, JsonParameter
+from runnable.executor import BaseJobExecutor
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class GenericJobExecutor(BaseJobExecutor):
+    """
+    The skeleton of an executor class.
+    Any implementation of an executor should inherit this class and over-ride accordingly.
+
+    This is a loaded base class which has a lot of methods already implemented for "typical" executions.
+    Look at the function docs to understand how to use them appropriately.
+
+    For any implementation:
+    1). Who/when should the run log be set up?
+    2). Who/When should the step log be set up?
+
+    """
+
+    service_name: str = ""
+    service_type: str = "job_executor"
+
+    @property
+    def _context(self):
+        assert context.run_context
+        return context.run_context
+
+    def _get_parameters(self) -> Dict[str, JsonParameter]:
+        """
+        Consolidate the parameters from the environment variables
+        and the parameters file.
+
+        The parameters defined in the environment variables take precedence over the parameters file.
+
+        Returns:
+            _type_: _description_
+        """
+        params: Dict[str, JsonParameter] = {}
+        if self._context.parameters_file:
+            user_defined = utils.load_yaml(self._context.parameters_file) or {}
+
+            for key, value in user_defined.items():
+                params[key] = JsonParameter(value=value, kind="json")
+
+        # Update these with some from the environment variables
+        params.update(parameters.get_user_set_parameters())
+        logger.debug(f"parameters as seen by executor: {params}")
+        return params
+
+    def _set_up_run_log(self, exists_ok=False):
+        """
+        Create a run log and put that in the run log store
+
+        If exists_ok, we allow the run log to be already present in the run log store.
+        """
+        try:
+            attempt_run_log = self._context.run_log_store.get_run_log_by_id(
+                run_id=self._context.run_id, full=False
+            )
+
+            logger.warning(
+                f"The run log by id: {self._context.run_id} already exists, is this designed?"
+            )
+            raise exceptions.RunLogExistsError(
+                f"The run log by id: {self._context.run_id} already exists and is {attempt_run_log.status}"
+            )
+        except exceptions.RunLogNotFoundError:
+            pass
+        except exceptions.RunLogExistsError:
+            if exists_ok:
+                return
+            raise
+
+        # Consolidate and get the parameters
+        params = self._get_parameters()
+
+        self._context.run_log_store.create_run_log(
+            run_id=self._context.run_id,
+            tag=self._context.tag,
+            status=defaults.PROCESSING,
+            dag_hash=self._context.dag_hash,
+        )
+        # Any interaction with run log store attributes should happen via API if available.
+        self._context.run_log_store.set_parameters(
+            run_id=self._context.run_id, parameters=params
+        )
+
+        # Update run_config
+        run_config = utils.get_run_config()
+        logger.debug(f"run_config as seen by executor: {run_config}")
+        self._context.run_log_store.set_run_config(
+            run_id=self._context.run_id, run_config=run_config
+        )
+
+    @property
+    def step_attempt_number(self) -> int:
+        """
+        The attempt number of the current step.
+        Orchestrators should use this step to submit multiple attempts of the job.
+
+        Returns:
+            int: The attempt number of the current step. Defaults to 1.
+        """
+        return int(os.environ.get(defaults.ATTEMPT_NUMBER, 1))
+
+    def add_code_identities(self, job_log: JobLog, **kwargs):
+        """
+        Add code identities specific to the implementation.
+
+        The Base class has an implementation of adding git code identities.
+
+        Args:
+            step_log (object): The step log object
+            node (BaseNode): The node we are adding the step log for
+        """
+        job_log.code_identities.append(utils.get_git_code_identity())
+
+    def send_return_code(self, stage="traversal"):
+        """
+        Convenience function used by pipeline to send return code to the caller of the cli
+
+        Raises:
+            Exception: If the pipeline execution failed
+        """
+        run_id = self._context.run_id
+
+        run_log = self._context.run_log_store.get_run_log_by_id(
+            run_id=run_id, full=False
+        )
+        if run_log.status == defaults.FAIL:
+            raise exceptions.ExecutionFailedError(run_id=run_id)
+
+    def _sync_catalog(
+        self,
+        catalog_settings=Optional[List[str]],
+    ) -> List[DataCatalog] | None:
+        if not catalog_settings:
+            logger.info("No catalog settings found")
+            return None
+
+        compute_data_folder = self._context.catalog_handler.compute_data_folder
+
+        data_catalogs = []
+        for name_pattern in catalog_settings:
+            data_catalog = self._context.catalog_handler.put(
+                name=name_pattern,
+                run_id=self._context.run_id,
+                compute_data_folder=compute_data_folder,
+            )
+
+            logger.debug(f"Added data catalog: {data_catalog} to job log")
+            data_catalogs.extend(data_catalog)
+
+        return data_catalogs