runnable 0.50.0__py3-none-any.whl

extensions/catalog/any_path.py

@@ -0,0 +1,214 @@
+import logging
+import os
+import shutil
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List
+
+from cloudpathlib import CloudPath
+
+from runnable import defaults, utils
+from runnable.catalog import BaseCatalog
+from runnable.datastore import DataCatalog
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class AnyPathCatalog(BaseCatalog):
+    """
+    A Catalog handler that uses the local file system for cataloging.
+
+    Note: Do not use this if the steps of the pipeline run on different compute environments.
+
+    Example config:
+
+    catalog:
+      type: file-system
+      config:
+        catalog_location: The location to store the catalog.
+        compute_data_folder: The folder to source the data from.
+
+    """
+
+    @abstractmethod
+    def get_summary(self) -> Dict[str, Any]: ...
+
+    @abstractmethod
+    def upload_to_catalog(self, file: Path) -> None: ...
+
+    @abstractmethod
+    def download_from_catalog(self, file: Path | CloudPath) -> None: ...
+
+    @abstractmethod
+    def get_catalog_location(self) -> Path | CloudPath:
+        """
+        For local file systems, this is the .catalog/run_id/compute_data_folder
+        For cloud systems, this is s3://bucket/run_id/compute_data_folder
+        """
+        ...
+
+    def get(self, name: str) -> List[DataCatalog]:
+        """
+        Get the file by matching glob pattern to the name
+
+        Args:
+            name ([str]): A glob matching the file name
+            run_id ([str]): The run id
+
+        Raises:
+            Exception: If the catalog location does not exist
+
+        Returns:
+            List(object) : A list of catalog objects
+        """
+        run_catalog = self.get_catalog_location()
+
+        # Iterate through the contents of the run_catalog and copy the files that fit the name pattern
+        # We should also return a list of data hashes
+        glob_files = run_catalog.glob(name)
+        logger.debug(
+            f"Glob identified {glob_files} as matches to from the catalog location: {run_catalog}"
+        )
+
+        data_catalogs = []
+        run_log_store = self._context.run_log_store
+        for file in glob_files:
+            if file.is_dir():
+                # Need not add a data catalog for the folder
+                continue
+
+            if str(file).endswith(".execution.log"):
+                continue
+
+            self.download_from_catalog(file)
+            relative_file_path = file.relative_to(run_catalog)  # type: ignore
+
+            data_catalog = run_log_store.create_data_catalog(str(relative_file_path))
+            data_catalog.catalog_relative_path = str(relative_file_path)
+            data_catalog.data_hash = utils.get_data_hash(str(relative_file_path))
+            data_catalog.stage = "get"
+            data_catalogs.append(data_catalog)
+
+        if not data_catalogs:
+            raise Exception(f"Did not find any files matching {name} in {run_catalog}")
+
+        return data_catalogs
+
+    def put(
+        self,
+        name: str,
+        allow_file_not_found_exc: bool = False,
+        store_copy: bool = True,
+    ) -> List[DataCatalog]:
+        """
+        Put the files matching the glob pattern into the catalog.
+
+        If previously synced catalogs are provided, and no changes were observed, we do not sync them.
+
+        Args:
+            name (str): The glob pattern of the files to catalog
+            run_id (str): The run id of the run
+            compute_data_folder (str, optional): The compute data folder to sync from. Defaults to settings default.
+            synced_catalogs (dict, optional): dictionary of previously synced catalogs. Defaults to None.
+
+        Raises:
+            Exception: If the compute data folder does not exist.
+
+        Returns:
+            List(object) : A list of catalog objects
+        """
+        run_id = self._context.run_id
+        logger.info(
+            f"Using the {self.service_name} catalog and trying to put {name} for run_id: {run_id}"
+        )
+
+        copy_from = Path(self.compute_data_folder)
+
+        if not copy_from.is_dir():
+            msg = (
+                f"Expected compute data folder to be present at: {copy_from} but not found. \n"
+                "Note: runnable does not create the compute data folder for you. Please ensure that the "
+                "folder exists.\n"
+            )
+            raise Exception(msg)
+
+        # Iterate through the contents of copy_from and if the name matches, we move them to the run_catalog
+        # We should also return a list of datastore.DataCatalog items
+        glob_files = copy_from.glob(name)
+        logger.debug(
+            f"Glob identified {glob_files} as matches to from the compute data folder: {copy_from}"
+        )
+
+        data_catalogs = []
+        run_log_store = self._context.run_log_store
+        for file in glob_files:
+            if file.is_dir():
+                # Need not add a data catalog for the folder
+                continue
+
+            relative_file_path = file.relative_to(copy_from)
+
+            data_catalog = run_log_store.create_data_catalog(str(relative_file_path))
+            data_catalog.catalog_relative_path = (
+                run_id + os.sep + str(relative_file_path)
+            )
+            data_catalog.data_hash = utils.get_data_hash(str(file))
+            data_catalog.stage = "put"
+            data_catalogs.append(data_catalog)
+
+            # TODO: Think about syncing only if the file is changed
+            if store_copy:
+                logger.debug(
+                    f"Copying file {file} to the catalog location for run_id: {run_id}"
+                )
+                self.upload_to_catalog(file)
+            else:
+                logger.debug(
+                    f"Not copying file {file} to the catalog location for run_id: {run_id}"
+                )
+
+        if not data_catalogs and not allow_file_not_found_exc:
+            raise Exception(f"Did not find any files matching {name} in {copy_from}")
+
+        return data_catalogs
+
+    def sync_between_runs(self, previous_run_id: str, run_id: str):
+        """
+        Given the previous run id, sync the catalogs between the current one and previous
+
+        Args:
+            previous_run_id (str): The previous run id to sync the catalogs from
+            run_id (str): The run_id to which the data catalogs should be synced to.
+
+        Raises:
+            Exception: If the previous run log does not exist in the catalog
+
+        """
+        logger.info(
+            f"Using the {self.service_name} catalog and syncing catalogs"
+            "between old: {previous_run_id} to new: {run_id}"
+        )
+
+        catalog_location = Path(self.get_catalog_location())
+        run_catalog = catalog_location / run_id
+        utils.safe_make_dir(run_catalog)
+
+        if not utils.does_dir_exist(catalog_location / previous_run_id):
+            msg = (
+                f"Catalogs from previous run : {previous_run_id} are not found.\n"
+                "Note: Please provision the catalog objects generated by previous run in the same catalog location"
+                " as the current run, even if the catalog handler for the previous run was different"
+            )
+            raise Exception(msg)
+
+        cataloged_files = list((catalog_location / previous_run_id).glob("*"))
+
+        for cataloged_file in cataloged_files:
+            if str(cataloged_file).endswith("execution.log"):
+                continue
+
+            if cataloged_file.is_file():
+                shutil.copy(cataloged_file, run_catalog / cataloged_file.name)
+            else:
+                shutil.copytree(cataloged_file, run_catalog / cataloged_file.name)
+            logger.info(f"Copied file from: {cataloged_file} to {run_catalog}")

extensions/catalog/file_system.py

@@ -0,0 +1,52 @@
+import logging
+import shutil
+from pathlib import Path
+from typing import Any
+
+from cloudpathlib import CloudPath
+from pydantic import Field
+
+from extensions.catalog.any_path import AnyPathCatalog
+from runnable import defaults
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class FileSystemCatalog(AnyPathCatalog):
+    service_name: str = "file-system"
+
+    catalog_location: str = Field(default=defaults.CATALOG_LOCATION_FOLDER)
+
+    def get_summary(self) -> dict[str, Any]:
+        return {
+            "compute_data_folder": self.compute_data_folder,
+            "catalog_location": self.catalog_location,
+        }
+
+    def get_catalog_location(self) -> Path:
+        run_id = self._context.run_id
+        return Path(self.catalog_location) / run_id / self.compute_data_folder
+
+    def download_from_catalog(self, file: Path | CloudPath) -> None:
+        assert isinstance(file, Path)
+
+        run_catalog = self.get_catalog_location()
+        relative_file_path = file.relative_to(run_catalog)
+
+        copy_to = self.compute_data_folder
+        # Make the directory in the data folder if required
+        Path(copy_to / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
+        shutil.copy(file, copy_to / relative_file_path)
+
+    def upload_to_catalog(self, file: Path) -> None:
+        run_catalog = self.get_catalog_location()
+        run_catalog.mkdir(parents=True, exist_ok=True)
+
+        logger.debug(
+            f"Copying objects from {self.compute_data_folder} to the run catalog location of {run_catalog}"
+        )
+
+        relative_file_path = file.relative_to(self.compute_data_folder)
+
+        (run_catalog / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
+        shutil.copy(file, run_catalog / relative_file_path)

extensions/catalog/minio.py

@@ -0,0 +1,72 @@
+import logging
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from cloudpathlib import CloudPath, S3Client, S3Path
+from pydantic import Field, SecretStr
+
+from extensions.catalog.any_path import AnyPathCatalog
+from runnable import defaults
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+@lru_cache
+def get_minio_client(
+    endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str
+) -> S3Client:
+    return S3Client(
+        endpoint_url=endpoint_url,
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+    )
+
+
+class MinioCatalog(AnyPathCatalog):
+    service_name: str = "minio"
+
+    endpoint_url: str = Field(default="http://localhost:9002")
+    aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
+    aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
+    bucket: str = "runnable"
+
+    def get_summary(self) -> dict[str, Any]:
+        return {
+            "service_name": self.service_name,
+            "compute_data_folder": self.compute_data_folder,
+            "endpoint_url": self.endpoint_url,
+            "bucket": self.bucket,
+        }
+
+    def get_catalog_location(self) -> S3Path:
+        run_id = self._context.run_id
+
+        return S3Path(
+            f"s3://{self.bucket}/{run_id}/{self.compute_data_folder}".strip("."),
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+    def download_from_catalog(self, file: Path | CloudPath) -> None:
+        assert isinstance(file, S3Path)
+
+        relative_file_path = file.relative_to(self.get_catalog_location())
+
+        file_to_download = Path(self.compute_data_folder) / relative_file_path
+        file_to_download.parent.mkdir(parents=True, exist_ok=True)
+
+        file.download_to(file_to_download)
+
+    def upload_to_catalog(self, file: Path) -> None:
+        run_catalog = self.get_catalog_location()
+
+        relative_file_path = file.relative_to(self.compute_data_folder)
+        (run_catalog / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
+
+        file_in_cloud = run_catalog / file
+        assert isinstance(file_in_cloud, S3Path)
+        file_in_cloud.upload_from(file)

extensions/catalog/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "catalog"
+version = "0.0.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

extensions/catalog/s3.py

@@ -0,0 +1,11 @@
+from cloudpathlib import S3Path
+
+from extensions.catalog.any_path import AnyPathCatalog
+
+
+class S3Catalog(AnyPathCatalog):
+    service_name: str = "s3"
+
+    def get_path(self, path: str) -> S3Path:
+        # TODO: Might need to assert the credentials are set
+        return S3Path(path)

extensions/job_executor/__init__.py

@@ -0,0 +1,236 @@
+import logging
+import os
+from typing import Dict, List, Optional
+
+from runnable import (
+    console,
+    context,
+    defaults,
+    exceptions,
+    parameters,
+    task_console,
+    utils,
+)
+from runnable.datastore import DataCatalog, JobLog, JsonParameter, StepAttempt
+from runnable.defaults import IterableParameterModel
+from runnable.executor import BaseJobExecutor
+from runnable.tasks import BaseTaskType
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class GenericJobExecutor(BaseJobExecutor):
+    """
+    The skeleton of an executor class.
+    Any implementation of an executor should inherit this class and over-ride accordingly.
+
+    This is a loaded base class which has a lot of methods already implemented for "typical" executions.
+    Look at the function docs to understand how to use them appropriately.
+
+    For any implementation:
+    1). Who/when should the run log be set up?
+    2). Who/When should the step log be set up?
+
+    """
+
+    service_name: str = ""
+    service_type: str = "job_executor"
+    mock: bool = False
+
+    @property
+    def _context(self):
+        current_context = context.get_run_context()
+        if current_context is None:
+            raise RuntimeError("No run context available")
+        if not isinstance(current_context, context.JobContext):
+            raise TypeError(
+                f"Expected JobContext, got {type(current_context).__name__}"
+            )
+        return current_context
+
+    def _get_parameters(self) -> Dict[str, JsonParameter]:
+        """
+        Consolidate the parameters from the environment variables
+        and the parameters file.
+
+        The parameters defined in the environment variables take precedence over the parameters file.
+
+        Returns:
+            _type_: _description_
+        """
+        params: Dict[str, JsonParameter] = {}
+        if self._context.parameters_file:
+            user_defined = utils.load_yaml(self._context.parameters_file) or {}
+
+            for key, value in user_defined.items():
+                params[key] = JsonParameter(value=value, kind="json")
+
+        # Update these with some from the environment variables
+        params.update(parameters.get_user_set_parameters())
+        logger.debug(f"parameters as seen by executor: {params}")
+        return params
+
+    def _set_up_run_log(self, exists_ok=False):
+        """
+        Create a run log and put that in the run log store
+
+        If exists_ok, we allow the run log to be already present in the run log store.
+        """
+        try:
+            attempt_run_log = self._context.run_log_store.get_run_log_by_id(
+                run_id=self._context.run_id, full=False
+            )
+
+            logger.warning(
+                f"The run log by id: {self._context.run_id} already exists, is this designed?"
+            )
+            raise exceptions.RunLogExistsError(
+                f"The run log by id: {self._context.run_id} already exists and is {attempt_run_log.status}"
+            )
+        except exceptions.RunLogNotFoundError:
+            pass
+        except exceptions.RunLogExistsError:
+            if exists_ok:
+                return
+            raise
+
+        # Consolidate and get the parameters
+        params = self._get_parameters()
+
+        self._context.run_log_store.create_run_log(
+            run_id=self._context.run_id,
+            tag=self._context.tag,
+            status=defaults.PROCESSING,
+        )
+        # Any interaction with run log store attributes should happen via API if available.
+        self._context.run_log_store.set_parameters(
+            run_id=self._context.run_id, parameters=params
+        )
+
+        # Update run_config
+        run_config = self._context.model_dump()
+        logger.debug(f"run_config as seen by executor: {run_config}")
+        self._context.run_log_store.set_run_config(
+            run_id=self._context.run_id, run_config=run_config
+        )
+
+    @property
+    def step_attempt_number(self) -> int:
+        """
+        The attempt number of the current step.
+        Orchestrators should use this step to submit multiple attempts of the job.
+
+        Returns:
+            int: The attempt number of the current step. Defaults to 1.
+        """
+        return int(os.environ.get(defaults.ATTEMPT_NUMBER, 1))
+
+    def add_code_identities(self, job_log: JobLog, **kwargs):
+        """
+        Add code identities specific to the implementation.
+
+        The Base class has an implementation of adding git code identities.
+
+        Args:
+            step_log (object): The step log object
+            node (BaseNode): The node we are adding the step log for
+        """
+        job_log.code_identities.append(utils.get_git_code_identity())
+
+    def send_return_code(self, stage="traversal"):
+        """
+        Convenience function used by pipeline to send return code to the caller of the cli
+
+        Raises:
+            Exception: If the pipeline execution failed
+        """
+        run_id = self._context.run_id
+
+        run_log = self._context.run_log_store.get_run_log_by_id(
+            run_id=run_id, full=False
+        )
+        if run_log.status == defaults.FAIL:
+            raise exceptions.ExecutionFailedError(run_id=run_id)
+
+    def _sync_catalog(
+        self,
+        catalog_settings=Optional[List[str]],
+        allow_file_not_found_exc: bool = False,
+    ) -> List[DataCatalog] | None:
+        if not catalog_settings:
+            logger.info("No catalog settings found")
+            return None
+
+        data_catalogs = []
+        for name_pattern in catalog_settings:
+            data_catalog = self._context.catalog.put(
+                name=name_pattern,
+                allow_file_not_found_exc=allow_file_not_found_exc,
+                store_copy=self._context.catalog_store_copy,
+            )
+
+            logger.debug(f"Added data catalog: {data_catalog} to job log")
+            data_catalogs.extend(data_catalog)
+
+        return data_catalogs
+
+    def add_task_log_to_catalog(
+        self, name: str, iter_variable: Optional[IterableParameterModel] = None
+    ):
+        log_file_name = utils.make_log_file_name(
+            name=name,
+            iter_variable=iter_variable,
+        )
+        task_console.save_text(log_file_name)
+        # Put the log file in the catalog
+        self._context.catalog.put(name=log_file_name)
+        os.remove(log_file_name)
+
+    def execute_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        """
+        Focusses on execution of the job.
+        """
+        logger.info("Trying to execute job")
+
+        job_log = self._context.run_log_store.get_job_log(run_id=self._context.run_id)
+        self.add_code_identities(job_log)
+
+        if not self.mock:
+            attempt_log = job.execute_command()
+            attempt_log.retry_indicator = self._context.retry_indicator
+        else:
+            attempt_log = StepAttempt(
+                status=defaults.SUCCESS,
+                retry_indicator=self._context.retry_indicator,
+            )
+
+        job_log.status = attempt_log.status
+        job_log.attempts.append(attempt_log)
+
+        allow_file_not_found_exc = True
+        if job_log.status == defaults.SUCCESS:
+            allow_file_not_found_exc = False
+
+        data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
+            catalog_settings=catalog_settings,
+            allow_file_not_found_exc=allow_file_not_found_exc,
+        )
+
+        logger.debug(f"data_catalogs_put: {data_catalogs_put}")
+        job_log.add_data_catalogs(data_catalogs_put or [])
+
+        console.print("Summary of job")
+        console.print(job_log.get_summary())
+
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+        job_log.add_data_catalogs(data_catalogs_put or [])
+
+        console.print("Summary of job")
+        console.print(job_log.get_summary())
+
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+        self.add_task_log_to_catalog("job")

extensions/job_executor/emulate.py

@@ -0,0 +1,70 @@
+import logging
+import shlex
+import subprocess
+import sys
+from typing import List, Optional
+
+from extensions.job_executor import GenericJobExecutor
+from runnable import context, defaults
+from runnable.tasks import BaseTaskType
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class EmulatorJobExecutor(GenericJobExecutor):
+    """
+    The EmulatorJobExecutor is a job executor that emulates the job execution.
+    """
+
+    service_name: str = "emulator"
+
+    def submit_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
+        """
+        This method gets invoked by the CLI.
+        """
+        self._set_up_run_log()
+
+        # Call the container job
+        job_log = self._context.run_log_store.create_job_log()
+        self._context.run_log_store.add_job_log(
+            run_id=self._context.run_id, job_log=job_log
+        )
+        self.run_click_command()
+
+    def run_click_command(self) -> str:
+        """
+        Execute a Click-based CLI command in the current virtual environment.
+
+        Args:
+            command: Click command to execute
+        """
+        assert isinstance(self._context, context.JobContext)
+        command = self._context.get_job_callable_command()
+
+        sub_command = [sys.executable, "-m", "runnable.cli"] + shlex.split(command)[1:]
+
+        process = subprocess.Popen(
+            sub_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+        output = []
+        try:
+            while True:
+                line = process.stdout.readline()  # type: ignore
+                if not line and process.poll() is not None:
+                    break
+                print(line, end="")
+                output.append(line)
+        finally:
+            process.stdout.close()  # type: ignore
+
+        if process.returncode != 0:
+            raise subprocess.CalledProcessError(
+                process.returncode, command, "".join(output)
+            )
+
+        return "".join(output)