runnable 0.13.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/extensions/catalog/__init__.py

@@ -1,21 +0,0 @@
-from typing import List, Optional
-
-from runnable.datastore import DataCatalog
-
-
-def is_catalog_out_of_sync(catalog, synced_catalogs=Optional[List[DataCatalog]]) -> bool:
-    """
-    Check if the catalog items are out of sync from already cataloged objects.
-    If they are, return False.
-    If the object does not exist or synced catalog does not exist, return True
-    """
-    if not synced_catalogs:
-        return True  # If nothing has been synced in the past
-
-    for synced_catalog in synced_catalogs:
-        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
-            if synced_catalog.data_hash == catalog.data_hash:
-                return False
-            return True
-
-    return True  # The object does not exist, sync it

runnable/extensions/catalog/file_system/implementation.py

@@ -1,234 +0,0 @@
-import logging
-import os
-import shutil
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from runnable import defaults, utils
-from runnable.catalog import BaseCatalog
-from runnable.datastore import DataCatalog
-from runnable.extensions.catalog import is_catalog_out_of_sync
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class FileSystemCatalog(BaseCatalog):
-    """
-    A Catalog handler that uses the local file system for cataloging.
-
-    Note: Do not use this if the steps of the pipeline run on different compute environments.
-
-    Example config:
-
-    catalog:
-      type: file-system
-      config:
-        catalog_location: The location to store the catalog.
-        compute_data_folder: The folder to source the data from.
-
-    """
-
-    service_name: str = "file-system"
-    catalog_location: str = defaults.CATALOG_LOCATION_FOLDER
-
-    def get_catalog_location(self):
-        return self.catalog_location
-
-    def get_summary(self) -> Dict[str, Any]:
-        summary = {
-            "Catalog Location": self.get_catalog_location(),
-        }
-
-        return summary
-
-    def get(self, name: str, run_id: str, compute_data_folder: str = "", **kwargs) -> List[DataCatalog]:
-        """
-        Get the file by matching glob pattern to the name
-
-        Args:
-            name ([str]): A glob matching the file name
-            run_id ([str]): The run id
-
-        Raises:
-            Exception: If the catalog location does not exist
-
-        Returns:
-            List(object) : A list of catalog objects
-        """
-        logger.info(f"Using the {self.service_name} catalog and trying to get {name} for run_id: {run_id}")
-
-        copy_to = self.compute_data_folder
-        if compute_data_folder:
-            copy_to = compute_data_folder
-
-        copy_to = Path(copy_to)  # type: ignore
-
-        catalog_location = self.get_catalog_location()
-        run_catalog = Path(catalog_location) / run_id / copy_to
-
-        logger.debug(f"Copying objects to {copy_to} from the run catalog location of {run_catalog}")
-
-        if not utils.does_dir_exist(run_catalog):
-            msg = (
-                f"Expected Catalog to be present at: {run_catalog} but not found.\n"
-                "Note: Please make sure that some data was put in the catalog before trying to get from it.\n"
-            )
-            raise Exception(msg)
-
-        # Iterate through the contents of the run_catalog and copy the files that fit the name pattern
-        # We should also return a list of data hashes
-        glob_files = run_catalog.glob(name)
-        logger.debug(f"Glob identified {glob_files} as matches to from the catalog location: {run_catalog}")
-
-        data_catalogs = []
-        run_log_store = self._context.run_log_store
-        for file in glob_files:
-            if file.is_dir():
-                # Need not add a data catalog for the folder
-                continue
-
-            if str(file).endswith(".execution.log"):
-                continue
-
-            relative_file_path = file.relative_to(run_catalog)
-
-            data_catalog = run_log_store.create_data_catalog(str(relative_file_path))
-            data_catalog.catalog_handler_location = catalog_location
-            data_catalog.catalog_relative_path = str(relative_file_path)
-            data_catalog.data_hash = utils.get_data_hash(str(file))
-            data_catalog.stage = "get"
-            data_catalogs.append(data_catalog)
-
-            # Make the directory in the data folder if required
-            Path(copy_to / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
-            shutil.copy(file, copy_to / relative_file_path)
-
-            logger.info(f"Copied {file} from {run_catalog} to {copy_to}")
-
-        if not data_catalogs:
-            raise Exception(f"Did not find any files matching {name} in {run_catalog}")
-
-        return data_catalogs
-
-    def put(
-        self,
-        name: str,
-        run_id: str,
-        compute_data_folder: str = "",
-        synced_catalogs: Optional[List[DataCatalog]] = None,
-        **kwargs,
-    ) -> List[DataCatalog]:
-        """
-        Put the files matching the glob pattern into the catalog.
-
-        If previously synced catalogs are provided, and no changes were observed, we do not sync them.
-
-        Args:
-            name (str): The glob pattern of the files to catalog
-            run_id (str): The run id of the run
-            compute_data_folder (str, optional): The compute data folder to sync from. Defaults to settings default.
-            synced_catalogs (dict, optional): dictionary of previously synced catalogs. Defaults to None.
-
-        Raises:
-            Exception: If the compute data folder does not exist.
-
-        Returns:
-            List(object) : A list of catalog objects
-        """
-        logger.info(f"Using the {self.service_name} catalog and trying to put {name} for run_id: {run_id}")
-
-        copy_from = self.compute_data_folder
-        if compute_data_folder:
-            copy_from = compute_data_folder
-        copy_from = Path(copy_from)  # type: ignore
-
-        catalog_location = self.get_catalog_location()
-        run_catalog = Path(catalog_location) / run_id
-        utils.safe_make_dir(run_catalog)
-
-        logger.debug(f"Copying objects from {copy_from} to the run catalog location of {run_catalog}")
-
-        if not utils.does_dir_exist(copy_from):
-            msg = (
-                f"Expected compute data folder to be present at: {compute_data_folder} but not found. \n"
-                "Note: runnable does not create the compute data folder for you. Please ensure that the "
-                "folder exists.\n"
-            )
-            raise Exception(msg)
-
-        # Iterate through the contents of copy_from and if the name matches, we move them to the run_catalog
-        # We should also return a list of datastore.DataCatalog items
-
-        glob_files = copy_from.glob(name)  # type: ignore
-        logger.debug(f"Glob identified {glob_files} as matches to from the compute data folder: {copy_from}")
-
-        data_catalogs = []
-        run_log_store = self._context.run_log_store
-        for file in glob_files:
-            if file.is_dir():
-                # Need not add a data catalog for the folder
-                continue
-
-            relative_file_path = file.relative_to(".")
-
-            data_catalog = run_log_store.create_data_catalog(str(relative_file_path))
-            data_catalog.catalog_handler_location = catalog_location
-            data_catalog.catalog_relative_path = run_id + os.sep + str(relative_file_path)
-            data_catalog.data_hash = utils.get_data_hash(str(file))
-            data_catalog.stage = "put"
-            data_catalogs.append(data_catalog)
-
-            if is_catalog_out_of_sync(data_catalog, synced_catalogs):
-                logger.info(f"{data_catalog.name} was found to be changed, syncing")
-
-                # Make the directory in the catalog if required
-                Path(run_catalog / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
-                shutil.copy(file, run_catalog / relative_file_path)
-            else:
-                logger.info(f"{data_catalog.name} was found to be unchanged, ignoring syncing")
-
-        if not data_catalogs:
-            raise Exception(f"Did not find any files matching {name} in {copy_from}")
-
-        return data_catalogs
-
-    def sync_between_runs(self, previous_run_id: str, run_id: str):
-        """
-        Given the previous run id, sync the catalogs between the current one and previous
-
-        Args:
-            previous_run_id (str): The previous run id to sync the catalogs from
-            run_id (str): The run_id to which the data catalogs should be synced to.
-
-        Raises:
-            Exception: If the previous run log does not exist in the catalog
-
-        """
-        logger.info(
-            f"Using the {self.service_name} catalog and syncing catalogs"
-            "between old: {previous_run_id} to new: {run_id}"
-        )
-
-        catalog_location = Path(self.get_catalog_location())
-        run_catalog = catalog_location / run_id
-        utils.safe_make_dir(run_catalog)
-
-        if not utils.does_dir_exist(catalog_location / previous_run_id):
-            msg = (
-                f"Catalogs from previous run : {previous_run_id} are not found.\n"
-                "Note: Please provision the catalog objects generated by previous run in the same catalog location"
-                " as the current run, even if the catalog handler for the previous run was different"
-            )
-            raise Exception(msg)
-
-        cataloged_files = list((catalog_location / previous_run_id).glob("*"))
-
-        for cataloged_file in cataloged_files:
-            if str(cataloged_file).endswith("execution.log"):
-                continue
-
-            if cataloged_file.is_file():
-                shutil.copy(cataloged_file, run_catalog / cataloged_file.name)
-            else:
-                shutil.copytree(cataloged_file, run_catalog / cataloged_file.name)
-            logger.info(f"Copied file from: {cataloged_file} to {run_catalog}")

runnable/extensions/catalog/k8s_pvc/implementation.py

@@ -1,16 +0,0 @@
-import logging
-from pathlib import Path
-
-from runnable import defaults
-from runnable.extensions.catalog.file_system.implementation import FileSystemCatalog
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class K8sPVCatalog(FileSystemCatalog):
-    service_name: str = "k8s-pvc"
-    persistent_volume_name: str
-    mount_path: str
-
-    def get_catalog_location(self):
-        return str(Path(self.mount_path) / self.catalog_location)

runnable/extensions/catalog/k8s_pvc/integration.py

@@ -1,59 +0,0 @@
-import logging
-from typing import cast
-
-from runnable import defaults
-from runnable.integration import BaseIntegration
-
-logger = logging.getLogger(defaults.NAME)
-
-
-class LocalCompute(BaseIntegration):
-    """
-    Integration between local and k8's pvc
-    """
-
-    executor_type = "local"
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "k8s-pvc"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "We can't use the local compute k8s pvc store integration."
-        raise Exception(msg)
-
-
-class LocalContainerCompute(BaseIntegration):
-    """
-    Integration between local-container and k8's pvc
-    """
-
-    executor_type = "local-container"
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "k8s-pvc"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "We can't use the local-container compute k8s pvc store integration."
-        raise Exception(msg)
-
-
-class ArgoCompute(BaseIntegration):
-    """
-    Integration between argo and k8's pvc
-    """
-
-    executor_type = "argo"
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "k8s-pvc"  # The actual implementation of the service
-
-    def configure_for_traversal(self, **kwargs):
-        from runnable.extensions.catalog.k8s_pvc.implementation import K8sPVCatalog
-        from runnable.extensions.executor.argo.implementation import ArgoExecutor, UserVolumeMounts
-
-        self.executor = cast(ArgoExecutor, self.executor)
-        self.service = cast(K8sPVCatalog, self.service)
-
-        volume_mount = UserVolumeMounts(
-            name=self.service.persistent_volume_name,
-            mount_path=self.service.mount_path,
-        )
-
-        self.executor.persistent_volumes.append(volume_mount)