runnable 0.12.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

runnable-0.14.0.dist-info/RECORD

@@ -0,0 +1,24 @@
+runnable/__init__.py,sha256=WuJwXEBxjiz2E1jBapkOkYpIaCAPZ1Udyep0dnN4bkE,666
+runnable/catalog.py,sha256=5eTYwZWqfVBXIIn8WbweTMqiXZ9ccvtJBnAiIxSQ3Vk,4835
+runnable/cli.py,sha256=rBTvkNDetN6psHmDLa0kko8IHGvND5xMuO30hU_gcvY,9931
+runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
+runnable/datastore.py,sha256=Q_KKb4PNP2IXnUlR2bjOclDFAsAVJ_oNiCd5x0vB5jc,28127
+runnable/defaults.py,sha256=HYkXNI2hg0Y-SsXySjliwdc-3FUGJvJV3TnarmMIFFs,4656
+runnable/entrypoints.py,sha256=gMywHyoUheSAXCyqLMJ0QWK4IxiFVgEYwRDuZWsk-uI,18612
+runnable/exceptions.py,sha256=3gyN2bhqYvaZF_bo8hA7I09u8aQCAeh8NclBp5lCH8w,2574
+runnable/executor.py,sha256=Y-yCw4ZIz88nHn47QzCXvXm7VjByTIyBWzsqsaIpNP8,14653
+runnable/graph.py,sha256=EuH0210DcbEFlc6J-aSvfXJOb0SqORUiTpgFYyb_KPM,16602
+runnable/integration.py,sha256=IXBH20QKpFYW7pQwwbTI0qQvrg4kJseM0KMacQKli74,6791
+runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
+runnable/nodes.py,sha256=I9C65nj3kAHHXJSwn5QYximFjV7tbjBiTk0ayEgrmK4,16526
+runnable/parameters.py,sha256=g_bJurLjuppFDiDpfFqy6BRF36o_EY0OC5APl7HJFok,5450
+runnable/pickler.py,sha256=ydJ_eti_U1F4l-YacFp7BWm6g5vTn04UXye25S1HVok,2684
+runnable/sdk.py,sha256=tEwTwcfm1KVfnEql3G_yJpgymDWOqoIIA4q3RzKmHp0,30365
+runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
+runnable/tasks.py,sha256=QPCgH_D7YkN2oAi7-w6Ipt9IZ397SayjhAl_PPyVto8,29822
+runnable/utils.py,sha256=THMHnWVrUhNKdIvUbeZdDiXnP1WEOuee9e9OB8zzW5M,20441
+runnable-0.14.0.dist-info/METADATA,sha256=n3mrLZadanBuHaUz4h5WgtsfuXmkjaHdtsfOCSAmEEk,9994
+runnable-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.14.0.dist-info/entry_points.txt,sha256=8yBeduXOnO3SUnafZQwzXiE8rQMPXGDbqueyL7G9euM,1297
+runnable-0.14.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.14.0.dist-info/RECORD,,

{runnable-0.12.3.dist-info → runnable-0.14.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

runnable-0.14.0.dist-info/entry_points.txt

@@ -0,0 +1,40 @@
+[console_scripts]
+runnable = runnable.cli:cli
+
+[catalog]
+do-nothing = runnable.catalog:DoNothingCatalog
+file-system = extensions.catalog.file_system:FileSystemCatalog
+
+[executor]
+argo = extensions.executor.argo:ArgoExecutor
+local = extensions.executor.local:LocalExecutor
+local-container = extensions.executor.local_container:LocalContainerExecutor
+mocked = extensions.executor.mocked:MockedExecutor
+retry = extensions.executor.retry:RetryExecutor
+
+[nodes]
+dag = extensions.nodes.nodes:DagNode
+fail = extensions.nodes.nodes:FailNode
+map = extensions.nodes.nodes:MapNode
+parallel = extensions.nodes.nodes:ParallelNode
+stub = extensions.nodes.nodes:StubNode
+success = extensions.nodes.nodes:SuccessNode
+task = extensions.nodes.nodes:TaskNode
+
+[pickler]
+pickle = runnable.pickler:NativePickler
+
+[run_log_store]
+buffered = runnable.datastore:BufferRunLogstore
+chunked-fs = extensions.run_log_store.chunked_fs:ChunkedFileSystemRunLogStore
+file-system = extensions.run_log_store.file_system:FileSystemRunLogstore
+
+[secrets]
+do-nothing = runnable.secrets:DoNothingSecretManager
+dotenv = extensions.secrets.dotenv:DotEnvSecrets
+env-secrets = runnable.secrets:EnvSecretsManager
+
+[tasks]
+notebook = runnable.tasks:NotebookTaskType
+python = runnable.tasks:PythonTaskType
+shell = runnable.tasks:ShellTaskType

runnable/extensions/catalog/__init__.py

@@ -1,21 +0,0 @@
-from typing import List, Optional
-
-from runnable.datastore import DataCatalog
-
-
-def is_catalog_out_of_sync(catalog, synced_catalogs=Optional[List[DataCatalog]]) -> bool:
-    """
-    Check if the catalog items are out of sync from already cataloged objects.
-    If they are, return False.
-    If the object does not exist or synced catalog does not exist, return True
-    """
-    if not synced_catalogs:
-        return True  # If nothing has been synced in the past
-
-    for synced_catalog in synced_catalogs:
-        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
-            if synced_catalog.data_hash == catalog.data_hash:
-                return False
-            return True
-
-    return True  # The object does not exist, sync it

runnable/extensions/catalog/file_system/implementation.py

@@ -1,234 +0,0 @@
-import logging
-import os
-import shutil
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from runnable import defaults, utils
-from runnable.catalog import BaseCatalog
-from runnable.datastore import DataCatalog
-from runnable.extensions.catalog import is_catalog_out_of_sync
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class FileSystemCatalog(BaseCatalog):
-    """
-    A Catalog handler that uses the local file system for cataloging.
-
-    Note: Do not use this if the steps of the pipeline run on different compute environments.
-
-    Example config:
-
-    catalog:
-      type: file-system
-      config:
-        catalog_location: The location to store the catalog.
-        compute_data_folder: The folder to source the data from.
-
-    """
-
-    service_name: str = "file-system"
-    catalog_location: str = defaults.CATALOG_LOCATION_FOLDER
-
-    def get_catalog_location(self):
-        return self.catalog_location
-
-    def get_summary(self) -> Dict[str, Any]:
-        summary = {
-            "Catalog Location": self.get_catalog_location(),
-        }
-
-        return summary
-
-    def get(self, name: str, run_id: str, compute_data_folder: str = "", **kwargs) -> List[DataCatalog]:
-        """
-        Get the file by matching glob pattern to the name
-
-        Args:
-            name ([str]): A glob matching the file name
-            run_id ([str]): The run id
-
-        Raises:
-            Exception: If the catalog location does not exist
-
-        Returns:
-            List(object) : A list of catalog objects
-        """
-        logger.info(f"Using the {self.service_name} catalog and trying to get {name} for run_id: {run_id}")
-
-        copy_to = self.compute_data_folder
-        if compute_data_folder:
-            copy_to = compute_data_folder
-
-        copy_to = Path(copy_to)  # type: ignore
-
-        catalog_location = self.get_catalog_location()
-        run_catalog = Path(catalog_location) / run_id / copy_to
-
-        logger.debug(f"Copying objects to {copy_to} from the run catalog location of {run_catalog}")
-
-        if not utils.does_dir_exist(run_catalog):
-            msg = (
-                f"Expected Catalog to be present at: {run_catalog} but not found.\n"
-                "Note: Please make sure that some data was put in the catalog before trying to get from it.\n"
-            )
-            raise Exception(msg)
-
-        # Iterate through the contents of the run_catalog and copy the files that fit the name pattern
-        # We should also return a list of data hashes
-        glob_files = run_catalog.glob(name)
-        logger.debug(f"Glob identified {glob_files} as matches to from the catalog location: {run_catalog}")
-
-        data_catalogs = []
-        run_log_store = self._context.run_log_store
-        for file in glob_files:
-            if file.is_dir():
-                # Need not add a data catalog for the folder
-                continue
-
-            if str(file).endswith(".execution.log"):
-                continue
-
-            relative_file_path = file.relative_to(run_catalog)
-
-            data_catalog = run_log_store.create_data_catalog(str(relative_file_path))
-            data_catalog.catalog_handler_location = catalog_location
-            data_catalog.catalog_relative_path = str(relative_file_path)
-            data_catalog.data_hash = utils.get_data_hash(str(file))
-            data_catalog.stage = "get"
-            data_catalogs.append(data_catalog)
-
-            # Make the directory in the data folder if required
-            Path(copy_to / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
-            shutil.copy(file, copy_to / relative_file_path)
-
-            logger.info(f"Copied {file} from {run_catalog} to {copy_to}")
-
-        if not data_catalogs:
-            raise Exception(f"Did not find any files matching {name} in {run_catalog}")
-
-        return data_catalogs
-
-    def put(
-        self,
-        name: str,
-        run_id: str,
-        compute_data_folder: str = "",
-        synced_catalogs: Optional[List[DataCatalog]] = None,
-        **kwargs,
-    ) -> List[DataCatalog]:
-        """
-        Put the files matching the glob pattern into the catalog.
-
-        If previously synced catalogs are provided, and no changes were observed, we do not sync them.
-
-        Args:
-            name (str): The glob pattern of the files to catalog
-            run_id (str): The run id of the run
-            compute_data_folder (str, optional): The compute data folder to sync from. Defaults to settings default.
-            synced_catalogs (dict, optional): dictionary of previously synced catalogs. Defaults to None.
-
-        Raises:
-            Exception: If the compute data folder does not exist.
-
-        Returns:
-            List(object) : A list of catalog objects
-        """
-        logger.info(f"Using the {self.service_name} catalog and trying to put {name} for run_id: {run_id}")
-
-        copy_from = self.compute_data_folder
-        if compute_data_folder:
-            copy_from = compute_data_folder
-        copy_from = Path(copy_from)  # type: ignore
-
-        catalog_location = self.get_catalog_location()
-        run_catalog = Path(catalog_location) / run_id
-        utils.safe_make_dir(run_catalog)
-
-        logger.debug(f"Copying objects from {copy_from} to the run catalog location of {run_catalog}")
-
-        if not utils.does_dir_exist(copy_from):
-            msg = (
-                f"Expected compute data folder to be present at: {compute_data_folder} but not found. \n"
-                "Note: runnable does not create the compute data folder for you. Please ensure that the "
-                "folder exists.\n"
-            )
-            raise Exception(msg)
-
-        # Iterate through the contents of copy_from and if the name matches, we move them to the run_catalog
-        # We should also return a list of datastore.DataCatalog items
-
-        glob_files = copy_from.glob(name)  # type: ignore
-        logger.debug(f"Glob identified {glob_files} as matches to from the compute data folder: {copy_from}")
-
-        data_catalogs = []
-        run_log_store = self._context.run_log_store
-        for file in glob_files:
-            if file.is_dir():
-                # Need not add a data catalog for the folder
-                continue
-
-            relative_file_path = file.relative_to(".")
-
-            data_catalog = run_log_store.create_data_catalog(str(relative_file_path))
-            data_catalog.catalog_handler_location = catalog_location
-            data_catalog.catalog_relative_path = run_id + os.sep + str(relative_file_path)
-            data_catalog.data_hash = utils.get_data_hash(str(file))
-            data_catalog.stage = "put"
-            data_catalogs.append(data_catalog)
-
-            if is_catalog_out_of_sync(data_catalog, synced_catalogs):
-                logger.info(f"{data_catalog.name} was found to be changed, syncing")
-
-                # Make the directory in the catalog if required
-                Path(run_catalog / relative_file_path.parent).mkdir(parents=True, exist_ok=True)
-                shutil.copy(file, run_catalog / relative_file_path)
-            else:
-                logger.info(f"{data_catalog.name} was found to be unchanged, ignoring syncing")
-
-        if not data_catalogs:
-            raise Exception(f"Did not find any files matching {name} in {copy_from}")
-
-        return data_catalogs
-
-    def sync_between_runs(self, previous_run_id: str, run_id: str):
-        """
-        Given the previous run id, sync the catalogs between the current one and previous
-
-        Args:
-            previous_run_id (str): The previous run id to sync the catalogs from
-            run_id (str): The run_id to which the data catalogs should be synced to.
-
-        Raises:
-            Exception: If the previous run log does not exist in the catalog
-
-        """
-        logger.info(
-            f"Using the {self.service_name} catalog and syncing catalogs"
-            "between old: {previous_run_id} to new: {run_id}"
-        )
-
-        catalog_location = Path(self.get_catalog_location())
-        run_catalog = catalog_location / run_id
-        utils.safe_make_dir(run_catalog)
-
-        if not utils.does_dir_exist(catalog_location / previous_run_id):
-            msg = (
-                f"Catalogs from previous run : {previous_run_id} are not found.\n"
-                "Note: Please provision the catalog objects generated by previous run in the same catalog location"
-                " as the current run, even if the catalog handler for the previous run was different"
-            )
-            raise Exception(msg)
-
-        cataloged_files = list((catalog_location / previous_run_id).glob("*"))
-
-        for cataloged_file in cataloged_files:
-            if str(cataloged_file).endswith("execution.log"):
-                continue
-
-            if cataloged_file.is_file():
-                shutil.copy(cataloged_file, run_catalog / cataloged_file.name)
-            else:
-                shutil.copytree(cataloged_file, run_catalog / cataloged_file.name)
-            logger.info(f"Copied file from: {cataloged_file} to {run_catalog}")

runnable/extensions/catalog/k8s_pvc/implementation.py

@@ -1,16 +0,0 @@
-import logging
-from pathlib import Path
-
-from runnable import defaults
-from runnable.extensions.catalog.file_system.implementation import FileSystemCatalog
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class K8sPVCatalog(FileSystemCatalog):
-    service_name: str = "k8s-pvc"
-    persistent_volume_name: str
-    mount_path: str
-
-    def get_catalog_location(self):
-        return str(Path(self.mount_path) / self.catalog_location)

runnable/extensions/catalog/k8s_pvc/integration.py

@@ -1,59 +0,0 @@
-import logging
-from typing import cast
-
-from runnable import defaults
-from runnable.integration import BaseIntegration
-
-logger = logging.getLogger(defaults.NAME)
-
-
-class LocalCompute(BaseIntegration):
-    """
-    Integration between local and k8's pvc
-    """
-
-    executor_type = "local"
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "k8s-pvc"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "We can't use the local compute k8s pvc store integration."
-        raise Exception(msg)
-
-
-class LocalContainerCompute(BaseIntegration):
-    """
-    Integration between local-container and k8's pvc
-    """
-
-    executor_type = "local-container"
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "k8s-pvc"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "We can't use the local-container compute k8s pvc store integration."
-        raise Exception(msg)
-
-
-class ArgoCompute(BaseIntegration):
-    """
-    Integration between argo and k8's pvc
-    """
-
-    executor_type = "argo"
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "k8s-pvc"  # The actual implementation of the service
-
-    def configure_for_traversal(self, **kwargs):
-        from runnable.extensions.catalog.k8s_pvc.implementation import K8sPVCatalog
-        from runnable.extensions.executor.argo.implementation import ArgoExecutor, UserVolumeMounts
-
-        self.executor = cast(ArgoExecutor, self.executor)
-        self.service = cast(K8sPVCatalog, self.service)
-
-        volume_mount = UserVolumeMounts(
-            name=self.service.persistent_volume_name,
-            mount_path=self.service.mount_path,
-        )
-
-        self.executor.persistent_volumes.append(volume_mount)