runnable 0.13.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/extensions/secrets/dotenv/implementation.py

@@ -1,100 +0,0 @@
-import logging
-import os
-
-from runnable import defaults, exceptions, utils
-from runnable.secrets import BaseSecrets
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class DotEnvSecrets(BaseSecrets):
-    """
-    A secret manager which uses .env files for secrets.
-
-    We recommend this secrets manager only for local development and should not be used for anything close to
-    production.
-    """
-
-    service_name: str = "dotenv"
-    location: str = defaults.DOTENV_FILE_LOCATION
-    secrets: dict = {}
-
-    @property
-    def secrets_location(self):
-        """
-        Return the location of the .env file.
-        If the user has not over-ridden it, it defaults to .env file in the project root.
-
-        Returns:
-            str: The location of the secrets file
-        """
-        return self.location
-
-    def _load_secrets(self):
-        """
-        We assume that a dotenv file is of format,
-            key=value  -> secrets[key]='value'
-            key=value# comment  -> secrets[key1]='value1'
-            key=value2 # comment. -> secrets[key2]='value2'
-
-        Any of the above formats with export or set in front of them.
-
-        We strip the secret value of any empty spaces at the start and end.
-
-        Raises:
-            Exception: If the file at secrets_location is not found.
-            Exception: If the secrets are not formatted correctly.
-        """
-        # It was loaded in the previous call and need not to be reloaded
-        if self.secrets:
-            return
-
-        secrets_location = self.secrets_location
-        if not utils.does_file_exist(secrets_location):
-            raise Exception(f"Did not find the secrets file in {secrets_location}")
-
-        with open(secrets_location, "r") as fr:
-            for secret_line in fr:
-                # The order of removing fluff around the expression
-                # the new line
-                # the comment
-                # the white space
-                # Any export or set in front of the key any spaces after that.
-
-                secret_line = secret_line.strip(os.linesep).split("#")[0].strip()
-
-                if secret_line == "":
-                    continue
-
-                secret_line = utils.remove_prefix(secret_line, prefix="export").strip()
-                secret_line = utils.remove_prefix(secret_line, prefix="EXPORT").strip()
-                secret_line = utils.remove_prefix(secret_line, prefix="set").strip()
-                secret_line = utils.remove_prefix(secret_line, prefix="SET").strip()
-
-                data = secret_line.split("=")
-                if len(data) != 2:
-                    raise Exception("A secret should be of format, secret_name=secret_value[# any comment]")
-
-                key, value = data
-                self.secrets[key] = value.strip().strip('"').strip(os.linesep)
-
-    def get(self, name: str = "", **kwargs) -> str:
-        """
-        Get a secret of name from the secrets file.
-
-
-        Args:
-            name (str): The name of the secret to retrieve
-
-        Raises:
-            Exception: If the secret by the name is not found.
-
-        Returns:
-            str: The value of the secret
-        """
-        self._load_secrets()
-
-        if name in self.secrets:
-            return self.secrets[name]
-
-        raise exceptions.SecretNotFoundError(secret_name=name, secret_setting=self.secrets_location)

runnable/integration.py

@@ -1,192 +0,0 @@
-import logging
-
-from stevedore import extension
-
-from runnable import defaults
-from runnable.executor import BaseExecutor
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-logging.getLogger("stevedore").setLevel(logging.CRITICAL)
-
-# --8<-- [start:docs]
-
-
-class BaseIntegration:
-    """
-    Base class for handling integration between Executor and one of Catalog, Secrets, RunLogStore.
-    """
-
-    executor_type = ""
-    service_type = ""  # One of secret, catalog, datastore, experiment tracker
-    service_provider = ""  # The actual implementation of the service
-
-    def __init__(self, executor: "BaseExecutor", integration_service: object):
-        self.executor = executor
-        self.service = integration_service
-
-    def validate(self, **kwargs):
-        """
-        Raise an exception if the executor_type is not compatible with service provider.
-
-        By default, it is considered as compatible.
-        """
-
-    def configure_for_traversal(self, **kwargs):
-        """
-        Do any changes needed to both executor and service provider during traversal of the graph.
-
-        By default, no change is required.
-        """
-
-    def configure_for_execution(self, **kwargs):
-        """
-        Do any changes needed to both executor and service provider during execution of a node.
-
-        By default, no change is required.
-        """
-
-
-# --8<-- [end:docs]
-
-
-def get_integration_handler(executor: "BaseExecutor", service: object) -> BaseIntegration:
-    """
-    Return the integration handler between executor and the service.
-
-    If none found to be implemented, return the BaseIntegration which does nothing.
-
-    Args:
-        executor (BaseExecutor): The executor
-        service (object): The service provider
-
-    Returns:
-        [BaseIntegration]: The implemented integration handler or BaseIntegration if none found
-
-    Raises:
-        Exception: If multiple integrations are found for the executor and service
-    """
-    service_type = service.service_type  # type: ignore
-    service_name = getattr(service, "service_name")
-    integrations = []
-
-    # Get all the integrations defined by the 3rd party in their pyproject.toml
-    mgr = extension.ExtensionManager(
-        namespace="integration",
-        invoke_on_load=True,
-        invoke_kwds={"executor": executor, "integration_service": service},
-    )
-    for _, kls in mgr.items():
-        if (
-            kls.obj.executor_type == executor.service_name
-            and kls.obj.service_type == service_type
-            and kls.obj.service_provider == service_name
-        ):
-            logger.info(f"Identified an integration pattern {kls.obj}")
-            integrations.append(kls.obj)
-
-    # Get all the implementations defined by the runnable package
-    for kls in BaseIntegration.__subclasses__():
-        # Match the exact service type
-        if kls.service_type == service_type and kls.service_provider == service_name:
-            # Match either all executor or specific ones provided
-            if kls.executor_type == "" or kls.executor_type == executor.service_name:
-                integrations.append(kls(executor=executor, integration_service=service))
-
-    if len(integrations) > 1:
-        msg = (
-            f"Multiple integrations between {executor.service_name} and {service_name} of type {service_type} found. "
-            "If you defined an integration pattern, please ensure it is specific and does not conflict with runnable "
-            " implementations."
-        )
-        logger.exception(msg)
-        raise Exception(msg)
-
-    if not integrations:
-        logger.info(
-            f"Could not find an integration pattern for {executor.service_name} and {service_name} for {service_type}."
-            " This implies that there is no need to change the configurations."
-        )
-        return BaseIntegration(executor, service)
-
-    return integrations[0]
-
-
-def validate(executor: "BaseExecutor", service: object, **kwargs):
-    """
-    Helper function to resolve the Integration class and validate the compatibility between executor and service
-
-    Args:
-        executor (BaseExecutor) : The executor
-        service (object): The service provider
-    """
-    integration_handler = get_integration_handler(executor, service)
-    integration_handler.validate(**kwargs)
-
-
-def configure_for_traversal(executor: "BaseExecutor", service: object, **kwargs):
-    """
-    Helper function to resolve the Integration class and configure the executor and service for graph traversal
-
-    Args:
-        executor (BaseExecutor) : The executor
-        service (object): The service provider
-    """
-    integration_handler = get_integration_handler(executor, service)
-    integration_handler.configure_for_traversal(**kwargs)
-
-
-def configure_for_execution(executor: "BaseExecutor", service: object, **kwargs):
-    """
-    Helper function to resolve the Integration class and configure the executor and service for execution
-
-    Args:
-        executor (BaseExecutor) : The executor
-        service (object): The service provider
-    """
-    integration_handler = get_integration_handler(executor, service)
-    integration_handler.configure_for_execution(**kwargs)
-
-
-class BufferedRunLogStore(BaseIntegration):
-    """
-    Integration between any executor and buffered run log store
-    """
-
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "buffered"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        if not self.executor.service_name == "local":
-            raise Exception("Buffered run log store is only supported for local executor")
-
-        msg = (
-            "Run log generated by buffered run log store are not persisted. "
-            "Re-running this run, in case of a failure, is not possible"
-        )
-        logger.info(msg)
-
-
-class DoNothingCatalog(BaseIntegration):
-    """
-    Integration between any executor and do nothing catalog
-    """
-
-    service_type = "catalog"  # One of secret, catalog, datastore
-    service_provider = "do-nothing"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "A do-nothing catalog does not hold any data and therefore cannot pass data between nodes."
-        logger.info(msg)
-
-
-class DoNothingSecrets(BaseIntegration):
-    """
-    Integration between any executor and do nothing secrets
-    """
-
-    service_type = "secrets"  # One of secret, catalog, datastore
-    service_provider = "do-nothing"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "A do-nothing secrets does not hold any secrets and therefore cannot return you any secrets."
-        logger.info(msg)

runnable-0.13.0.dist-info/RECORD

@@ -1,63 +0,0 @@
-runnable/__init__.py,sha256=G5EUe1eaTOEu1UihDBP9F43PKs3yxCYFfp8DOOpPvms,1038
-runnable/catalog.py,sha256=22OECi5TrpHErxYIhfx-lJ2vgBUi4-5V9CaYEVm98hE,4138
-runnable/cli.py,sha256=RILUrEfzernuKD3dNdXPBkqN_1OgE5GosYRuInj0FVs,9618
-runnable/context.py,sha256=QhiXJHRcEBfSKB1ijvL5yB9w44x0HCe7VEiwK1cUJ9U,1124
-runnable/datastore.py,sha256=8aQZ15KAMdre7a7G61bNRmcTeJFzOdnx_9O9UP4JQc8,27910
-runnable/defaults.py,sha256=r9l3jCPlmMFNdYXOj6X1uIhruO2FyLZ8kuzus9Fd6OQ,4704
-runnable/entrypoints.py,sha256=ZdXnjEjtkOqKYyjc0AwqIdfWgxJsKFB8ilHYcUj7oZo,17630
-runnable/exceptions.py,sha256=6NIYoTAzdKyGQ9PvW1Hu7b80OS746395KiGDhM7ThH8,2526
-runnable/executor.py,sha256=xfBighQ5t_vejohip000XfxLwsgechUE1ZMIJWrZbUA,14484
-runnable/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/catalog/__init__.py,sha256=uXZ6D-Myr_J4HnBA4F5Hd7LZ0IAjQiFQYxRhMzejhQc,761
-runnable/extensions/catalog/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/catalog/file_system/implementation.py,sha256=mFPsAwPMNGWbHczpQ84o3mfkPkOEz5zjsT7a3rqNzoE,9092
-runnable/extensions/catalog/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/catalog/k8s_pvc/implementation.py,sha256=oJDDI0APT7lrtjWmzYJRDHLGn3Vhbn2MdFSRYvFBUpY,436
-runnable/extensions/catalog/k8s_pvc/integration.py,sha256=OfrHbNFN8sR-wsVa4os3ajmWJFSd5H4KOHGVAmjRZTQ,1850
-runnable/extensions/executor/__init__.py,sha256=-Xw8ZYVOfM-bYAieluEOhd98ncyrAyde8WSPRg1yFN0,26615
-runnable/extensions/executor/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/argo/implementation.py,sha256=FE5jDtzv5nlsnOjN9k0VjtEFQQWAAFaSXVJlroi7q6I,43911
-runnable/extensions/executor/argo/specification.yaml,sha256=wXQcm2gOQYqy-IOQIhucohS32ZrHKCfGA5zZ0RraPYc,1276
-runnable/extensions/executor/k8s_job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/k8s_job/implementation_FF.py,sha256=1IfVG1GRcJcVFzQ-WhkJsmzdJuj51QMxXylY9UrWM0U,10259
-runnable/extensions/executor/k8s_job/integration_FF.py,sha256=pG6HKhPMgCRIgu1PAnBvsfJQE1FxcjuSiC2I-Hn5sWo,2165
-runnable/extensions/executor/local.py,sha256=pV1tP9lRn6hwZtt-moKM1SXG_QmohbEF2QGdBbzCpkU,2396
-runnable/extensions/executor/local_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/local_container/implementation.py,sha256=6iwt9tNCQawVEfIErzoqys2hrErWK0DHcAOkO49Ov9w,17322
-runnable/extensions/executor/mocked/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/mocked/implementation.py,sha256=ChvlcLGpBxO6QwJcoqhBgKBR6NfWVnMdOWKQhMgcEjY,5762
-runnable/extensions/executor/retry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/executor/retry/implementation.py,sha256=-g6PBOhSG7IL4D_IlQOcf9H_En9IXiUzCt-6vKeCB6Q,6892
-runnable/extensions/nodes.py,sha256=Jz8uJBgvcpdZ8R63EWtzULFE_DjSwsrWgY0P1Z1PYTs,33506
-runnable/extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/chunked_file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/chunked_file_system/implementation.py,sha256=EW2P8lr3eH-pIOsMTJPr5eb-iWc48GQ97W15JzkpC_4,3326
-runnable/extensions/run_log_store/chunked_k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py,sha256=iGzy-s1eT_kAJP7XgzDLmEMOGrBLvACIiGE_wM62jGE,579
-runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py,sha256=atzdTy5HJ-bZsd6AzDP8kYRI1TshKxviBKeqY359TUs,1979
-runnable/extensions/run_log_store/db/implementation_FF.py,sha256=oEiG5ASWYYbwlBbnryKarQENB-L_yOsnZahbj2U0GdQ,5155
-runnable/extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/file_system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/file_system/implementation.py,sha256=WxxfGCaDAB5zHMM3zv9aeDwXZ4DhtyzjXOjfjvyDoZ4,4288
-runnable/extensions/run_log_store/generic_chunked.py,sha256=PtYK1dheKYdxODwu_ygpGRIHIepgLVaIORSqvsrg0No,19876
-runnable/extensions/run_log_store/k8s_pvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/run_log_store/k8s_pvc/implementation.py,sha256=tLgXy9HUB_vlFVQ0Itk6PpNU3GlCOILN4vA3fm80jXI,542
-runnable/extensions/run_log_store/k8s_pvc/integration.py,sha256=lxQg327mwC0ykhNp5Kg34a9g8o1DzJAhfkiqMGmsABs,1873
-runnable/extensions/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/secrets/dotenv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runnable/extensions/secrets/dotenv/implementation.py,sha256=3J5pofWahdZbnwnETwpspE5-PKyvmZF_vkfwA1X_bkA,3365
-runnable/graph.py,sha256=xJ5Q6Rv5T87wOBzQ1eIUvV0Iynnp7sRrZaIeytcDRlA,16313
-runnable/integration.py,sha256=eb9qJVZR7Ehg0N1UnGPuyjJvoA-xQ1-xP7AlZHUXHqM,6705
-runnable/names.py,sha256=vn92Kv9ANROYSZX6Z4z1v_WA3WiEdIYmG6KEStBFZug,8134
-runnable/nodes.py,sha256=UqR-bJx0Hi7uLSUw_saB7VsNdFh3POKtdgsEPsasHfE,16576
-runnable/parameters.py,sha256=yZkMDnwnkdYXIwQ8LflBzn50Y0xRGxEvLlxwno6ovvs,5163
-runnable/pickler.py,sha256=5SDNf0miMUJ3ZauhQdzwk8_t-9jeOqaTjP5bvRnu9sU,2685
-runnable/sdk.py,sha256=sZWiyGVIqaAmmJuc4p1OhuqLBSqS7svN1VuSOqeZ_xA,29254
-runnable/secrets.py,sha256=dakb7WRloWVo-KpQp6Vy4rwFdGi58BTlT4OifQY106I,2324
-runnable/tasks.py,sha256=61BYi5DNyTPi8GRlIy8vh2T0DZ1jkTj7gxBx88sK_XY,28058
-runnable/utils.py,sha256=fXOLoFZKYqh3wQgzA2V-VZOu-dSgLPGqCZIbMmsNzOw,20016
-runnable-0.13.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.13.0.dist-info/METADATA,sha256=p4fwokQJKU8EJfniOP_DW22yDgSSZmltZDP2reYlBck,10436
-runnable-0.13.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-runnable-0.13.0.dist-info/entry_points.txt,sha256=cl_K9XPV-qnAVaXfmNuGpZZKaoUu8YwjPsqZW1iSKlo,1479
-runnable-0.13.0.dist-info/RECORD,,

runnable-0.13.0.dist-info/entry_points.txt

@@ -1,41 +0,0 @@
-[catalog]
-do-nothing=runnable.catalog:DoNothingCatalog
-file-system=runnable.extensions.catalog.file_system.implementation:FileSystemCatalog
-
-[console_scripts]
-runnable=runnable.cli:cli
-
-[executor]
-argo=runnable.extensions.executor.argo.implementation:ArgoExecutor
-local=runnable.extensions.executor.local:LocalExecutor
-local-container=runnable.extensions.executor.local_container.implementation:LocalContainerExecutor
-mocked=runnable.extensions.executor.mocked.implementation:MockedExecutor
-retry=runnable.extensions.executor.retry.implementation:RetryExecutor
-
-[nodes]
-dag=runnable.extensions.nodes:DagNode
-fail=runnable.extensions.nodes:FailNode
-map=runnable.extensions.nodes:MapNode
-parallel=runnable.extensions.nodes:ParallelNode
-stub=runnable.extensions.nodes:StubNode
-success=runnable.extensions.nodes:SuccessNode
-task=runnable.extensions.nodes:TaskNode
-
-[pickler]
-pickle=runnable.pickler:NativePickler
-
-[run_log_store]
-buffered=runnable.datastore:BufferRunLogstore
-chunked-fs=runnable.extensions.run_log_store.chunked_file_system.implementation:ChunkedFileSystemRunLogStore
-file-system=runnable.extensions.run_log_store.file_system.implementation:FileSystemRunLogstore
-
-[secrets]
-do-nothing=runnable.secrets:DoNothingSecretManager
-dotenv=runnable.extensions.secrets.dotenv.implementation:DotEnvSecrets
-env-secrets=runnable.secrets:EnvSecretsManager
-
-[tasks]
-notebook=runnable.tasks:NotebookTaskType
-python=runnable.tasks:PythonTaskType
-shell=runnable.tasks:ShellTaskType
-