runnable 0.13.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/extensions/run_log_store/chunked_file_system/implementation.py

@@ -1,111 +0,0 @@
-import json
-import logging
-from pathlib import Path
-from string import Template
-from typing import Any, Dict, Optional, Sequence, Union
-
-from runnable import defaults, utils
-from runnable.extensions.run_log_store.generic_chunked import ChunkedRunLogStore
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-T = Union[str, Path]
-
-
-class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
-    """
-    File system run log store but chunks the run log into thread safe chunks.
-    This enables executions to be parallel.
-    """
-
-    service_name: str = "chunked-fs"
-    log_folder: str = defaults.LOG_LOCATION_FOLDER
-
-    def get_summary(self) -> Dict[str, Any]:
-        summary = {"Type": self.service_name, "Location": self.log_folder}
-
-        return summary
-
-    def get_matches(self, run_id: str, name: str, multiple_allowed: bool = False) -> Optional[Union[Sequence[T], T]]:
-        """
-        Get contents of files matching the pattern name*
-
-        Args:
-            run_id (str): The run id
-            name (str): The suffix of the file name to check in the run log store.
-        """
-        log_folder = self.log_folder_with_run_id(run_id=run_id)
-        sub_name = Template(name).safe_substitute({"creation_time": ""})
-
-        matches = list(log_folder.glob(f"{sub_name}*"))
-
-        if matches:
-            if not multiple_allowed:
-                if len(matches) > 1:
-                    msg = f"Multiple matches found for {name} while multiple is not allowed"
-                    raise Exception(msg)
-                return matches[0]
-            return matches
-
-        return None
-
-    def log_folder_with_run_id(self, run_id: str) -> Path:
-        """
-        Utility function to get the log folder for a run id.
-
-        Args:
-            run_id (str): The run id
-
-        Returns:
-            Path: The path to the log folder with the run id
-        """
-        return Path(self.log_folder) / run_id
-
-    def safe_suffix_json(self, name: Union[Path, str]) -> str:
-        """
-        Safely attach a suffix to a json file.
-
-        Args:
-            name (Path): The name of the file with or without suffix of json
-
-        Returns:
-            str : The name of the file with .json
-        """
-        if str(name).endswith("json"):
-            return str(name)
-
-        return str(name) + ".json"
-
-    def _store(self, run_id: str, contents: dict, name: Union[Path, str], insert=False):
-        """
-        Store the contents against the name in the folder.
-
-        Args:
-            run_id (str): The run id
-            contents (dict): The dict to store
-            name (str): The name to store as
-        """
-        if insert:
-            name = self.log_folder_with_run_id(run_id=run_id) / name
-
-        utils.safe_make_dir(self.log_folder_with_run_id(run_id=run_id))
-
-        with open(self.safe_suffix_json(name), "w") as fw:
-            json.dump(contents, fw, ensure_ascii=True, indent=4)
-
-    def _retrieve(self, name: Union[str, Path]) -> dict:
-        """
-        Does the job of retrieving from the folder.
-
-        Args:
-            name (str): the name of the file to retrieve
-
-        Returns:
-            dict: The contents
-        """
-        contents: dict = {}
-
-        with open(self.safe_suffix_json(name), "r") as fr:
-            contents = json.load(fr)
-
-        return contents

runnable/extensions/run_log_store/chunked_k8s_pvc/implementation.py

@@ -1,21 +0,0 @@
-import logging
-from pathlib import Path
-
-from runnable import defaults
-from runnable.extensions.run_log_store.chunked_file_system.implementation import ChunkedFileSystemRunLogStore
-
-logger = logging.getLogger(defaults.NAME)
-
-
-class ChunkedK8PersistentVolumeRunLogstore(ChunkedFileSystemRunLogStore):
-    """
-    Uses the K8s Persistent Volumes to store run logs.
-    """
-
-    service_name: str = "chunked-k8s-pvc"
-    persistent_volume_name: str
-    mount_path: str
-
-    @property
-    def log_folder_name(self) -> str:
-        return str(Path(self.mount_path) / self.log_folder)

runnable/extensions/run_log_store/chunked_k8s_pvc/integration.py

@@ -1,61 +0,0 @@
-import logging
-from typing import cast
-
-from runnable import defaults
-from runnable.integration import BaseIntegration
-
-logger = logging.getLogger(defaults.NAME)
-
-
-class LocalCompute(BaseIntegration):
-    """
-    Integration between local and k8's pvc
-    """
-
-    executor_type = "local"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "chunked-k8s-pvc"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "We can't use the local compute k8s pvc store integration."
-        raise Exception(msg)
-
-
-class LocalContainerCompute(BaseIntegration):
-    """
-    Integration between local-container and k8's pvc
-    """
-
-    executor_type = "local-container"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "chunked-k8s-pvc"  # The actual implementation of the service
-
-    def validate(self, **kwargs):
-        msg = "We can't use the local-container compute k8s pvc store integration."
-        raise Exception(msg)
-
-
-class ArgoCompute(BaseIntegration):
-    """
-    Integration between argo and k8's pvc
-    """
-
-    executor_type = "argo"
-    service_type = "run_log_store"  # One of secret, catalog, datastore
-    service_provider = "chunked-k8s-pvc"  # The actual implementation of the service
-
-    def configure_for_traversal(self, **kwargs):
-        from runnable.extensions.executor.argo.implementation import ArgoExecutor, UserVolumeMounts
-        from runnable.extensions.run_log_store.chunked_k8s_pvc.implementation import (
-            ChunkedK8PersistentVolumeRunLogstore,
-        )
-
-        self.executor = cast(ArgoExecutor, self.executor)
-        self.service = cast(ChunkedK8PersistentVolumeRunLogstore, self.service)
-
-        volume_mount = UserVolumeMounts(
-            name=self.service.persistent_volume_name,
-            mount_path=self.service.mount_path,
-        )
-
-        self.executor.persistent_volumes.append(volume_mount)

runnable/extensions/run_log_store/db/implementation_FF.py

@@ -1,157 +0,0 @@
-import datetime
-import json
-import logging
-from pathlib import Path
-from string import Template
-from typing import Any, Dict, List, Optional, Union, cast
-
-from runnable import defaults, utils
-from runnable.extensions.run_log_store.generic_chunked import ChunkedRunLogStore
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class DBRunLogStore(ChunkedRunLogStore):
-    """
-    File system run log store but chunks the run log into thread safe chunks.
-    This enables executions to be parallel.
-    """
-
-    service_name: str = "chunked-fs"
-    connection_string: str
-    db_name: str
-
-    _DB_LOG: Any = None
-    _engine: Any = None
-    _session: Any = None
-    _connection_string: str = ""
-    _base: Any = None
-
-    def model_post_init(self, _: Any) -> None:
-        run_context = self._context
-
-        secrets = cast(Dict[str, str], run_context.secrets_handler.get())
-        connection_string = Template(self.connection_string).safe_substitute(**secrets)
-
-        try:
-            import sqlalchemy
-            from sqlalchemy import Column, DateTime, Integer, Sequence, Text
-            from sqlalchemy.orm import declarative_base, sessionmaker
-
-            Base = declarative_base()
-
-            class DBLog(Base):
-                """
-                Base table for storing run logs in database.
-
-                In this model, we fragment the run log into logical units that are concurrent safe.
-                """
-
-                __tablename__ = self.db_name
-                pk = Column(Integer, Sequence("id_seq"), primary_key=True)
-                run_id = Column(Text, index=True)
-                attribute_key = Column(Text)  # run_log, step_internal_name, parameter_key etc
-                attribute_type = Column(Text)  # RunLog, Step, Branch, Parameter
-                attribute_value = Column(Text)  # The JSON string
-                created_at = Column(DateTime, default=datetime.datetime.utcnow)
-
-            self._engine = sqlalchemy.create_engine(connection_string, pool_pre_ping=True)
-            self._session = sessionmaker(bind=self._engine)
-            self._DB_LOG = DBLog
-            self._connection_string = connection_string
-            self._base = Base
-
-        except ImportError as _e:
-            logger.exception("Unable to import SQLalchemy, is it installed?")
-            msg = "SQLAlchemy is required for this extension. Please install it"
-            raise Exception(msg) from _e
-
-    def create_tables(self):
-        import sqlalchemy
-
-        engine = sqlalchemy.create_engine(self._connection_string)
-        self._base.metadata.create_all(engine)
-
-    def get_matches(self, run_id: str, name: str, multiple_allowed: bool = False) -> Optional[Union[List[Path], Path]]:
-        """
-        Get contents of files matching the pattern name*
-
-        Args:
-            run_id (str): The run id
-            name (str): The suffix of the file name to check in the run log store.
-        """
-        log_folder = self.log_folder_with_run_id(run_id=run_id)
-
-        sub_name = Template(name).safe_substitute({"creation_time": ""})
-
-        matches = list(log_folder.glob(f"{sub_name}*"))
-        if matches:
-            if not multiple_allowed:
-                if len(matches) > 1:
-                    msg = f"Multiple matches found for {name} while multiple is not allowed"
-                    raise Exception(msg)
-                return matches[0]
-            return matches
-
-        return None
-
-    def log_folder_with_run_id(self, run_id: str) -> Path:
-        """
-        Utility function to get the log folder for a run id.
-
-        Args:
-            run_id (str): The run id
-
-        Returns:
-            Path: The path to the log folder with the run id
-        """
-        return Path(self.log_folder) / run_id
-
-    def safe_suffix_json(self, name: Union[Path, str]) -> str:
-        """
-        Safely attach a suffix to a json file.
-
-        Args:
-            name (Path): The name of the file with or without suffix of json
-
-        Returns:
-            str : The name of the file with .json
-        """
-        if str(name).endswith("json"):
-            return str(name)
-
-        return str(name) + ".json"
-
-    def _store(self, run_id: str, contents: dict, name: Union[Path, str], insert=False):
-        """
-        Store the contents against the name in the folder.
-
-        Args:
-            run_id (str): The run id
-            contents (dict): The dict to store
-            name (str): The name to store as
-        """
-        if insert:
-            name = self.log_folder_with_run_id(run_id=run_id) / name
-
-        utils.safe_make_dir(self.log_folder_with_run_id(run_id=run_id))
-
-        with open(self.safe_suffix_json(name), "w") as fw:
-            json.dump(contents, fw, ensure_ascii=True, indent=4)
-
-    def _retrieve(self, name: Path) -> dict:
-        """
-        Does the job of retrieving from the folder.
-
-        Args:
-            name (str): the name of the file to retrieve
-
-        Returns:
-            dict: The contents
-        """
-        contents: dict = {}
-
-        with open(self.safe_suffix_json(name), "r") as fr:
-            contents = json.load(fr)
-
-        return contents

runnable/extensions/run_log_store/file_system/implementation.py

@@ -1,140 +0,0 @@
-import json
-import logging
-from pathlib import Path
-from typing import Any, Dict
-
-from runnable import defaults, exceptions, utils
-from runnable.datastore import BaseRunLogStore, RunLog
-
-logger = logging.getLogger(defaults.LOGGER_NAME)
-
-
-class FileSystemRunLogstore(BaseRunLogStore):
-    """
-    In this type of Run Log store, we use a file system to store the JSON run log.
-
-    Every single run is stored as a different file which makes it compatible across other store types.
-
-    When to use:
-        When locally testing a pipeline and have the need to compare across runs.
-        Its fully featured and perfectly fine if your local environment is where you would do everything.
-
-    Do not use:
-        If you need parallelization on local, this run log would not support it.
-
-    Example config:
-
-    run_log:
-      type: file-system
-      config:
-        log_folder: The folder to out the logs. Defaults to .run_log_store
-
-    """
-
-    service_name: str = "file-system"
-    log_folder: str = defaults.LOG_LOCATION_FOLDER
-
-    @property
-    def log_folder_name(self):
-        return self.log_folder
-
-    def get_summary(self) -> Dict[str, Any]:
-        summary = {"Type": self.service_name, "Location": self.log_folder}
-
-        return summary
-
-    def write_to_folder(self, run_log: RunLog):
-        """
-        Write the run log to the folder
-
-        Args:
-            run_log (RunLog): The run log to be added to the database
-        """
-        write_to = self.log_folder_name
-        utils.safe_make_dir(write_to)
-
-        write_to_path = Path(write_to)
-        run_id = run_log.run_id
-        json_file_path = write_to_path / f"{run_id}.json"
-
-        with json_file_path.open("w") as fw:
-            json.dump(run_log.model_dump(), fw, ensure_ascii=True, indent=4)  # pylint: disable=no-member
-
-    def get_from_folder(self, run_id: str) -> RunLog:
-        """
-        Look into the run log folder for the run log for the run id.
-
-        If the run log does not exist, raise an exception. If it does, decode it
-        as a RunLog and return it
-
-        Args:
-            run_id (str): The requested run id to retrieve the run log store
-
-        Raises:
-            FileNotFoundError: If the Run Log has not been found.
-
-        Returns:
-            RunLog: The decoded Run log
-        """
-        write_to = self.log_folder_name
-
-        read_from_path = Path(write_to)
-        json_file_path = read_from_path / f"{run_id}.json"
-
-        if not json_file_path.exists():
-            raise FileNotFoundError(f"Expected {json_file_path} is not present")
-
-        with json_file_path.open("r") as fr:
-            json_str = json.load(fr)
-            run_log = RunLog(**json_str)  # pylint: disable=no-member
-        return run_log
-
-    def create_run_log(
-        self,
-        run_id: str,
-        dag_hash: str = "",
-        use_cached: bool = False,
-        tag: str = "",
-        original_run_id: str = "",
-        status: str = defaults.CREATED,
-        **kwargs,
-    ) -> RunLog:
-        """
-        # Creates a Run log
-        # Adds it to the db
-        """
-
-        try:
-            self.get_run_log_by_id(run_id=run_id, full=False)
-            raise exceptions.RunLogExistsError(run_id=run_id)
-        except exceptions.RunLogNotFoundError:
-            pass
-
-        logger.info(f"{self.service_name} Creating a Run Log for : {run_id}")
-        run_log = RunLog(
-            run_id=run_id,
-            dag_hash=dag_hash,
-            tag=tag,
-            status=status,
-        )
-        self.write_to_folder(run_log)
-        return run_log
-
-    def get_run_log_by_id(self, run_id: str, full: bool = False, **kwargs) -> RunLog:
-        """
-        # Returns the run_log defined by id
-        # Raises Exception if not found
-        """
-        try:
-            logger.info(f"{self.service_name} Getting a Run Log for : {run_id}")
-            run_log = self.get_from_folder(run_id)
-            return run_log
-        except FileNotFoundError as e:
-            raise exceptions.RunLogNotFoundError(run_id) from e
-
-    def put_run_log(self, run_log: RunLog, **kwargs):
-        """
-        # Puts the run_log into the database
-        """
-        logger.info(f"{self.service_name} Putting the run log in the DB: {run_log.run_id}")
-        self.write_to_folder(run_log)