runnable 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

extensions/run_log_store/any_path.py

@@ -0,0 +1,104 @@
+import logging
+from abc import abstractmethod
+from typing import Any, Dict
+
+from runnable import defaults, exceptions
+from runnable.datastore import BaseRunLogStore, RunLog
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class AnyPathRunLogStore(BaseRunLogStore):
+    """
+    In this type of Run Log store, we use a file system to store the JSON run log.
+
+    Every single run is stored as a different file which makes it compatible across other store types.
+
+    When to use:
+        When locally testing a pipeline and have the need to compare across runs.
+        Its fully featured and perfectly fine if your local environment is where you would do everything.
+
+    Do not use:
+        If you need parallelization on local, this run log would not support it.
+
+    Example config:
+
+    run_log:
+      type: file-system
+      config:
+        log_folder: The folder to out the logs. Defaults to .run_log_store
+
+    """
+
+    service_name: str = "file-system"
+    log_folder: str = defaults.LOG_LOCATION_FOLDER
+
+    @property
+    def log_folder_name(self):
+        return self.log_folder
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {"Type": self.service_name, "Location": self.log_folder}
+
+        return summary
+
+    @abstractmethod
+    def write_to_path(self, run_log: RunLog): ...
+
+    @abstractmethod
+    def read_from_path(self, run_id: str) -> RunLog: ...
+
+    def create_run_log(
+        self,
+        run_id: str,
+        dag_hash: str = "",
+        use_cached: bool = False,
+        tag: str = "",
+        original_run_id: str = "",
+        status: str = defaults.CREATED,
+    ) -> RunLog:
+        """
+        # Creates a Run log
+        # Adds it to the db
+        """
+
+        try:
+            self.get_run_log_by_id(run_id=run_id, full=False)
+            raise exceptions.RunLogExistsError(run_id=run_id)
+        except exceptions.RunLogNotFoundError:
+            pass
+
+        logger.info(f"{self.service_name} Creating a Run Log for : {run_id}")
+        run_log = RunLog(
+            run_id=run_id,
+            dag_hash=dag_hash,
+            tag=tag,
+            status=status,
+        )
+        self.write_to_path(run_log)
+        return run_log
+
+    def get_run_log_by_id(
+        self,
+        run_id: str,
+        full: bool = False,
+    ) -> RunLog:
+        """
+        # Returns the run_log defined by id
+        # Raises Exception if not found
+        """
+        try:
+            logger.info(f"{self.service_name} Getting a Run Log for : {run_id}")
+            run_log = self.read_from_path(run_id)
+            return run_log
+        except FileNotFoundError as e:
+            raise exceptions.RunLogNotFoundError(run_id) from e
+
+    def put_run_log(self, run_log: RunLog):
+        """
+        # Puts the run_log into the database
+        """
+        logger.info(
+            f"{self.service_name} Putting the run log in the DB: {run_log.run_id}"
+        )
+        self.write_to_path(run_log)

extensions/run_log_store/chunked_fs.py

@@ -2,14 +2,16 @@ import json
 import logging
 from pathlib import Path
 from string import Template
-from typing import Any, Dict, Optional, Sequence, Union
+from typing import Any, Dict, Optional, Union
+
+from cloudpathlib import CloudPath
 
 from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
 from runnable import defaults, utils
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
-T = Union[str, Path]
+MixT = Union[CloudPath, Path]
 
 
 class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
@@ -28,7 +30,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
 
     def get_matches(
         self, run_id: str, name: str, multiple_allowed: bool = False
-    ) -> Optional[Union[Sequence[T], T]]:
+    ) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
         """
         Get contents of files matching the pattern name*
 
@@ -78,7 +80,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
 
         return str(name) + ".json"
 
-    def _store(self, run_id: str, contents: dict, name: Union[Path, str], insert=False):
+    def _store(self, run_id: str, contents: dict, name: MixT, insert=False):
         """
         Store the contents against the name in the folder.
 
@@ -87,15 +89,16 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
             contents (dict): The dict to store
             name (str): The name to store as
         """
+        log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
         if insert:
-            name = self.log_folder_with_run_id(run_id=run_id) / name
+            name = log_folder_with_run_id / name
 
-        utils.safe_make_dir(self.log_folder_with_run_id(run_id=run_id))
+        utils.safe_make_dir(log_folder_with_run_id)
 
-        with open(self.safe_suffix_json(name), "w") as fw:
+        with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "w") as fw:
             json.dump(contents, fw, ensure_ascii=True, indent=4)
 
-    def _retrieve(self, name: Union[str, Path]) -> dict:
+    def _retrieve(self, run_id: str, name: MixT) -> dict:
         """
         Does the job of retrieving from the folder.
 
@@ -106,8 +109,9 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
             dict: The contents
         """
         contents: dict = {}
+        log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
 
-        with open(self.safe_suffix_json(name), "r") as fr:
+        with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "r") as fr:
             contents = json.load(fr)
 
         return contents

extensions/run_log_store/file_system.py

@@ -3,13 +3,14 @@ import logging
 from pathlib import Path
 from typing import Any, Dict
 
-from runnable import defaults, exceptions, utils
-from runnable.datastore import BaseRunLogStore, RunLog
+from extensions.run_log_store.any_path import AnyPathRunLogStore
+from runnable import defaults, utils
+from runnable.datastore import RunLog
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-class FileSystemRunLogstore(BaseRunLogStore):
+class FileSystemRunLogstore(AnyPathRunLogStore):
     """
     In this type of Run Log store, we use a file system to store the JSON run log.
 
@@ -43,7 +44,7 @@ class FileSystemRunLogstore(BaseRunLogStore):
 
         return summary
 
-    def write_to_folder(self, run_log: RunLog):
+    def write_to_path(self, run_log: RunLog):
         """
         Write the run log to the folder
 
@@ -60,7 +61,7 @@ class FileSystemRunLogstore(BaseRunLogStore):
         with json_file_path.open("w") as fw:
             json.dump(run_log.model_dump(), fw, ensure_ascii=True, indent=4)  # pylint: disable=no-member
 
-    def get_from_folder(self, run_id: str) -> RunLog:
+    def read_from_path(self, run_id: str) -> RunLog:
         """
         Look into the run log folder for the run log for the run id.
 
@@ -88,58 +89,3 @@ class FileSystemRunLogstore(BaseRunLogStore):
             json_str = json.load(fr)
             run_log = RunLog(**json_str)  # pylint: disable=no-member
         return run_log
-
-    def create_run_log(
-        self,
-        run_id: str,
-        dag_hash: str = "",
-        use_cached: bool = False,
-        tag: str = "",
-        original_run_id: str = "",
-        status: str = defaults.CREATED,
-    ) -> RunLog:
-        """
-        # Creates a Run log
-        # Adds it to the db
-        """
-
-        try:
-            self.get_run_log_by_id(run_id=run_id, full=False)
-            raise exceptions.RunLogExistsError(run_id=run_id)
-        except exceptions.RunLogNotFoundError:
-            pass
-
-        logger.info(f"{self.service_name} Creating a Run Log for : {run_id}")
-        run_log = RunLog(
-            run_id=run_id,
-            dag_hash=dag_hash,
-            tag=tag,
-            status=status,
-        )
-        self.write_to_folder(run_log)
-        return run_log
-
-    def get_run_log_by_id(
-        self,
-        run_id: str,
-        full: bool = False,
-    ) -> RunLog:
-        """
-        # Returns the run_log defined by id
-        # Raises Exception if not found
-        """
-        try:
-            logger.info(f"{self.service_name} Getting a Run Log for : {run_id}")
-            run_log = self.get_from_folder(run_id)
-            return run_log
-        except FileNotFoundError as e:
-            raise exceptions.RunLogNotFoundError(run_id) from e
-
-    def put_run_log(self, run_log: RunLog):
-        """
-        # Puts the run_log into the database
-        """
-        logger.info(
-            f"{self.service_name} Putting the run log in the DB: {run_log.run_id}"
-        )
-        self.write_to_folder(run_log)

extensions/run_log_store/generic_chunked.py

@@ -4,7 +4,9 @@ from abc import abstractmethod
 from enum import Enum
 from pathlib import Path
 from string import Template
-from typing import Any, Dict, Optional, Sequence, Union
+from typing import Any, Dict, Optional, Union
+
+from cloudpathlib import CloudPath
 
 from runnable import defaults, exceptions
 from runnable.datastore import (
@@ -21,7 +23,7 @@ from runnable.datastore import (
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-T = Union[str, Path]  # Holds str, path
+MixT = Union[CloudPath, Path]  # Holds str, path
 
 
 class EntityNotFoundError(Exception):
@@ -87,7 +89,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
     @abstractmethod
     def get_matches(
         self, run_id: str, name: str, multiple_allowed: bool = False
-    ) -> Optional[Union[Sequence[T], T]]:
+    ) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
         """
         Get contents of persistence layer matching the pattern name*
 
@@ -98,7 +100,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         ...
 
     @abstractmethod
-    def _store(self, run_id: str, contents: dict, name: T, insert: bool = False):
+    def _store(self, run_id: str, contents: dict, name: MixT, insert: bool = False):
         """
         Store the contents against the name in the persistence layer.
 
@@ -110,7 +112,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         ...
 
     @abstractmethod
-    def _retrieve(self, name: T) -> dict:
+    def _retrieve(self, run_id: str, name: MixT) -> dict:
         """
         Does the job of retrieving from the persistent layer.
 
@@ -140,7 +142,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         insert = False
 
         if match:
-            existing_contents = self._retrieve(name=match)  # type: ignore
+            existing_contents = self._retrieve(run_id=run_id, name=match)  # type: ignore
             contents = dict(existing_contents, **contents)
             name_to_give = match  # type: ignore
         else:
@@ -149,7 +151,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
             )
             insert = True
 
-        self._store(run_id=run_id, contents=contents, name=name_to_give, insert=insert)
+        self._store(
+            run_id=run_id, contents=contents, name=Path(name_to_give), insert=insert
+        )
 
     def retrieve(
         self, run_id: str, log_type: LogTypes, name: str = "", multiple_allowed=False
@@ -190,13 +194,13 @@ class ChunkedRunLogStore(BaseRunLogStore):
 
         if matches:
             if not multiple_allowed:
-                contents = self._retrieve(name=matches)  # type: ignore
+                contents = self._retrieve(run_id=run_id, name=matches)  # type: ignore
                 model = self.ModelTypes[log_type.name].value
                 return model(**contents)
 
             models = []
             for match in matches:  # type: ignore
-                contents = self._retrieve(name=match)
+                contents = self._retrieve(run_id=run_id, name=match)
                 model = self.ModelTypes[log_type.name].value
                 models.append(model(**contents))
             return models
@@ -225,7 +229,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
             # No branch logs are found
             return {}
         # Forcing get_matches to always return a list is a better design
-        epoch_created = [str(match).split("-")[-1] for match in matches]  # type: ignore
+
+        assert isinstance(matches, list)
+        epoch_created = [str(match).split("-")[-1] for match in matches]
 
         # sort matches by epoch created
         epoch_created, matches = zip(*sorted(zip(epoch_created, matches)))  # type: ignore
@@ -234,7 +240,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
 
         for match in matches:
             model = self.ModelTypes[log_type.name].value
-            log_model = model(**self._retrieve(match))
+            log_model = model(**self._retrieve(run_id=run_id, name=match))
             logs[log_model.internal_name] = log_model  # type: ignore
 
         return logs

extensions/run_log_store/minio.py

@@ -0,0 +1,111 @@
+import json
+import logging
+from functools import lru_cache
+from typing import Any, Dict
+
+from cloudpathlib import S3Client, S3Path
+from pydantic import Field, SecretStr
+
+from extensions.run_log_store.any_path import AnyPathRunLogStore
+from runnable import defaults
+from runnable.datastore import RunLog
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+@lru_cache
+def get_minio_client(
+    endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str
+) -> S3Client:
+    return S3Client(
+        endpoint_url=endpoint_url,
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+    )
+
+
+class MinioRunLogStore(AnyPathRunLogStore):
+    """
+    In this type of Run Log store, we use a file system to store the JSON run log.
+
+    Every single run is stored as a different file which makes it compatible across other store types.
+
+    When to use:
+        When locally testing a pipeline and have the need to compare across runs.
+        Its fully featured and perfectly fine if your local environment is where you would do everything.
+
+    Do not use:
+        If you need parallelization on local, this run log would not support it.
+
+    Example config:
+
+    run_log:
+      type: file-system
+      config:
+        log_folder: The folder to out the logs. Defaults to .run_log_store
+
+    """
+
+    service_name: str = "file-system"
+
+    endpoint_url: str = Field(default="http://localhost:9002")
+    aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
+    aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
+    bucket: str = Field(default="runnable/run-logs")
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {"Type": self.service_name, "Location": self.log_folder}
+
+        return summary
+
+    def get_run_log_bucket(self) -> S3Path:
+        run_id = self._context.run_id
+
+        return S3Path(
+            f"s3://{self.bucket}/{run_id}/",
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+    def write_to_path(self, run_log: RunLog):
+        """
+        Write the run log to the folder
+
+        Args:
+            run_log (RunLog): The run log to be added to the database
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        run_log_bucket.mkdir(parents=True, exist_ok=True)
+
+        run_log_object = run_log_bucket / f"{run_log.run_id}.json"
+        run_log_object.write_text(
+            json.dumps(run_log.model_dump_json(), ensure_ascii=True, indent=4)
+        )
+
+    def read_from_path(self, run_id: str) -> RunLog:
+        """
+        Look into the run log folder for the run log for the run id.
+
+        If the run log does not exist, raise an exception. If it does, decode it
+        as a RunLog and return it
+
+        Args:
+            run_id (str): The requested run id to retrieve the run log store
+
+        Raises:
+            FileNotFoundError: If the Run Log has not been found.
+
+        Returns:
+            RunLog: The decoded Run log
+        """
+        run_log_bucket = self.get_run_log_bucket()
+
+        run_log_object = run_log_bucket / f"{run_id}.json"
+
+        run_log_text = json.loads(run_log_object.read_text())
+        run_log = RunLog(**json.loads(run_log_text))
+
+        return run_log

runnable/catalog.py

@@ -2,7 +2,7 @@ import logging
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 
 import runnable.context as context
 from runnable import defaults
@@ -43,6 +43,9 @@ class BaseCatalog(ABC, BaseModel):
 
     service_name: str = ""
     service_type: str = "catalog"
+
+    compute_data_folder: str = Field(default=defaults.COMPUTE_DATA_FOLDER)
+
     model_config = ConfigDict(extra="forbid")
 
     @abstractmethod
@@ -52,14 +55,8 @@ class BaseCatalog(ABC, BaseModel):
     def _context(self):
         return context.run_context
 
-    @property
-    def compute_data_folder(self) -> str:
-        return defaults.COMPUTE_DATA_FOLDER
-
     @abstractmethod
-    def get(
-        self, name: str, run_id: str, compute_data_folder: str = "", **kwargs
-    ) -> List[DataCatalog]:
+    def get(self, name: str) -> List[DataCatalog]:
         """
         Get the catalog item by 'name' for the 'run id' and store it in compute data folder.
 
@@ -79,14 +76,7 @@ class BaseCatalog(ABC, BaseModel):
         raise NotImplementedError
 
     @abstractmethod
-    def put(
-        self,
-        name: str,
-        run_id: str,
-        compute_data_folder: str = "",
-        synced_catalogs: Optional[List[DataCatalog]] = None,
-        **kwargs,
-    ) -> List[DataCatalog]:
+    def put(self, name: str) -> List[DataCatalog]:
         """
         Put the file by 'name' from the 'compute_data_folder' in the catalog for the run_id.
 
@@ -140,23 +130,14 @@ class DoNothingCatalog(BaseCatalog):
     def get_summary(self) -> Dict[str, Any]:
         return {}
 
-    def get(
-        self, name: str, run_id: str, compute_data_folder: str = "", **kwargs
-    ) -> List[DataCatalog]:
+    def get(self, name: str) -> List[DataCatalog]:
         """
         Does nothing
         """
         logger.info("Using a do-nothing catalog, doing nothing in get")
         return []
 
-    def put(
-        self,
-        name: str,
-        run_id: str,
-        compute_data_folder: str = "",
-        synced_catalogs: Optional[List[DataCatalog]] = None,
-        **kwargs,
-    ) -> List[DataCatalog]:
+    def put(self, name: str) -> List[DataCatalog]:
         """
         Does nothing
         """
@@ -168,4 +149,3 @@ class DoNothingCatalog(BaseCatalog):
         Does nothing
         """
         logger.info("Using a do-nothing catalog, doing nothing while sync between runs")
-        logger.info("Using a do-nothing catalog, doing nothing while sync between runs")

runnable/datastore.py

@@ -114,7 +114,7 @@ class ObjectParameter(BaseModel):
 
         # If the object was serialised, get it from the catalog
         catalog_handler = context.run_context.catalog_handler
-        catalog_handler.get(name=self.file_name, run_id=context.run_context.run_id)
+        catalog_handler.get(name=self.file_name)
         obj = context.run_context.pickler.load(path=self.file_name)
         os.remove(self.file_name)  # Remove after loading
         return obj
@@ -128,7 +128,7 @@ class ObjectParameter(BaseModel):
         context.run_context.pickler.dump(data=data, path=self.file_name)
 
         catalog_handler = context.run_context.catalog_handler
-        catalog_handler.put(name=self.file_name, run_id=context.run_context.run_id)
+        catalog_handler.put(name=self.file_name)
         os.remove(self.file_name)  # Remove after loading
 
 

runnable/executor.py

@@ -173,23 +173,6 @@ class BasePipelineExecutor(BaseExecutor):
         """
         ...
 
-    @abstractmethod
-    def get_effective_compute_data_folder(self) -> Optional[str]:
-        """
-        Get the effective compute data folder for the given stage.
-        If there is nothing to catalog, we return None.
-
-        The default is the compute data folder of the catalog but this can be over-ridden by the node.
-
-        Args:
-            stage (str): The stage we are in the process of cataloging
-
-
-        Returns:
-            Optional[str]: The compute data folder as defined by catalog handler or the node or None.
-        """
-        ...
-
     @abstractmethod
     def _sync_catalog(
         self, stage: str, synced_catalogs=None

runnable/tasks.py

@@ -501,9 +501,7 @@ class NotebookTaskType(BaseTaskType):
                     pm.execute_notebook(**kwds)
                 task_console.print(out_file.getvalue())
 
-                context.run_context.catalog_handler.put(
-                    name=notebook_output_path, run_id=context.run_context.run_id
-                )
+                context.run_context.catalog_handler.put(name=notebook_output_path)
 
                 client = PloomberClient.from_path(path=notebook_output_path)
                 namespace = client.get_namespace()

runnable/utils.py

@@ -359,26 +359,26 @@ def diff_dict(d1: Dict[str, Any], d2: Dict[str, Any]) -> Dict[str, Any]:
     return diff
 
 
-def hash_bytestr_iter(bytesiter, hasher, ashexstr=True):  # pylint: disable=C0116
-    """Hashes the given bytesiter using the given hasher."""
-    for block in bytesiter:  # pragma: no cover
-        hasher.update(block)
-    return hasher.hexdigest() if ashexstr else hasher.digest()  # pragma: no cover
+# def hash_bytestr_iter(bytesiter, hasher, ashexstr=True):  # pylint: disable=C0116
+#     """Hashes the given bytesiter using the given hasher."""
+#     for block in bytesiter:  # pragma: no cover
+#         hasher.update(block)
+#     return hasher.hexdigest() if ashexstr else hasher.digest()  # pragma: no cover
 
 
-def file_as_blockiter(afile, blocksize=65536):  # pylint: disable=C0116
-    """From a StackOverflow answer: that is used to generate a MD5 hash of a large files.
-    # https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file.
+# def file_as_blockiter(afile, blocksize=65536):  # pylint: disable=C0116
+#     """From a StackOverflow answer: that is used to generate a MD5 hash of a large files.
+#     # https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file.
 
-    """
-    with afile:  # pragma: no cover
-        block = afile.read(blocksize)
-        while len(block) > 0:
-            yield block
-            block = afile.read(blocksize)
+#     """
+#     with afile:  # pragma: no cover
+#         block = afile.read(blocksize)
+#         while len(block) > 0:
+#             yield block
+#             block = afile.read(blocksize)
 
 
-def get_data_hash(file_name: str):
+def get_data_hash(file_name: str) -> str:
     """Returns the hash of the data file.
 
     Args:
@@ -389,9 +389,12 @@ def get_data_hash(file_name: str):
     """
     # https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file
     # TODO: For a big file, we should only hash the first few bytes
-    return hash_bytestr_iter(
-        file_as_blockiter(open(file_name, "rb")), hashlib.sha256()
-    )  # pragma: no cover
+    with open(file_name, "rb") as f:
+        file_hash = hashlib.md5()
+        for chunk in iter(lambda: f.read(4096), b""):
+            file_hash.update(chunk)
+
+    return file_hash.hexdigest()
 
 
 # TODO: This is not the right place for this.