runnable 0.27.0__tar.gz → 0.28.0__tar.gz

{runnable-0.27.0 → runnable-0.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.27.0
+Version: 0.28.0
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.27.0 → runnable-0.28.0}/extensions/run_log_store/any_path.py

@@ -33,10 +33,6 @@ class AnyPathRunLogStore(BaseRunLogStore):
     service_name: str = "file-system"
     log_folder: str = defaults.LOG_LOCATION_FOLDER
 
-    @property
-    def log_folder_name(self):
-        return self.log_folder
-
     def get_summary(self) -> Dict[str, Any]:
         summary = {"Type": self.service_name, "Location": self.log_folder}
 

{runnable-0.27.0 → runnable-0.28.0}/extensions/run_log_store/chunked_fs.py

@@ -2,17 +2,13 @@ import json
 import logging
 from pathlib import Path
 from string import Template
-from typing import Any, Dict, Optional, Union
-
-from cloudpathlib import CloudPath
+from typing import Any, Dict, Union
 
 from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
 from runnable import defaults, utils
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
-MixT = Union[CloudPath, Path]
-
 
 class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
     """
@@ -30,7 +26,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
 
     def get_matches(
         self, run_id: str, name: str, multiple_allowed: bool = False
-    ) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
+    ) -> str | list[str] | None:
         """
         Get contents of files matching the pattern name*
 
@@ -48,8 +44,8 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
                 if len(matches) > 1:
                     msg = f"Multiple matches found for {name} while multiple is not allowed"
                     raise Exception(msg)
-                return matches[0]
-            return matches
+                return str(matches[0])
+            return [str(match) for match in matches]
 
         return None
 
@@ -80,7 +76,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
 
         return str(name) + ".json"
 
-    def _store(self, run_id: str, contents: dict, name: MixT, insert=False):
+    def _store(self, run_id: str, contents: dict, name: str, insert=False):
         """
         Store the contents against the name in the folder.
 
@@ -89,16 +85,17 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
             contents (dict): The dict to store
             name (str): The name to store as
         """
+
         log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
         if insert:
-            name = log_folder_with_run_id / name
+            name = str(log_folder_with_run_id / name)
 
         utils.safe_make_dir(log_folder_with_run_id)
 
-        with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "w") as fw:
+        with open(self.safe_suffix_json(name), "w") as fw:
             json.dump(contents, fw, ensure_ascii=True, indent=4)
 
-    def _retrieve(self, run_id: str, name: MixT) -> dict:
+    def _retrieve(self, run_id: str, name: str) -> dict:
         """
         Does the job of retrieving from the folder.
 
@@ -108,10 +105,10 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
         Returns:
             dict: The contents
         """
+
         contents: dict = {}
-        log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
 
-        with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "r") as fr:
+        with open(self.safe_suffix_json(name), "r") as fr:
             contents = json.load(fr)
 
         return contents

runnable-0.28.0/extensions/run_log_store/chunked_minio.py

@@ -0,0 +1,131 @@
+import json
+import logging
+from functools import lru_cache
+from string import Template
+from typing import Any, Dict
+
+from cloudpathlib import S3Client, S3Path
+from pydantic import Field, SecretStr
+
+from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
+from runnable import defaults
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+@lru_cache
+def get_minio_client(
+    endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str
+) -> S3Client:
+    return S3Client(
+        endpoint_url=endpoint_url,
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+    )
+
+
+class ChunkedMinioRunLogStore(ChunkedRunLogStore):
+    """
+    File system run log store but chunks the run log into thread safe chunks.
+    This enables executions to be parallel.
+    """
+
+    service_name: str = "chunked-minio"
+    endpoint_url: str = Field(default="http://localhost:9002")
+    aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
+    aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
+    bucket: str = Field(default="runnable/run-logs")
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {
+            "Type": self.service_name,
+            "Location": f"{self.endpoint_url}/{self.bucket}",
+        }
+
+        return summary
+
+    def get_run_log_bucket(self) -> S3Path:
+        run_id = self._context.run_id
+
+        return S3Path(
+            f"s3://{self.bucket}/{run_id}/",
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+    def get_matches(
+        self, run_id: str, name: str, multiple_allowed: bool = False
+    ) -> None | str | list[str]:
+        """
+        Get contents of files matching the pattern name*
+
+        Args:
+            run_id (str): The run id
+            name (str): The suffix of the file name to check in the run log store.
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        run_log_bucket.mkdir(parents=True, exist_ok=True)
+
+        sub_name = Template(name).safe_substitute({"creation_time": ""})
+        matches = list(run_log_bucket.glob(f"{sub_name}*"))
+
+        if matches:
+            if not multiple_allowed:
+                if len(matches) > 1:
+                    msg = f"Multiple matches found for {name} while multiple is not allowed"
+                    raise Exception(msg)
+                return str(matches[0])
+            return [str(match) for match in matches]
+
+        return None
+
+    def _store(self, run_id: str, contents: dict, name: str, insert=False):
+        """
+        Store the contents against the name in the folder.
+
+        Args:
+            run_id (str): The run id
+            contents (dict): The dict to store
+            name (str): The name to store as
+        """
+
+        if insert:
+            name = str(self.get_run_log_bucket() / name)
+
+        self.get_run_log_bucket().mkdir(parents=True, exist_ok=True)
+        obj = S3Path(
+            name,
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+        obj.write_text(json.dumps(contents, ensure_ascii=True, indent=4))
+
+    def _retrieve(self, run_id: str, name: str) -> dict:
+        """
+        Does the job of retrieving from the folder.
+
+        Args:
+            name (str): the name of the file to retrieve
+
+        Returns:
+            dict: The contents
+        """
+
+        obj = S3Path(
+            name,
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+        run_log_text = json.loads(obj.read_text())
+        return run_log_text

{runnable-0.27.0 → runnable-0.28.0}/extensions/run_log_store/generic_chunked.py

@@ -1,12 +1,10 @@
+import json
 import logging
 import time
 from abc import abstractmethod
 from enum import Enum
-from pathlib import Path
 from string import Template
-from typing import Any, Dict, Optional, Union
-
-from cloudpathlib import CloudPath
+from typing import Any, Dict, Union
 
 from runnable import defaults, exceptions
 from runnable.datastore import (
@@ -23,9 +21,6 @@ from runnable.datastore import (
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-MixT = Union[CloudPath, Path]  # Holds str, path
-
-
 class EntityNotFoundError(Exception):
     pass
 
@@ -89,7 +84,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
     @abstractmethod
     def get_matches(
         self, run_id: str, name: str, multiple_allowed: bool = False
-    ) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
+    ) -> None | str | list[str]:
         """
         Get contents of persistence layer matching the pattern name*
 
@@ -100,7 +95,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         ...
 
     @abstractmethod
-    def _store(self, run_id: str, contents: dict, name: MixT, insert: bool = False):
+    def _store(self, run_id: str, contents: dict, name: str, insert: bool = False):
         """
         Store the contents against the name in the persistence layer.
 
@@ -112,7 +107,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         ...
 
     @abstractmethod
-    def _retrieve(self, run_id: str, name: MixT) -> dict:
+    def _retrieve(self, run_id: str, name: str) -> dict:
         """
         Does the job of retrieving from the persistent layer.
 
@@ -142,18 +137,17 @@ class ChunkedRunLogStore(BaseRunLogStore):
         insert = False
 
         if match:
-            existing_contents = self._retrieve(run_id=run_id, name=match)  # type: ignore
+            assert isinstance(match, str)
+            existing_contents = self._retrieve(run_id=run_id, name=match)
             contents = dict(existing_contents, **contents)
-            name_to_give = match  # type: ignore
+            name_to_give = match
         else:
             name_to_give = Template(naming_pattern).safe_substitute(
                 {"creation_time": str(int(time.time_ns()))}
             )
             insert = True
 
-        self._store(
-            run_id=run_id, contents=contents, name=Path(name_to_give), insert=insert
-        )
+        self._store(run_id=run_id, contents=contents, name=name_to_give, insert=insert)
 
     def retrieve(
         self, run_id: str, log_type: LogTypes, name: str = "", multiple_allowed=False
@@ -194,12 +188,14 @@ class ChunkedRunLogStore(BaseRunLogStore):
 
         if matches:
             if not multiple_allowed:
-                contents = self._retrieve(run_id=run_id, name=matches)  # type: ignore
+                assert isinstance(matches, str)
+                contents = self._retrieve(run_id=run_id, name=matches)
                 model = self.ModelTypes[log_type.name].value
                 return model(**contents)
 
+            assert isinstance(matches, list)
             models = []
-            for match in matches:  # type: ignore
+            for match in matches:
                 contents = self._retrieve(run_id=run_id, name=match)
                 model = self.ModelTypes[log_type.name].value
                 models.append(model(**contents))
@@ -347,7 +343,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
         )
 
         self.store(
-            run_id=run_id, contents=run_log.model_dump(), log_type=self.LogTypes.RUN_LOG
+            run_id=run_id,
+            contents=json.loads(run_log.model_dump_json()),
+            log_type=self.LogTypes.RUN_LOG,
         )
         return run_log
 
@@ -394,7 +392,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
         """
         run_id = run_log.run_id
         self.store(
-            run_id=run_id, contents=run_log.model_dump(), log_type=self.LogTypes.RUN_LOG
+            run_id=run_id,
+            contents=json.loads(run_log.model_dump_json()),
+            log_type=self.LogTypes.RUN_LOG,
         )
 
     def get_parameters(self, run_id: str) -> dict:
@@ -453,7 +453,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
             self.store(
                 run_id=run_id,
                 log_type=self.LogTypes.PARAMETER,
-                contents={key: value.model_dump(by_alias=True)},
+                contents={key: json.loads(value.model_dump_json(by_alias=True))},
                 name=key,
             )
 
@@ -544,7 +544,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         self.store(
             run_id=run_id,
             log_type=self.LogTypes.STEP_LOG,
-            contents=step_log.model_dump(),
+            contents=json.loads(step_log.model_dump_json()),
             name=step_log.internal_name,
         )
 
@@ -600,6 +600,6 @@ class ChunkedRunLogStore(BaseRunLogStore):
         self.store(
             run_id=run_id,
             log_type=self.LogTypes.BRANCH_LOG,
-            contents=branch_log.model_dump(),
+            contents=json.loads(branch_log.model_dump_json()),
             name=internal_branch_name,
         )

{runnable-0.27.0 → runnable-0.28.0}/extensions/run_log_store/minio.py

@@ -46,7 +46,7 @@ class MinioRunLogStore(AnyPathRunLogStore):
 
     """
 
-    service_name: str = "file-system"
+    service_name: str = "minio"
 
     endpoint_url: str = Field(default="http://localhost:9002")
     aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
@@ -54,7 +54,10 @@ class MinioRunLogStore(AnyPathRunLogStore):
     bucket: str = Field(default="runnable/run-logs")
 
     def get_summary(self) -> Dict[str, Any]:
-        summary = {"Type": self.service_name, "Location": self.log_folder}
+        summary = {
+            "Type": self.service_name,
+            "Location": f"{self.endpoint_url}/{self.bucket}",
+        }
 
         return summary
 

{runnable-0.27.0 → runnable-0.28.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runnable"
-version = "0.27.0"
+version = "0.28.0"
 description = "Add your description here"
 readme = "README.md"
 authors = [
@@ -124,6 +124,7 @@ include = [
 file-system = "extensions.run_log_store.file_system:FileSystemRunLogstore"
 "minio" = "extensions.run_log_store.minio:MinioRunLogStore"
 "chunked-fs" = "extensions.run_log_store.chunked_fs:ChunkedFileSystemRunLogStore"
+"chunked-minio" = "extensions.run_log_store.chunked_minio:ChunkedMinioRunLogStore"
 
 [project.entry-points.'pickler']
 "pickle" = "runnable.pickler:NativePickler"