runnable 0.27.0__py3-none-any.whl → 0.28.1__py3-none-any.whl

extensions/job_executor/__init__.py

@@ -144,14 +144,10 @@ class GenericJobExecutor(BaseJobExecutor):
             logger.info("No catalog settings found")
             return None
 
-        compute_data_folder = self._context.catalog_handler.compute_data_folder
-
         data_catalogs = []
         for name_pattern in catalog_settings:
             data_catalog = self._context.catalog_handler.put(
                 name=name_pattern,
-                run_id=self._context.run_id,
-                compute_data_folder=compute_data_folder,
             )
 
             logger.debug(f"Added data catalog: {data_catalog} to job log")
@@ -168,7 +164,5 @@ class GenericJobExecutor(BaseJobExecutor):
         )
         task_console.save_text(log_file_name)
         # Put the log file in the catalog
-        self._context.catalog_handler.put(
-            name=log_file_name, run_id=self._context.run_id
-        )
+        self._context.catalog_handler.put(name=log_file_name)
         os.remove(log_file_name)

extensions/run_log_store/any_path.py

@@ -33,10 +33,6 @@ class AnyPathRunLogStore(BaseRunLogStore):
     service_name: str = "file-system"
     log_folder: str = defaults.LOG_LOCATION_FOLDER
 
-    @property
-    def log_folder_name(self):
-        return self.log_folder
-
     def get_summary(self) -> Dict[str, Any]:
         summary = {"Type": self.service_name, "Location": self.log_folder}
 

extensions/run_log_store/chunked_fs.py

@@ -2,17 +2,13 @@ import json
 import logging
 from pathlib import Path
 from string import Template
-from typing import Any, Dict, Optional, Union
-
-from cloudpathlib import CloudPath
+from typing import Any, Dict, Union
 
 from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
 from runnable import defaults, utils
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
-MixT = Union[CloudPath, Path]
-
 
 class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
     """
@@ -30,7 +26,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
 
     def get_matches(
         self, run_id: str, name: str, multiple_allowed: bool = False
-    ) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
+    ) -> str | list[str] | None:
         """
         Get contents of files matching the pattern name*
 
@@ -48,8 +44,8 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
                 if len(matches) > 1:
                     msg = f"Multiple matches found for {name} while multiple is not allowed"
                     raise Exception(msg)
-                return matches[0]
-            return matches
+                return str(matches[0])
+            return [str(match) for match in matches]
 
         return None
 
@@ -80,7 +76,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
 
         return str(name) + ".json"
 
-    def _store(self, run_id: str, contents: dict, name: MixT, insert=False):
+    def _store(self, run_id: str, contents: dict, name: str, insert=False):
         """
         Store the contents against the name in the folder.
 
@@ -89,16 +85,17 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
             contents (dict): The dict to store
             name (str): The name to store as
         """
+
         log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
         if insert:
-            name = log_folder_with_run_id / name
+            name = str(log_folder_with_run_id / name)
 
         utils.safe_make_dir(log_folder_with_run_id)
 
-        with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "w") as fw:
+        with open(self.safe_suffix_json(name), "w") as fw:
             json.dump(contents, fw, ensure_ascii=True, indent=4)
 
-    def _retrieve(self, run_id: str, name: MixT) -> dict:
+    def _retrieve(self, run_id: str, name: str) -> dict:
         """
         Does the job of retrieving from the folder.
 
@@ -108,10 +105,10 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
         Returns:
             dict: The contents
         """
+
         contents: dict = {}
-        log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
 
-        with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "r") as fr:
+        with open(self.safe_suffix_json(name), "r") as fr:
             contents = json.load(fr)
 
         return contents

extensions/run_log_store/chunked_minio.py

@@ -0,0 +1,131 @@
+import json
+import logging
+from functools import lru_cache
+from string import Template
+from typing import Any, Dict
+
+from cloudpathlib import S3Client, S3Path
+from pydantic import Field, SecretStr
+
+from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
+from runnable import defaults
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+@lru_cache
+def get_minio_client(
+    endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str
+) -> S3Client:
+    return S3Client(
+        endpoint_url=endpoint_url,
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+    )
+
+
+class ChunkedMinioRunLogStore(ChunkedRunLogStore):
+    """
+    File system run log store but chunks the run log into thread safe chunks.
+    This enables executions to be parallel.
+    """
+
+    service_name: str = "chunked-minio"
+    endpoint_url: str = Field(default="http://localhost:9002")
+    aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
+    aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
+    bucket: str = Field(default="runnable/run-logs")
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {
+            "Type": self.service_name,
+            "Location": f"{self.endpoint_url}/{self.bucket}",
+        }
+
+        return summary
+
+    def get_run_log_bucket(self) -> S3Path:
+        run_id = self._context.run_id
+
+        return S3Path(
+            f"s3://{self.bucket}/{run_id}/",
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+    def get_matches(
+        self, run_id: str, name: str, multiple_allowed: bool = False
+    ) -> None | str | list[str]:
+        """
+        Get contents of files matching the pattern name*
+
+        Args:
+            run_id (str): The run id
+            name (str): The suffix of the file name to check in the run log store.
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        run_log_bucket.mkdir(parents=True, exist_ok=True)
+
+        sub_name = Template(name).safe_substitute({"creation_time": ""})
+        matches = list(run_log_bucket.glob(f"{sub_name}*"))
+
+        if matches:
+            if not multiple_allowed:
+                if len(matches) > 1:
+                    msg = f"Multiple matches found for {name} while multiple is not allowed"
+                    raise Exception(msg)
+                return str(matches[0])
+            return [str(match) for match in matches]
+
+        return None
+
+    def _store(self, run_id: str, contents: dict, name: str, insert=False):
+        """
+        Store the contents against the name in the folder.
+
+        Args:
+            run_id (str): The run id
+            contents (dict): The dict to store
+            name (str): The name to store as
+        """
+
+        if insert:
+            name = str(self.get_run_log_bucket() / name)
+
+        self.get_run_log_bucket().mkdir(parents=True, exist_ok=True)
+        obj = S3Path(
+            name,
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+        obj.write_text(json.dumps(contents, ensure_ascii=True, indent=4))
+
+    def _retrieve(self, run_id: str, name: str) -> dict:
+        """
+        Does the job of retrieving from the folder.
+
+        Args:
+            name (str): the name of the file to retrieve
+
+        Returns:
+            dict: The contents
+        """
+
+        obj = S3Path(
+            name,
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+        run_log_text = json.loads(obj.read_text())
+        return run_log_text

extensions/run_log_store/generic_chunked.py

@@ -1,12 +1,10 @@
+import json
 import logging
 import time
 from abc import abstractmethod
 from enum import Enum
-from pathlib import Path
 from string import Template
-from typing import Any, Dict, Optional, Union
-
-from cloudpathlib import CloudPath
+from typing import Any, Dict, Union
 
 from runnable import defaults, exceptions
 from runnable.datastore import (
@@ -23,9 +21,6 @@ from runnable.datastore import (
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
-MixT = Union[CloudPath, Path]  # Holds str, path
-
-
 class EntityNotFoundError(Exception):
     pass
 
@@ -89,7 +84,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
     @abstractmethod
     def get_matches(
         self, run_id: str, name: str, multiple_allowed: bool = False
-    ) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
+    ) -> None | str | list[str]:
         """
         Get contents of persistence layer matching the pattern name*
 
@@ -100,7 +95,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         ...
 
     @abstractmethod
-    def _store(self, run_id: str, contents: dict, name: MixT, insert: bool = False):
+    def _store(self, run_id: str, contents: dict, name: str, insert: bool = False):
         """
         Store the contents against the name in the persistence layer.
 
@@ -112,7 +107,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         ...
 
     @abstractmethod
-    def _retrieve(self, run_id: str, name: MixT) -> dict:
+    def _retrieve(self, run_id: str, name: str) -> dict:
         """
         Does the job of retrieving from the persistent layer.
 
@@ -142,18 +137,17 @@ class ChunkedRunLogStore(BaseRunLogStore):
         insert = False
 
         if match:
-            existing_contents = self._retrieve(run_id=run_id, name=match)  # type: ignore
+            assert isinstance(match, str)
+            existing_contents = self._retrieve(run_id=run_id, name=match)
             contents = dict(existing_contents, **contents)
-            name_to_give = match  # type: ignore
+            name_to_give = match
         else:
             name_to_give = Template(naming_pattern).safe_substitute(
                 {"creation_time": str(int(time.time_ns()))}
             )
             insert = True
 
-        self._store(
-            run_id=run_id, contents=contents, name=Path(name_to_give), insert=insert
-        )
+        self._store(run_id=run_id, contents=contents, name=name_to_give, insert=insert)
 
     def retrieve(
         self, run_id: str, log_type: LogTypes, name: str = "", multiple_allowed=False
@@ -194,12 +188,14 @@ class ChunkedRunLogStore(BaseRunLogStore):
 
         if matches:
             if not multiple_allowed:
-                contents = self._retrieve(run_id=run_id, name=matches)  # type: ignore
+                assert isinstance(matches, str)
+                contents = self._retrieve(run_id=run_id, name=matches)
                 model = self.ModelTypes[log_type.name].value
                 return model(**contents)
 
+            assert isinstance(matches, list)
             models = []
-            for match in matches:  # type: ignore
+            for match in matches:
                 contents = self._retrieve(run_id=run_id, name=match)
                 model = self.ModelTypes[log_type.name].value
                 models.append(model(**contents))
@@ -347,7 +343,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
         )
 
         self.store(
-            run_id=run_id, contents=run_log.model_dump(), log_type=self.LogTypes.RUN_LOG
+            run_id=run_id,
+            contents=json.loads(run_log.model_dump_json()),
+            log_type=self.LogTypes.RUN_LOG,
         )
         return run_log
 
@@ -394,7 +392,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
         """
         run_id = run_log.run_id
         self.store(
-            run_id=run_id, contents=run_log.model_dump(), log_type=self.LogTypes.RUN_LOG
+            run_id=run_id,
+            contents=json.loads(run_log.model_dump_json()),
+            log_type=self.LogTypes.RUN_LOG,
         )
 
     def get_parameters(self, run_id: str) -> dict:
@@ -453,7 +453,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
             self.store(
                 run_id=run_id,
                 log_type=self.LogTypes.PARAMETER,
-                contents={key: value.model_dump(by_alias=True)},
+                contents={key: json.loads(value.model_dump_json(by_alias=True))},
                 name=key,
             )
 
@@ -544,7 +544,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
         self.store(
             run_id=run_id,
             log_type=self.LogTypes.STEP_LOG,
-            contents=step_log.model_dump(),
+            contents=json.loads(step_log.model_dump_json()),
             name=step_log.internal_name,
         )
 
@@ -600,6 +600,6 @@ class ChunkedRunLogStore(BaseRunLogStore):
         self.store(
             run_id=run_id,
             log_type=self.LogTypes.BRANCH_LOG,
-            contents=branch_log.model_dump(),
+            contents=json.loads(branch_log.model_dump_json()),
             name=internal_branch_name,
         )

extensions/run_log_store/minio.py

@@ -46,7 +46,7 @@ class MinioRunLogStore(AnyPathRunLogStore):
 
     """
 
-    service_name: str = "file-system"
+    service_name: str = "minio"
 
     endpoint_url: str = Field(default="http://localhost:9002")
     aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
@@ -54,7 +54,10 @@ class MinioRunLogStore(AnyPathRunLogStore):
     bucket: str = Field(default="runnable/run-logs")
 
     def get_summary(self) -> Dict[str, Any]:
-        summary = {"Type": self.service_name, "Location": self.log_folder}
+        summary = {
+            "Type": self.service_name,
+            "Location": f"{self.endpoint_url}/{self.bucket}",
+        }
 
         return summary
 

{runnable-0.27.0.dist-info → runnable-0.28.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runnable
-Version: 0.27.0
+Version: 0.28.1
 Summary: Add your description here
 Author-email: "Vammi, Vijay" <vijay.vammi@astrazeneca.com>
 License-File: LICENSE

{runnable-0.27.0.dist-info → runnable-0.28.1.dist-info}/RECORD

@@ -7,7 +7,7 @@ extensions/catalog/minio.py,sha256=R3GvfCxN1GTcs4bQIAWh79_GHDTVd14gnpKlzwFeKUI,2
 extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
 extensions/catalog/s3.py,sha256=Sw5t8_kVRprn3uGGJCiHn7M9zw1CLaCOFj6YErtfG0o,287
 extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/job_executor/__init__.py,sha256=3zS2m6dg-L6SkKfL0kr4AxVUVmVJcepV6eipyMvQR6s,6006
+extensions/job_executor/__init__.py,sha256=E2R6GV5cZTlZdqA5SVJ6ajZFh4oruM0k8AKHkpOZ3W8,5772
 extensions/job_executor/k8s.py,sha256=V5k6Rnf_sAFqptVbCrWs_x5sl3x3fSHwO96IZoiJxKU,15342
 extensions/job_executor/k8s_job_spec.yaml,sha256=7aFpxHdO_p6Hkc3YxusUOuAQTD1Myu0yTPX9DrhxbOg,1158
 extensions/job_executor/local.py,sha256=FvxTk0vyxdrbLOAyNkLyjvmmowypabWOSITQBK_ffVE,1907
@@ -26,11 +26,12 @@ extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZ
 extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
 extensions/run_log_store/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-extensions/run_log_store/any_path.py,sha256=2Ff9-jd1mWazFXn4HvsF0MF-oQAdMNYrl698Kjpip4Q,2938
-extensions/run_log_store/chunked_fs.py,sha256=_2KmTaxK-p1e_YJqnQwyOqkLoCr80y-Wg1X2XdCC_9k,3546
+extensions/run_log_store/any_path.py,sha256=0nN_LHbm2W6AHkerQmsVHq3EoybFQF8lxpCicacHo8Y,2861
+extensions/run_log_store/chunked_fs.py,sha256=wHMKcAx6uFI4OOTp7QWCdGq9WvEFesbLp9VxHZU28l0,3341
+extensions/run_log_store/chunked_minio.py,sha256=Itfkw4Ycf0uLCqxH3Uk_itmVgT7ipJp05yKfD22WBiY,4007
 extensions/run_log_store/file_system.py,sha256=hhrbhSnuzv8yzBr6DAu45NT8-sawPP86WA2-LY70vjw,2781
-extensions/run_log_store/generic_chunked.py,sha256=D08ADWK_rw4ed46_yeEankO-g9rwOjVYi4V-JaOOT6M,20445
-extensions/run_log_store/minio.py,sha256=pTHUzCHnehJ5JZoIrlZGjb4Cl-6yinp3MgqLZPvIxq4,3403
+extensions/run_log_store/generic_chunked.py,sha256=bsGgChTDZN3dSbLmLJ9SIpcvArzVmzhTVAOYZytAUNc,20483
+extensions/run_log_store/minio.py,sha256=omrKDSdRzmnVBg9xXkkdQb-icBIgBDRdpmwGRlMyCGk,3453
 extensions/run_log_store/pyproject.toml,sha256=YnmXsFvFG9uv_c0spLYBsNI_1sbktqxtHsOuClyvZ3g,288
 extensions/run_log_store/db/implementation_FF.py,sha256=euTnh0xzNF0e_DyfHQ4W-kG1AwTr8u7OuO3_cZkR5bM,5237
 extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -55,8 +56,8 @@ runnable/sdk.py,sha256=T1nqDpLN9fULvvU9L-oY0EHqYdKUI9qk7oekLynm02Y,33568
 runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
 runnable/tasks.py,sha256=X6xijut7ffwpfYDcXoN6y0AcRVd7fWHs676DJ00Kma4,29134
 runnable/utils.py,sha256=hBr7oGwGL2VgfITlQCTz-a1iwvvf7Mfl-HY8UdENZac,19929
-runnable-0.27.0.dist-info/METADATA,sha256=Bmncq3RAxN2eZGtlWQf5X8O0CkWRqmiCQiY-rH90odM,10047
-runnable-0.27.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-runnable-0.27.0.dist-info/entry_points.txt,sha256=VZ-VOpcr16mj9jwxxMycwGNJlG_cziICRIe-LQrDwig,1760
-runnable-0.27.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-runnable-0.27.0.dist-info/RECORD,,
+runnable-0.28.1.dist-info/METADATA,sha256=j-mrRLCkfv8GkWDLLFDtheBccn9CP4H-NvKAwm2L9ak,10047
+runnable-0.28.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+runnable-0.28.1.dist-info/entry_points.txt,sha256=ioMbWojILtdibYVgh1jXJ00SpK-tX3gy7oVGDq61cSk,1839
+runnable-0.28.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+runnable-0.28.1.dist-info/RECORD,,

{runnable-0.27.0.dist-info → runnable-0.28.1.dist-info}/entry_points.txt

@@ -35,6 +35,7 @@ retry = extensions.pipeline_executor.retry:RetryExecutor
 [run_log_store]
 buffered = runnable.datastore:BufferRunLogstore
 chunked-fs = extensions.run_log_store.chunked_fs:ChunkedFileSystemRunLogStore
+chunked-minio = extensions.run_log_store.chunked_minio:ChunkedMinioRunLogStore
 file-system = extensions.run_log_store.file_system:FileSystemRunLogstore
 minio = extensions.run_log_store.minio:MinioRunLogStore