runnable 0.50.0__py3-none-any.whl

extensions/run_log_store/chunked_fs.py

@@ -0,0 +1,122 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, Union
+
+from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
+from runnable import defaults, utils
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
+    """
+    File system run log store but chunks the run log into thread safe chunks.
+    This enables executions to be parallel.
+    """
+
+    service_name: str = "chunked-fs"
+    log_folder: str = defaults.LOG_LOCATION_FOLDER
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {"Type": self.service_name, "Location": self.log_folder}
+
+        return summary
+
+    def _exists(self, run_id: str, name: str) -> bool:
+        """
+        Check if a file exists in the log folder.
+
+        Args:
+            run_id (str): The run id
+            name (str): The exact file name to check
+
+        Returns:
+            bool: True if file exists, False otherwise
+        """
+        log_folder = self.log_folder_with_run_id(run_id=run_id)
+        file_path = log_folder / self.safe_suffix_json(name)
+        return file_path.exists()
+
+    def _list_branch_logs(self, run_id: str) -> list[str]:
+        """
+        List all branch log file names for a run_id.
+
+        Args:
+            run_id (str): The run id
+
+        Returns:
+            list[str]: List of branch log file names without .json extension
+        """
+        log_folder = self.log_folder_with_run_id(run_id=run_id)
+        if not log_folder.exists():
+            return []
+
+        # Find all files starting with "BranchLog-"
+        branch_files = list(log_folder.glob("BranchLog-*.json"))
+        # Return file names without path and without .json extension
+        return [f.stem for f in branch_files]
+
+    def log_folder_with_run_id(self, run_id: str) -> Path:
+        """
+        Utility function to get the log folder for a run id.
+
+        Args:
+            run_id (str): The run id
+
+        Returns:
+            Path: The path to the log folder with the run id
+        """
+        return Path(self.log_folder) / run_id
+
+    def safe_suffix_json(self, name: Union[Path, str]) -> str:
+        """
+        Safely attach a suffix to a json file.
+
+        Args:
+            name (Path): The name of the file with or without suffix of json
+
+        Returns:
+            str : The name of the file with .json
+        """
+        if str(name).endswith("json"):
+            return str(name)
+
+        return str(name) + ".json"
+
+    def _store(self, run_id: str, contents: dict, name: str, insert=False):
+        """
+        Store the contents against the name in the folder.
+
+        Args:
+            run_id (str): The run id
+            contents (dict): The dict to store
+            name (str): The name to store as (without path)
+            insert (bool): Whether this is a new insert (unused, kept for compatibility)
+        """
+        log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
+        file_path = log_folder_with_run_id / name
+
+        utils.safe_make_dir(log_folder_with_run_id)
+
+        with open(self.safe_suffix_json(file_path), "w") as fw:
+            json.dump(contents, fw, ensure_ascii=True, indent=4)
+
+    def _retrieve(self, run_id: str, name: str) -> dict:
+        """
+        Does the job of retrieving from the folder.
+
+        Args:
+            run_id (str): The run id
+            name (str): the name of the file to retrieve (without path)
+
+        Returns:
+            dict: The contents
+        """
+        log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
+        file_path = log_folder_with_run_id / name
+
+        with open(self.safe_suffix_json(file_path), "r") as fr:
+            contents = json.load(fr)
+
+        return contents

extensions/run_log_store/chunked_minio.py

@@ -0,0 +1,141 @@
+import json
+import logging
+from functools import lru_cache
+from typing import Any, Dict
+
+from cloudpathlib import S3Client, S3Path
+from pydantic import Field, SecretStr
+
+from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
+from runnable import defaults
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+@lru_cache
+def get_minio_client(
+    endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str
+) -> S3Client:
+    return S3Client(
+        endpoint_url=endpoint_url,
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+    )
+
+
+class ChunkedMinioRunLogStore(ChunkedRunLogStore):
+    """
+    File system run log store but chunks the run log into thread safe chunks.
+    This enables executions to be parallel.
+    """
+
+    service_name: str = "chunked-minio"
+    endpoint_url: str = Field(default="http://localhost:9002")
+    aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
+    aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
+    bucket: str = Field(default="runnable/run-logs")
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {
+            "Type": self.service_name,
+            "Location": f"{self.endpoint_url}/{self.bucket}",
+        }
+
+        return summary
+
+    def get_run_log_bucket(self) -> S3Path:
+        run_id = self._context.run_id
+
+        return S3Path(
+            f"s3://{self.bucket}/{run_id}/",
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+    def _exists(self, run_id: str, name: str) -> bool:
+        """
+        Check if a file exists in the Minio bucket.
+
+        Args:
+            run_id (str): The run id
+            name (str): The exact file name to check
+
+        Returns:
+            bool: True if file exists, False otherwise
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        file_path = run_log_bucket / name
+        return file_path.exists()
+
+    def _list_branch_logs(self, run_id: str) -> list[str]:
+        """
+        List all branch log file names for a run_id.
+
+        Args:
+            run_id (str): The run id
+
+        Returns:
+            list[str]: List of branch log file names (e.g., ["BranchLog-map.1", "BranchLog-map.2"])
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        if not run_log_bucket.exists():
+            return []
+
+        # Find all files starting with "BranchLog-"
+        branch_files = list(run_log_bucket.glob("BranchLog-*"))
+        # Return file names without path (just the name)
+        return [f.name for f in branch_files]
+
+    def _store(self, run_id: str, contents: dict, name: str, insert=False):
+        """
+        Store the contents against the name in the folder.
+
+        Args:
+            run_id (str): The run id
+            contents (dict): The dict to store
+            name (str): The name to store as (without path)
+            insert (bool): Whether this is a new insert (unused, kept for compatibility)
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        run_log_bucket.mkdir(parents=True, exist_ok=True)
+
+        file_path = str(run_log_bucket / name)
+        obj = S3Path(
+            file_path,
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+        obj.write_text(json.dumps(contents, ensure_ascii=True, indent=4))
+
+    def _retrieve(self, run_id: str, name: str) -> dict:
+        """
+        Does the job of retrieving from the folder.
+
+        Args:
+            run_id (str): The run id
+            name (str): the name of the file to retrieve (without path)
+
+        Returns:
+            dict: The contents
+        """
+        run_log_bucket = self.get_run_log_bucket()
+        file_path = str(run_log_bucket / name)
+
+        obj = S3Path(
+            file_path,
+            client=get_minio_client(
+                self.endpoint_url,
+                self.aws_access_key_id.get_secret_value(),
+                self.aws_secret_access_key.get_secret_value(),
+            ),
+        )
+
+        run_log_text = json.loads(obj.read_text())
+        return run_log_text

extensions/run_log_store/file_system.py

@@ -0,0 +1,91 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict
+
+from extensions.run_log_store.any_path import AnyPathRunLogStore
+from runnable import defaults, utils
+from runnable.datastore import RunLog
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class FileSystemRunLogstore(AnyPathRunLogStore):
+    """
+    In this type of Run Log store, we use a file system to store the JSON run log.
+
+    Every single run is stored as a different file which makes it compatible across other store types.
+
+    When to use:
+        When locally testing a pipeline and have the need to compare across runs.
+        Its fully featured and perfectly fine if your local environment is where you would do everything.
+
+    Do not use:
+        If you need parallelization on local, this run log would not support it.
+
+    Example config:
+
+    run_log:
+      type: file-system
+      config:
+        log_folder: The folder to out the logs. Defaults to .run_log_store
+
+    """
+
+    service_name: str = "file-system"
+    log_folder: str = defaults.LOG_LOCATION_FOLDER
+
+    @property
+    def log_folder_name(self):
+        return self.log_folder
+
+    def get_summary(self) -> Dict[str, Any]:
+        summary = {"Type": self.service_name, "Location": self.log_folder}
+
+        return summary
+
+    def write_to_path(self, run_log: RunLog):
+        """
+        Write the run log to the folder
+
+        Args:
+            run_log (RunLog): The run log to be added to the database
+        """
+        write_to = self.log_folder_name
+        utils.safe_make_dir(write_to)
+
+        write_to_path = Path(write_to)
+        run_id = run_log.run_id
+        json_file_path = write_to_path / f"{run_id}.json"
+
+        with json_file_path.open("w") as fw:
+            json.dump(run_log.model_dump(), fw, ensure_ascii=True, indent=4)  # pylint: disable=no-member
+
+    def read_from_path(self, run_id: str) -> RunLog:
+        """
+        Look into the run log folder for the run log for the run id.
+
+        If the run log does not exist, raise an exception. If it does, decode it
+        as a RunLog and return it
+
+        Args:
+            run_id (str): The requested run id to retrieve the run log store
+
+        Raises:
+            FileNotFoundError: If the Run Log has not been found.
+
+        Returns:
+            RunLog: The decoded Run log
+        """
+        write_to = self.log_folder_name
+
+        read_from_path = Path(write_to)
+        json_file_path = read_from_path / f"{run_id}.json"
+
+        if not json_file_path.exists():
+            raise FileNotFoundError(f"Expected {json_file_path} is not present")
+
+        with json_file_path.open("r") as fr:
+            json_str = json.load(fr)
+            run_log = RunLog(**json_str)  # pylint: disable=no-member
+        return run_log