runnable 0.13.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

runnable/datastore.py

@@ -18,14 +18,13 @@ from typing import (
 
 from pydantic import BaseModel, Field, computed_field
 
-import runnable.context as context
 from runnable import defaults, exceptions
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
 JSONType = Union[
-    str, int, float, bool, List[Any], Dict[str, Any]
+    Union[None, bool, str, float, int, List[Any], Dict[str, Any]]
 ]  # This is actually JSONType, but pydantic doesn't support TypeAlias yet
 
 
@@ -56,14 +55,11 @@ class DataCatalog(BaseModel, extra="allow"):
         return other.name == self.name
 
 
-"""
-The theory behind reduced:
-    parameters returned by steps in map node are only reduced by the end of the map step, fan-in.
-    If they are accessed within the map step, the value should be the value returned by the step in the map step.
-
-    Once the map state is complete, we can set the reduce to true and have the value as
-    the reduced value. Its either a list or a custom function return.
-"""
+# The theory behind reduced:
+#     parameters returned by steps in map node are only reduced by the end of the map step, fan-in.
+#     If they are accessed within the map step, the value should be the value returned by the step in the map step.
+#     Once the map state is complete, we can set the reduce to true and have the value as
+#     the reduced value. Its either a list or a custom function return.
 
 
 class JsonParameter(BaseModel):
@@ -125,7 +121,9 @@ class ObjectParameter(BaseModel):
         os.remove(self.file_name)  # Remove after loading
 
 
-Parameter = Annotated[Union[JsonParameter, ObjectParameter, MetricParameter], Field(discriminator="kind")]
+Parameter = Annotated[
+    Union[JsonParameter, ObjectParameter, MetricParameter], Field(discriminator="kind")
+]
 
 
 class StepAttempt(BaseModel):
@@ -157,8 +155,12 @@ class CodeIdentity(BaseModel, extra="allow"):
 
     code_identifier: Optional[str] = ""  # GIT sha code or docker image id
     code_identifier_type: Optional[str] = ""  # git or docker
-    code_identifier_dependable: Optional[bool] = False  # If git, checks if the tree is clean.
-    code_identifier_url: Optional[str] = ""  # The git remote url or docker repository url
+    code_identifier_dependable: Optional[bool] = (
+        False  # If git, checks if the tree is clean.
+    )
+    code_identifier_url: Optional[str] = (
+        ""  # The git remote url or docker repository url
+    )
     code_identifier_message: Optional[str] = ""  # Any optional message
 
 
@@ -185,18 +187,28 @@ class StepLog(BaseModel):
         summary: Dict[str, Any] = {}
 
         summary["Name"] = self.internal_name
-        summary["Input catalog content"] = [dc.name for dc in self.data_catalog if dc.stage == "get"]
+        summary["Input catalog content"] = [
+            dc.name for dc in self.data_catalog if dc.stage == "get"
+        ]
         summary["Available parameters"] = [
-            (p, v.description) for attempt in self.attempts for p, v in attempt.input_parameters.items()
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.input_parameters.items()
         ]
 
-        summary["Output catalog content"] = [dc.name for dc in self.data_catalog if dc.stage == "put"]
+        summary["Output catalog content"] = [
+            dc.name for dc in self.data_catalog if dc.stage == "put"
+        ]
         summary["Output parameters"] = [
-            (p, v.description) for attempt in self.attempts for p, v in attempt.output_parameters.items()
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.output_parameters.items()
         ]
 
         summary["Metrics"] = [
-            (p, v.description) for attempt in self.attempts for p, v in attempt.user_defined_metrics.items()
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.user_defined_metrics.items()
         ]
 
         cis = []
@@ -288,6 +300,73 @@ class BranchLog(BaseModel):
 StepLog.model_rebuild()
 
 
+class JobLog(BaseModel):
+    """
+    The data class capturing the data of a job
+    This should be treated as a step log
+    """
+
+    status: str = defaults.FAIL
+    message: str = ""
+    mock: bool = False
+    code_identities: List[CodeIdentity] = Field(default_factory=list)
+    attempts: List[StepAttempt] = Field(default_factory=list)
+    data_catalog: List[DataCatalog] = Field(default_factory=list)
+
+    def add_data_catalogs(self, data_catalogs: List[DataCatalog]):
+        """
+        Add the data catalogs as asked by the user
+
+        Args:
+            dict_catalogs ([DataCatalog]): A list of data catalog items
+        """
+
+        if not self.data_catalog:
+            self.data_catalog = []
+        for data_catalog in data_catalogs:
+            self.data_catalog.append(data_catalog)
+
+    def get_summary(self) -> Dict[str, Any]:
+        """
+        Summarize the step log to log
+        """
+        summary: Dict[str, Any] = {}
+
+        summary["Available parameters"] = [
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.input_parameters.items()
+        ]
+
+        summary["Output catalog content"] = [
+            dc.name for dc in self.data_catalog if dc.stage == "put"
+        ]
+        summary["Output parameters"] = [
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.output_parameters.items()
+        ]
+
+        summary["Metrics"] = [
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.user_defined_metrics.items()
+        ]
+
+        cis = []
+        for ci in self.code_identities:
+            message = f"{ci.code_identifier_type}:{ci.code_identifier}"
+            if not ci.code_identifier_dependable:
+                message += " but is not dependable"
+            cis.append(message)
+
+        summary["Code identities"] = cis
+
+        summary["status"] = self.status
+
+        return summary
+
+
 class RunLog(BaseModel):
     """
     The data captured as part of Run Log
@@ -298,6 +377,7 @@ class RunLog(BaseModel):
     tag: Optional[str] = ""
     status: str = defaults.FAIL
     steps: OrderedDict[str, StepLog] = Field(default_factory=OrderedDict)
+    job: Optional[JobLog] = None
     parameters: Dict[str, Parameter] = Field(default_factory=dict)
     run_config: Dict[str, Any] = Field(default_factory=dict)
 
@@ -312,10 +392,18 @@ class RunLog(BaseModel):
         summary["Catalog Location"] = _context.catalog_handler.get_summary()
         summary["Full Run log present at: "] = _context.run_log_store.get_summary()
 
-        run_log = _context.run_log_store.get_run_log_by_id(run_id=_context.run_id, full=True)
+        run_log = _context.run_log_store.get_run_log_by_id(
+            run_id=_context.run_id, full=True
+        )
 
-        summary["Final Parameters"] = {p: v.description for p, v in run_log.parameters.items()}
-        summary["Collected metrics"] = {p: v.description for p, v in run_log.parameters.items() if v.kind == "metric"}
+        summary["Final Parameters"] = {
+            p: v.description for p, v in run_log.parameters.items()
+        }
+        summary["Collected metrics"] = {
+            p: v.description
+            for p, v in run_log.parameters.items()
+            if v.kind == "metric"
+        }
 
         return summary
 
@@ -338,7 +426,9 @@ class RunLog(BaseModel):
 
         return list(set(data_catalogs))
 
-    def search_branch_by_internal_name(self, i_name: str) -> Tuple[Union[BranchLog, RunLog], Union[StepLog, None]]:
+    def search_branch_by_internal_name(
+        self, i_name: str
+    ) -> Tuple[Union[BranchLog, RunLog], Union[StepLog, None]]:
         """
         Given a branch internal name, search for it in the run log.
 
@@ -385,7 +475,9 @@ class RunLog(BaseModel):
 
         raise exceptions.BranchLogNotFoundError(self.run_id, i_name)
 
-    def search_step_by_internal_name(self, i_name: str) -> Tuple[StepLog, Union[BranchLog, None]]:
+    def search_step_by_internal_name(
+        self, i_name: str
+    ) -> Tuple[StepLog, Union[BranchLog, None]]:
         """
         Given a steps internal name, search for the step name.
 
@@ -415,7 +507,9 @@ class RunLog(BaseModel):
                 # Its odd, so we are in brach name
                 current_branch = current_step.branches[".".join(dot_path[: i + 1])]  # type: ignore
                 current_steps = current_branch.steps
-                logger.debug(f"Finding step log for {i_name} in branch: {current_branch}")
+                logger.debug(
+                    f"Finding step log for {i_name} in branch: {current_branch}"
+                )
             else:
                 # Its even, so we are in step, we start here!
                 current_step = current_steps[".".join(dot_path[: i + 1])]
@@ -428,10 +522,6 @@ class RunLog(BaseModel):
         raise exceptions.StepLogNotFoundError(self.run_id, i_name)
 
 
-# All outside modules should interact with dataclasses using the RunLogStore to promote extensibility
-# If you want to customize dataclass, extend BaseRunLogStore and implement the methods as per the specification
-
-
 class BaseRunLogStore(ABC, BaseModel):
     """
     The base class of a Run Log Store with many common methods implemented.
@@ -441,13 +531,29 @@ class BaseRunLogStore(ABC, BaseModel):
     service_type: str = "run_log_store"
 
     @abstractmethod
-    def get_summary(self) -> Dict[str, Any]:
-        ...
+    def get_summary(self) -> Dict[str, Any]: ...
 
     @property
     def _context(self):
         return context.run_context
 
+        """
+        Retrieves a Job log from the database using the config and the job_id
+
+        Args:
+            job_id (str): The job_id of the job
+
+        Returns:
+            JobLog: The JobLog object identified by the job_id
+
+        Logically the method should:
+            * Returns the job_log defined by id from the data store defined by the config
+
+        Raises:
+            NotImplementedError: This is a base class and therefore has no default implementation
+            JobLogNotFoundError: If the job log for job_id is not found in the datastore
+        """
+
     @abstractmethod
     def create_run_log(
         self,
@@ -457,7 +563,6 @@ class BaseRunLogStore(ABC, BaseModel):
         tag: str = "",
         original_run_id: str = "",
         status: str = defaults.CREATED,
-        **kwargs,
     ):
         """
         Creates a Run Log object by using the config
@@ -473,7 +578,7 @@ class BaseRunLogStore(ABC, BaseModel):
         raise NotImplementedError
 
     @abstractmethod
-    def get_run_log_by_id(self, run_id: str, full: bool = False, **kwargs) -> RunLog:
+    def get_run_log_by_id(self, run_id: str, full: bool = False) -> RunLog:
         """
         Retrieves a Run log from the database using the config and the run_id
 
@@ -495,7 +600,7 @@ class BaseRunLogStore(ABC, BaseModel):
         raise NotImplementedError
 
     @abstractmethod
-    def put_run_log(self, run_log: RunLog, **kwargs):
+    def put_run_log(self, run_log: RunLog):
         """
         Puts the Run Log in the database as defined by the config
 
@@ -523,7 +628,7 @@ class BaseRunLogStore(ABC, BaseModel):
         run_log.status = status
         self.put_run_log(run_log)
 
-    def get_parameters(self, run_id: str, **kwargs) -> Dict[str, Parameter]:
+    def get_parameters(self, run_id: str) -> Dict[str, Parameter]:
         """
         Get the parameters from the Run log defined by the run_id
 
@@ -542,7 +647,7 @@ class BaseRunLogStore(ABC, BaseModel):
         run_log = self.get_run_log_by_id(run_id=run_id)
         return run_log.parameters
 
-    def set_parameters(self, run_id: str, parameters: Dict[str, Parameter], **kwargs):
+    def set_parameters(self, run_id: str, parameters: Dict[str, Parameter]):
         """
         Update the parameters of the Run log with the new parameters
 
@@ -563,7 +668,7 @@ class BaseRunLogStore(ABC, BaseModel):
         run_log.parameters.update(parameters)
         self.put_run_log(run_log=run_log)
 
-    def get_run_config(self, run_id: str, **kwargs) -> dict:
+    def get_run_config(self, run_id: str) -> dict:
         """
         Given a run_id, return the run_config used to perform the run.
 
@@ -577,7 +682,7 @@ class BaseRunLogStore(ABC, BaseModel):
         run_log = self.get_run_log_by_id(run_id=run_id)
         return run_log.run_config
 
-    def set_run_config(self, run_id: str, run_config: dict, **kwargs):
+    def set_run_config(self, run_id: str, run_config: dict):
         """Set the run config used to run the run_id
 
         Args:
@@ -589,7 +694,7 @@ class BaseRunLogStore(ABC, BaseModel):
         run_log.run_config.update(run_config)
         self.put_run_log(run_log=run_log)
 
-    def create_step_log(self, name: str, internal_name: str, **kwargs):
+    def create_step_log(self, name: str, internal_name: str):
         """
         Create a step log by the name and internal name
 
@@ -607,7 +712,7 @@ class BaseRunLogStore(ABC, BaseModel):
         logger.info(f"{self.service_name} Creating a Step Log: {internal_name}")
         return StepLog(name=name, internal_name=internal_name, status=defaults.CREATED)
 
-    def get_step_log(self, internal_name: str, run_id: str, **kwargs) -> StepLog:
+    def get_step_log(self, internal_name: str, run_id: str) -> StepLog:
         """
         Get a step log from the datastore for run_id and the internal naming of the step log
 
@@ -629,12 +734,14 @@ class BaseRunLogStore(ABC, BaseModel):
             RunLogNotFoundError: If the run log for run_id is not found in the datastore
             StepLogNotFoundError: If the step log for internal_name is not found in the datastore for run_id
         """
-        logger.info(f"{self.service_name} Getting the step log: {internal_name} of {run_id}")
+        logger.info(
+            f"{self.service_name} Getting the step log: {internal_name} of {run_id}"
+        )
         run_log = self.get_run_log_by_id(run_id=run_id)
         step_log, _ = run_log.search_step_by_internal_name(internal_name)
         return step_log
 
-    def add_step_log(self, step_log: StepLog, run_id: str, **kwargs):
+    def add_step_log(self, step_log: StepLog, run_id: str):
         """
         Add the step log in the run log as identified by the run_id in the datastore
 
@@ -664,7 +771,7 @@ class BaseRunLogStore(ABC, BaseModel):
         branch.steps[step_log.internal_name] = step_log
         self.put_run_log(run_log=run_log)
 
-    def create_branch_log(self, internal_branch_name: str, **kwargs) -> BranchLog:
+    def create_branch_log(self, internal_branch_name: str) -> BranchLog:
         """
         Creates a uncommitted branch log object by the internal name given
 
@@ -675,10 +782,14 @@ class BaseRunLogStore(ABC, BaseModel):
             BranchLog: Uncommitted and initialized with defaults BranchLog object
         """
         # Create a new BranchLog
-        logger.info(f"{self.service_name} Creating a Branch Log : {internal_branch_name}")
+        logger.info(
+            f"{self.service_name} Creating a Branch Log : {internal_branch_name}"
+        )
         return BranchLog(internal_name=internal_branch_name, status=defaults.CREATED)
 
-    def get_branch_log(self, internal_branch_name: str, run_id: str, **kwargs) -> Union[BranchLog, RunLog]:
+    def get_branch_log(
+        self, internal_branch_name: str, run_id: str
+    ) -> Union[BranchLog, RunLog]:
         """
         Returns the branch log by the internal branch name for the run id
 
@@ -697,7 +808,7 @@ class BaseRunLogStore(ABC, BaseModel):
         branch, _ = run_log.search_branch_by_internal_name(internal_branch_name)
         return branch
 
-    def add_branch_log(self, branch_log: Union[BranchLog, RunLog], run_id: str, **kwargs):
+    def add_branch_log(self, branch_log: Union[BranchLog, RunLog], run_id: str):
         """
         The method should:
         # Get the run log
@@ -729,8 +840,7 @@ class BaseRunLogStore(ABC, BaseModel):
         step.branches[internal_branch_name] = branch_log  # type: ignore
         self.put_run_log(run_log)
 
-    #
-    def create_code_identity(self, **kwargs) -> CodeIdentity:
+    def create_code_identity(self) -> CodeIdentity:
         """
         Creates an uncommitted Code identity class
 
@@ -740,7 +850,7 @@ class BaseRunLogStore(ABC, BaseModel):
         logger.info(f"{self.service_name} Creating Code identity")
         return CodeIdentity()
 
-    def create_data_catalog(self, name: str, **kwargs) -> DataCatalog:
+    def create_data_catalog(self, name: str) -> DataCatalog:
         """
         Create a uncommitted data catalog object
 
@@ -753,6 +863,45 @@ class BaseRunLogStore(ABC, BaseModel):
         logger.info(f"{self.service_name} Creating Data Catalog for {name}")
         return DataCatalog(name=name)
 
+    def create_job_log(self) -> JobLog:
+        """
+        Creates a Job log and adds it to the db
+
+        Refer to BaseRunLogStore.create_job_log
+        """
+        logger.info(f"{self.service_name} Creating a Job Log and adding it to DB")
+        return JobLog(status=defaults.CREATED)
+
+    def get_job_log(self, run_id: str) -> JobLog:
+        """
+        Returns the run_log defined by id
+
+        Raises Exception if not found
+        """
+        logger.info(f"{self.service_name} Getting the run log from DB for {run_id}")
+        run_log = self.get_run_log_by_id(run_id)
+
+        try:
+            assert run_log.job
+        except AssertionError as exc:
+            raise exceptions.JobLogNotFoundError(run_id) from exc
+
+        return run_log.job
+
+    def add_job_log(self, run_id: str, job_log: JobLog):
+        """
+        Adds the job log to the run log
+
+        Args:
+            run_id (str): The run_id of the run
+            job_log (JobLog): The job log to add to the run log
+        """
+        logger.info(f"{self.service_name} Adding the job log to DB for: {run_id}")
+        run_log = self.get_run_log_by_id(run_id=run_id)
+        run_log.job = job_log
+        run_log.status = job_log.status
+        self.put_run_log(run_log=run_log)
+
 
 class BufferRunLogstore(BaseRunLogStore):
     """
@@ -775,7 +924,13 @@ class BufferRunLogstore(BaseRunLogStore):
     """
 
     service_name: str = "buffered"
-    run_log: Optional[RunLog] = Field(default=None, exclude=True)  # For a buffered Run Log, this is the database
+
+    run_log: Optional[RunLog] = Field(
+        default=None, exclude=True
+    )  # For a buffered Run Log, this is the database
+    job_log: Optional[JobLog] = Field(
+        default=None, exclude=True
+    )  # For a buffered Run Log, this is the database
 
     def get_summary(self) -> Dict[str, Any]:
         summary = {"Type": self.service_name, "Location": "Not persisted"}
@@ -790,7 +945,6 @@ class BufferRunLogstore(BaseRunLogStore):
         tag: str = "",
         original_run_id: str = "",
         status: str = defaults.CREATED,
-        **kwargs,
     ) -> RunLog:
         """
         # Creates a Run log
@@ -809,7 +963,7 @@ class BufferRunLogstore(BaseRunLogStore):
         )
         return self.run_log
 
-    def get_run_log_by_id(self, run_id: str, full: bool = False, **kwargs):
+    def get_run_log_by_id(self, run_id: str, full: bool = False):
         """
         # Returns the run_log defined by id
         # Raises Exception if not found
@@ -821,10 +975,15 @@ class BufferRunLogstore(BaseRunLogStore):
 
         raise exceptions.RunLogNotFoundError(run_id)
 
-    def put_run_log(self, run_log: RunLog, **kwargs):
+    def put_run_log(self, run_log: RunLog):
         """
         # Puts the run log in the db
         # Raises Exception if not found
         """
-        logger.info(f"{self.service_name} Putting the run log in the DB: {run_log.run_id}")
+        logger.info(
+            f"{self.service_name} Putting the run log in the DB: {run_log.run_id}"
+        )
         self.run_log = run_log
+
+
+import runnable.context as context  # noqa: F401, E402

runnable/defaults.py

@@ -1,6 +1,11 @@
-from enum import Enum
-from typing import TypedDict  # type: ignore[unused-ignore]
-from typing import Any, Dict, Mapping, Optional, Union
+from typing import (
+    Any,
+    Dict,
+    Mapping,
+    Optional,
+    TypedDict,  # type: ignore[unused-ignore]
+    Union,
+)
 
 from rich.style import Style
 from typing_extensions import TypeAlias
@@ -12,16 +17,6 @@ LOGGER_NAME = "runnable"
 LOG_LEVEL = "WARNING"
 
 
-class EXECUTION_PLAN(Enum):
-    """
-    The possible execution plans for a runnable job.
-    """
-
-    CHAINED = "chained"  #  121 relationship between run log and the dag.
-    UNCHAINED = "unchained"  # Only captures execution of steps, no relation.
-    INTERACTIVE = "interactive"  # used for interactive sessions
-
-
 # Type definitions
 class ServiceConfig(TypedDict):
     type: str
@@ -32,8 +27,7 @@ class RunnableConfig(TypedDict, total=False):
     run_log_store: Optional[ServiceConfig]
     secrets: Optional[ServiceConfig]
     catalog: Optional[ServiceConfig]
-    executor: Optional[ServiceConfig]
-    experiment_tracker: Optional[ServiceConfig]
+    pipeline_executor: Optional[ServiceConfig]
     pickler: Optional[ServiceConfig]
 
 
@@ -46,7 +40,6 @@ RUNNABLE_RUN_TAG = "RUNNABLE_RUN_TAG"
 
 # Interaction settings
 TRACK_PREFIX = "RUNNABLE_TRACK_"
-STEP_INDICATOR = "_STEP_"
 PARAMETER_PREFIX = "RUNNABLE_PRM_"
 MAP_VARIABLE = "RUNNABLE_MAP_VARIABLE"
 VARIABLE_PREFIX = "RUNNABLE_VAR_"
@@ -67,18 +60,14 @@ TRIGGERED = "TRIGGERED"
 
 # Node and Command settings
 COMMAND_TYPE = "python"
-NODE_SPEC_FILE = "node_spec.yaml"
 COMMAND_FRIENDLY_CHARACTER = "%"
-DEFAULT_CONTAINER_CONTEXT_PATH = "/opt/runnable/"
-DEFAULT_CONTAINER_DATA_PATH = "data/"
-DEFAULT_CONTAINER_OUTPUT_PARAMETERS = "parameters.json"
 
 # Default services
-DEFAULT_EXECUTOR = ServiceConfig(type="local", config={})
+DEFAULT_PIPELINE_EXECUTOR = ServiceConfig(type="local", config={})
+DEFAULT_JOB_EXECUTOR = ServiceConfig(type="local", config={})
 DEFAULT_RUN_LOG_STORE = ServiceConfig(type="file-system", config={})
 DEFAULT_CATALOG = ServiceConfig(type="file-system", config={})
 DEFAULT_SECRETS = ServiceConfig(type="env-secrets", config={})
-DEFAULT_EXPERIMENT_TRACKER = ServiceConfig(type="do-nothing", config={})
 DEFAULT_PICKLER = ServiceConfig(type="pickle", config={})
 
 # Map state
@@ -110,41 +99,10 @@ COMPUTE_DATA_FOLDER = "."
 # Secrets settings
 DOTENV_FILE_LOCATION = ".env"
 
-
-# Docker settings
-DOCKERFILE_NAME = "Dockerfile"
-DOCKERFILE_CONTENT = r"""# Python 3.8 Image without Dependecies
-FROM python:3.8
-
-LABEL maintainer="mesanthu@gmail.com"
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-${INSTALL_STYLE}
-
-ENV VIRTUAL_ENV=/opt/venv
-RUN python -m virtualenv --python=/usr/local/bin/python $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-${COPY_CONTENT}
-WORKDIR /app
-
-${INSTALL_REQUIREMENTS}
-"""
-GIT_ARCHIVE_NAME = "git_tracked"
 LEN_SHA_FOR_TAG = 8
 
-
-class ENTRYPOINT(Enum):
-    """
-    The possible container entrypoint types.
-    """
-
-    USER = "user"
-    SYSTEM = "system"
-
+# JOB CONFIG
+DEFAULT_JOB_NAME = "job"
 
 ## Logging settings