runnable 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

runnable/datastore.py

@@ -113,7 +113,7 @@ class ObjectParameter(BaseModel):
             return context.run_context.return_objects[self.value]
 
         # If the object was serialised, get it from the catalog
-        catalog_handler = context.run_context.catalog_handler
+        catalog_handler = context.run_context.catalog
         catalog_handler.get(name=self.file_name)
         obj = context.run_context.pickler.load(path=self.file_name)
         os.remove(self.file_name)  # Remove after loading
@@ -127,7 +127,7 @@ class ObjectParameter(BaseModel):
         # If the object was serialised, put it in the catalog
         context.run_context.pickler.dump(data=data, path=self.file_name)
 
-        catalog_handler = context.run_context.catalog_handler
+        catalog_handler = context.run_context.catalog
         catalog_handler.put(name=self.file_name)
         os.remove(self.file_name)  # Remove after loading
 
@@ -400,7 +400,7 @@ class RunLog(BaseModel):
         summary["Unique execution id"] = self.run_id
         summary["status"] = self.status
 
-        summary["Catalog Location"] = _context.catalog_handler.get_summary()
+        summary["Catalog Location"] = _context.catalog.get_summary()
         summary["Full Run log present at: "] = _context.run_log_store.get_summary()
 
         run_log = _context.run_log_store.get_run_log_by_id(
@@ -467,23 +467,29 @@ class RunLog(BaseModel):
         current_step = None
         current_branch = None
 
-        for i in range(len(dot_path)):
-            if i % 2:
-                # Its odd, so we are in branch
-                # Get the branch that holds the step
-                current_branch = current_step.branches[".".join(dot_path[: i + 1])]  # type: ignore
-                current_steps = current_branch.steps
-                logger.debug(f"Finding branch {i_name} in branch: {current_branch}")
-            else:
-                # Its even, so we are in step, we start here!
-                # Get the step that holds the branch
-                current_step = current_steps[".".join(dot_path[: i + 1])]
-                logger.debug(f"Finding branch for {i_name} in step: {current_step}")
-
-        logger.debug(f"current branch : {current_branch}, current step {current_step}")
-        if current_branch and current_step:
-            return current_branch, current_step
-
+        try:
+            for i in range(len(dot_path)):
+                if i % 2:
+                    # Its odd, so we are in branch
+                    # Get the branch that holds the step
+                    current_branch = current_step.branches[".".join(dot_path[: i + 1])]  # type: ignore
+                    current_steps = current_branch.steps
+                    logger.debug(f"Finding branch {i_name} in branch: {current_branch}")
+                else:
+                    # Its even, so we are in step, we start here!
+                    # Get the step that holds the branch
+                    current_step = current_steps[".".join(dot_path[: i + 1])]
+                    logger.debug(f"Finding branch for {i_name} in step: {current_step}")
+
+            logger.debug(
+                f"current branch : {current_branch}, current step {current_step}"
+            )
+            if current_branch and current_step:
+                return current_branch, current_step
+        except KeyError as _e:
+            raise exceptions.BranchLogNotFoundError(self.run_id, i_name) from _e
+
+        # If we are here, we have not found the branch
         raise exceptions.BranchLogNotFoundError(self.run_id, i_name)
 
     def search_step_by_internal_name(
@@ -513,23 +519,31 @@ class RunLog(BaseModel):
         current_steps = self.steps
         current_step = None
         current_branch = None
-        for i in range(len(dot_path)):
-            if i % 2:
-                # Its odd, so we are in brach name
-                current_branch = current_step.branches[".".join(dot_path[: i + 1])]  # type: ignore
-                current_steps = current_branch.steps
-                logger.debug(
-                    f"Finding step log for {i_name} in branch: {current_branch}"
-                )
-            else:
-                # Its even, so we are in step, we start here!
-                current_step = current_steps[".".join(dot_path[: i + 1])]
-                logger.debug(f"Finding step log for {i_name} in step: {current_step}")
-
-        logger.debug(f"current branch : {current_branch}, current step {current_step}")
-        if current_branch and current_step:
-            return current_step, current_branch
-
+        try:
+            for i in range(len(dot_path)):
+                if i % 2:
+                    # Its odd, so we are in brach name
+                    current_branch = current_step.branches[".".join(dot_path[: i + 1])]  # type: ignore
+                    current_steps = current_branch.steps
+                    logger.debug(
+                        f"Finding step log for {i_name} in branch: {current_branch}"
+                    )
+                else:
+                    # Its even, so we are in step, we start here!
+                    current_step = current_steps[".".join(dot_path[: i + 1])]
+                    logger.debug(
+                        f"Finding step log for {i_name} in step: {current_step}"
+                    )
+
+            logger.debug(
+                f"current branch : {current_branch}, current step {current_step}"
+            )
+            if current_branch and current_step:
+                return current_step, current_branch
+        except KeyError as _e:
+            raise exceptions.StepLogNotFoundError(self.run_id, i_name) from _e
+
+        # If we are here, we have not found the step
         raise exceptions.StepLogNotFoundError(self.run_id, i_name)
 
 
@@ -548,23 +562,6 @@ class BaseRunLogStore(ABC, BaseModel):
     def _context(self):
         return context.run_context
 
-        """
-        Retrieves a Job log from the database using the config and the job_id
-
-        Args:
-            job_id (str): The job_id of the job
-
-        Returns:
-            JobLog: The JobLog object identified by the job_id
-
-        Logically the method should:
-            * Returns the job_log defined by id from the data store defined by the config
-
-        Raises:
-            NotImplementedError: This is a base class and therefore has no default implementation
-            JobLogNotFoundError: If the job log for job_id is not found in the datastore
-        """
-
     @abstractmethod
     def create_run_log(
         self,

runnable/defaults.py

@@ -1,11 +1,4 @@
-from typing import (
-    Any,
-    Dict,
-    Mapping,
-    Optional,
-    TypedDict,  # type: ignore[unused-ignore]
-    Union,
-)
+from typing import Any, Dict, Optional, Union
 
 from rich.style import Style
 from typing_extensions import TypeAlias
@@ -17,26 +10,12 @@ LOGGER_NAME = "runnable"
 LOG_LEVEL = "WARNING"
 
 
-# Type definitions
-class ServiceConfig(TypedDict):
-    type: str
-    config: Mapping[str, Any]
-
-
-class RunnableConfig(TypedDict, total=False):
-    run_log_store: Optional[ServiceConfig]
-    secrets: Optional[ServiceConfig]
-    catalog: Optional[ServiceConfig]
-    pipeline_executor: Optional[ServiceConfig]
-    pickler: Optional[ServiceConfig]
-
-
-TypeMapVariable: TypeAlias = Optional[Dict[str, Union[str, int, float]]]
-
+MapVariableType: TypeAlias = Optional[Dict[str, Union[str, int, float]]]
 
 # Config file environment variable
-RUNNABLE_CONFIG_FILE = "RUNNABLE_CONFIG_FILE"
+RUNNABLE_CONFIGURATION_FILE = "RUNNABLE_CONFIGURATION_FILE"
 RUNNABLE_RUN_TAG = "RUNNABLE_RUN_TAG"
+RUNNABLE_PARAMETERS_FILE = "RUNNABLE_PARAMETERS_FILE"
 
 # Interaction settings
 TRACK_PREFIX = "RUNNABLE_TRACK_"
@@ -56,19 +35,20 @@ CREATED = "CREATED"
 PROCESSING = "PROCESSING"
 SUCCESS = "SUCCESS"
 FAIL = "FAIL"
-TRIGGERED = "TRIGGERED"
 
 # Node and Command settings
 COMMAND_TYPE = "python"
 COMMAND_FRIENDLY_CHARACTER = "%"
 
 # Default services
-DEFAULT_PIPELINE_EXECUTOR = ServiceConfig(type="local", config={})
-DEFAULT_JOB_EXECUTOR = ServiceConfig(type="local", config={})
-DEFAULT_RUN_LOG_STORE = ServiceConfig(type="file-system", config={})
-DEFAULT_CATALOG = ServiceConfig(type="file-system", config={})
-DEFAULT_SECRETS = ServiceConfig(type="env-secrets", config={})
-DEFAULT_PICKLER = ServiceConfig(type="pickle", config={})
+DEFAULT_SERVICES: dict[str, Any] = {
+    "pipeline_executor": {"type": "local", "config": {}},
+    "job_executor": {"type": "local", "config": {}},
+    "run_log_store": {"type": "file-system", "config": {}},
+    "catalog": {"type": "file-system", "config": {}},
+    "pickler": {"type": "pickle", "config": {}},
+    "secrets": {"type": "env-secrets", "config": {}},
+}
 
 # Map state
 MAP_PLACEHOLDER = "map_variable_placeholder"
@@ -83,8 +63,6 @@ MAX_TIME = 86400  # 1 day in seconds
 # User extensions
 USER_CONFIG_FILE = "runnable-config.yaml"
 
-# Executor settings
-ENABLE_PARALLEL = False
 
 # RUN log store settings
 LOG_LOCATION_FOLDER = ".run_log_store"