runnable 0.34.0a1__py3-none-any.whl → 1.0.0__py3-none-any.whl

runnable/graph.py

@@ -329,7 +329,7 @@ def create_graph(dag_config: Dict[str, Any], internal_branch_name: str = "") ->
     Returns:
         Graph: The created graph object
     """
-    description: str = dag_config.get("description", None)
+    description: str | None = dag_config.get("description", None)
     start_at: str = cast(
         str, dag_config.get("start_at")
     )  # Let the start_at be relative to the graph

runnable/names.py

@@ -306,7 +306,7 @@ left = [
     "chalky",
     "moist",
     "formal",
-    "brute force",
+    "force",
     "ancient",
     "wan",
     "square",

runnable/nodes.py

@@ -7,7 +7,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
 import runnable.context as context
 from runnable import defaults, exceptions
 from runnable.datastore import StepLog
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
+from runnable.graph import Graph
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
@@ -41,6 +42,7 @@ class BaseNode(ABC, BaseModel):
 
     @property
     def _context(self):
+        assert isinstance(context.run_context, context.PipelineContext)
         return context.run_context
 
     model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=False)
@@ -80,7 +82,7 @@ class BaseNode(ABC, BaseModel):
 
     @classmethod
     def _resolve_map_placeholders(
-        cls, name: str, map_variable: TypeMapVariable = None
+        cls, name: str, map_variable: MapVariableType = None
     ) -> str:
         """
         If there is no map step used, then we just return the name as we find it.
@@ -131,7 +133,7 @@ class BaseNode(ABC, BaseModel):
 
         return name
 
-    def _get_step_log_name(self, map_variable: TypeMapVariable = None) -> str:
+    def _get_step_log_name(self, map_variable: MapVariableType = None) -> str:
         """
         For every step in the dag, there is a corresponding step log name.
         This method returns the step log name in dot path convention.
@@ -150,7 +152,7 @@ class BaseNode(ABC, BaseModel):
             self.internal_name, map_variable=map_variable
         )
 
-    def _get_branch_log_name(self, map_variable: TypeMapVariable = None) -> str:
+    def _get_branch_log_name(self, map_variable: MapVariableType = None) -> str:
         """
         For nodes that are internally branches, this method returns the branch log name.
         The branch log name is in dot path convention.
@@ -218,7 +220,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def _get_branch_by_name(self, branch_name: str):
+    def _get_branch_by_name(self, branch_name: str) -> Graph:
         """
         Retrieve a branch by name.
 
@@ -279,7 +281,7 @@ class BaseNode(ABC, BaseModel):
     def execute(
         self,
         mock=False,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         attempt_number: int = 1,
     ) -> StepLog:
         """
@@ -299,7 +301,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         """
         This function would be called to set up the execution of the individual
         branches of a composite node.
@@ -314,7 +316,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def fan_out(self, map_variable: TypeMapVariable = None):
+    def fan_out(self, map_variable: MapVariableType = None):
         """
         This function would be called to set up the execution of the individual
         branches of a composite node.
@@ -330,7 +332,7 @@ class BaseNode(ABC, BaseModel):
         """
 
     @abstractmethod
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         """
         This function would be called to tear down the execution of the individual
         branches of a composite node.
@@ -409,11 +411,13 @@ class TraversalNode(BaseNode):
         return self.overrides.get(executor_type) or ""
 
 
+# Unfortunately, this is defined in 2 places. Look in SDK
 class CatalogStructure(BaseModel):
     model_config = ConfigDict(extra="forbid")  # Need to forbid
 
     get: List[str] = Field(default_factory=list)
     put: List[str] = Field(default_factory=list)
+    store_copy: bool = Field(default=True, alias="store_copy")
 
 
 class ExecutableNode(TraversalNode):
@@ -439,17 +443,17 @@ class ExecutableNode(TraversalNode):
             "This is an executable node and does not have branches"
         )
 
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         raise exceptions.NodeMethodCallError(
             "This is an executable node and does not have a graph"
         )
 
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         raise exceptions.NodeMethodCallError(
             "This is an executable node and does not have a fan in"
         )
 
-    def fan_out(self, map_variable: TypeMapVariable = None):
+    def fan_out(self, map_variable: MapVariableType = None):
         raise exceptions.NodeMethodCallError(
             "This is an executable node and does not have a fan out"
         )
@@ -475,7 +479,7 @@ class CompositeNode(TraversalNode):
     def execute(
         self,
         mock=False,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
         attempt_number: int = 1,
     ) -> StepLog:
         raise exceptions.NodeMethodCallError(
@@ -505,15 +509,15 @@ class TerminalNode(BaseNode):
     def _get_max_attempts(self) -> int:
         return 1
 
-    def execute_as_graph(self, map_variable: TypeMapVariable = None):
+    def execute_as_graph(self, map_variable: MapVariableType = None):
         raise exceptions.TerminalNodeError()
 
-    def fan_in(self, map_variable: TypeMapVariable = None):
+    def fan_in(self, map_variable: MapVariableType = None):
         raise exceptions.TerminalNodeError()
 
     def fan_out(
         self,
-        map_variable: TypeMapVariable = None,
+        map_variable: MapVariableType = None,
     ):
         raise exceptions.TerminalNodeError()
 

runnable/parameters.py

@@ -1,16 +1,16 @@
+import argparse
 import inspect
 import json
 import logging
 import os
-from typing import Any, Dict, Type
+from typing import Any, Dict, Type, get_origin
 
-import pydantic
 from pydantic import BaseModel, ConfigDict
 from typing_extensions import Callable
 
 from runnable import defaults
 from runnable.datastore import JsonParameter, ObjectParameter
-from runnable.defaults import TypeMapVariable
+from runnable.defaults import MapVariableType
 from runnable.utils import remove_prefix
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -48,15 +48,40 @@ def get_user_set_parameters(remove: bool = False) -> Dict[str, JsonParameter]:
     return parameters
 
 
+def return_json_parameters(params: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Returns the parameters as a JSON serializable dictionary.
+
+    Args:
+        params (dict): The parameters to serialize.
+
+    Returns:
+        dict: The JSON serializable dictionary.
+    """
+    return_params = {}
+    for key, value in params.items():
+        if isinstance(value, ObjectParameter):
+            continue
+
+        return_params[key] = value.get_value()
+    return return_params
+
+
 def filter_arguments_for_func(
     func: Callable[..., Any],
     params: Dict[str, Any],
-    map_variable: TypeMapVariable = None,
+    map_variable: MapVariableType = None,
 ) -> Dict[str, Any]:
     """
     Inspects the function to be called as part of the pipeline to find the arguments of the function.
-    Matches the function arguments to the parameters available either by command line or by up stream steps.
+    Matches the function arguments to the parameters available either by static parameters or by up stream steps.
 
+    The function "func" signature could be:
+    - def my_function(arg1: int, arg2: str, arg3: float):
+    - def my_function(arg1: int, arg2: str, arg3: float, **kwargs):
+        in this case, we would need to send in remaining keyword arguments as a dictionary.
+    - def my_function(arg1: int, arg2: str, arg3: float, args: argparse.Namespace):
+        In this case, we need to send the rest of the parameters as attributes of the args object.
 
     Args:
         func (Callable): The function to inspect
@@ -72,63 +97,95 @@ def filter_arguments_for_func(
         params[key] = JsonParameter(kind="json", value=v)
 
     bound_args = {}
-    unassigned_params = set(params.keys())
-    # Check if VAR_KEYWORD is used, it is we send back everything
-    # If **kwargs is present in the function signature, we send back everything
-    for name, value in function_args.items():
-        if value.kind != inspect.Parameter.VAR_KEYWORD:
-            continue
-        # Found VAR_KEYWORD, we send back everything as found
-        for key, value in params.items():
-            bound_args[key] = params[key].get_value()
-
-        return bound_args
+    var_keyword_param = None
+    namespace_param = None
 
-    # Lets return what is asked for then!!
+    # First pass: Handle regular parameters and identify special parameters
     for name, value in function_args.items():
         # Ignore any *args
         if value.kind == inspect.Parameter.VAR_POSITIONAL:
             logger.warning(f"Ignoring parameter {name} as it is VAR_POSITIONAL")
             continue
 
-        if name not in params:
-            # No parameter of this name was provided
-            if value.default == inspect.Parameter.empty:
-                # No default value is given in the function signature. error as parameter is required.
-                raise ValueError(
-                    f"Parameter {name} is required for {func.__name__} but not provided"
-                )
-            # default value is given in the function signature, nothing further to do.
+        # Check for **kwargs parameter, we need to send in all the unnamed values in this as a dict
+        if value.kind == inspect.Parameter.VAR_KEYWORD:
+            var_keyword_param = name
             continue
 
-        param_value = params[name]
-
-        if type(value.annotation) in [
-            BaseModel,
-            pydantic._internal._model_construction.ModelMetaclass,
-        ] and not isinstance(param_value, ObjectParameter):
-            # Even if the annotation is a pydantic model, it can be passed as an object parameter
-            # We try to cast it as a pydantic model if asked
-            named_param = params[name].get_value()
-
-            if not isinstance(named_param, dict):
-                # A case where the parameter is a one attribute model
-                named_param = {name: named_param}
-
-            bound_model = bind_args_for_pydantic_model(named_param, value.annotation)
-            bound_args[name] = bound_model
+        # Check for argparse.Namespace parameter, we need to send in all the unnamed values in this as a namespace
+        if value.annotation == argparse.Namespace:
+            namespace_param = name
+            continue
 
-        elif value.annotation in [str, int, float, bool]:
-            # Cast it if its a primitive type. Ensure the type matches the annotation.
-            bound_args[name] = value.annotation(params[name].get_value())
+        # Handle regular parameters
+        if name not in params:
+            if value.default != inspect.Parameter.empty:
+                # Default value is given in the function signature, we can use it
+                bound_args[name] = value.default
+            else:
+                # This is a required parameter that's missing - error immediately
+                raise ValueError(
+                    f"Function {func.__name__} has required parameter '{name}' that is not present in the parameters"
+                )
         else:
-            bound_args[name] = params[name].get_value()
-
-        unassigned_params.remove(name)
-
-        params = {
-            key: params[key] for key in unassigned_params
-        }  # remove keys from params if they are assigned
+            # We have a parameter of this name, lets bind it
+            param_value = params[name]
+
+            if (issubclass(value.annotation, BaseModel)) and not isinstance(
+                param_value, ObjectParameter
+            ):
+                # Even if the annotation is a pydantic model, it can be passed as an object parameter
+                # We try to cast it as a pydantic model if asked
+                named_param = params[name].get_value()
+
+                if not isinstance(named_param, dict):
+                    # A case where the parameter is a one attribute model
+                    named_param = {name: named_param}
+
+                bound_model = bind_args_for_pydantic_model(
+                    named_param, value.annotation
+                )
+                bound_args[name] = bound_model
+
+            elif value.annotation is not inspect.Parameter.empty and callable(
+                value.annotation
+            ):
+                # Cast it if its a primitive type. Ensure the type matches the annotation.
+                try:
+                    # Handle typing generics like Dict[str, int], List[str] by using their origin
+                    origin = get_origin(value.annotation)
+                    if origin is not None:
+                        # For generics like Dict[str, int], use dict() instead of Dict[str, int]()
+                        bound_args[name] = origin(params[name].get_value())
+                    else:
+                        # Regular callable types like int, str, float, etc.
+                        bound_args[name] = value.annotation(params[name].get_value())
+                except (ValueError, TypeError) as e:
+                    annotation_name = getattr(
+                        value.annotation, "__name__", str(value.annotation)
+                    )
+                    raise ValueError(
+                        f"Cannot cast parameter '{name}' to {annotation_name}: {e}"
+                    )
+            else:
+                # We do not know type of parameter, we send the value as found
+                bound_args[name] = params[name].get_value()
+
+    # Find extra parameters (parameters in params but not consumed by regular function parameters)
+    consumed_param_names = set(bound_args.keys())
+    extra_params = {k: v for k, v in params.items() if k not in consumed_param_names}
+
+    # Second pass: Handle **kwargs and argparse.Namespace parameters
+    if var_keyword_param is not None:
+        # Function accepts **kwargs - add all extra parameters directly to bound_args
+        for param_name, param_value in extra_params.items():
+            bound_args[param_name] = param_value.get_value()
+    elif namespace_param is not None:
+        # Function accepts argparse.Namespace - create namespace with extra parameters
+        args_namespace = argparse.Namespace()
+        for param_name, param_value in extra_params.items():
+            setattr(args_namespace, param_name, param_value.get_value())
+        bound_args[namespace_param] = args_namespace
 
     return bound_args