mlrun 1.8.0rc10__py3-none-any.whl → 1.8.0rc12__py3-none-any.whl

mlrun/common/constants.py

@@ -25,6 +25,7 @@ MYSQL_MEDIUMBLOB_SIZE_BYTES = 16 * 1024 * 1024
 MLRUN_LABEL_PREFIX = "mlrun/"
 DASK_LABEL_PREFIX = "dask.org/"
 NUCLIO_LABEL_PREFIX = "nuclio.io/"
+RESERVED_TAG_NAME_LATEST = "latest"
 
 
 class MLRunInternalLabels:

mlrun/common/schemas/__init__.py

@@ -146,8 +146,10 @@ from .model_monitoring import (
     GrafanaTable,
     GrafanaTimeSeriesTarget,
     ModelEndpoint,
+    ModelEndpointCreationStrategy,
     ModelEndpointList,
     ModelEndpointMetadata,
+    ModelEndpointSchema,
     ModelEndpointSpec,
     ModelEndpointStatus,
     ModelMonitoringMode,

mlrun/common/schemas/model_monitoring/__init__.py

@@ -26,6 +26,7 @@ from .constants import (
     FileTargetKind,
     FunctionURI,
     MetricData,
+    ModelEndpointCreationStrategy,
     ModelEndpointMonitoringMetricType,
     ModelEndpointSchema,
     ModelEndpointTarget,

mlrun/common/schemas/model_monitoring/constants.py

@@ -71,6 +71,12 @@ class ModelEndpointSchema(MonitoringStrEnum):
     DRIFT_MEASURES = "drift_measures"
 
 
+class ModelEndpointCreationStrategy(MonitoringStrEnum):
+    INPLACE = "inplace"
+    ARCHIVE = "archive"
+    OVERWRITE = "overwrite"
+
+
 class EventFieldType:
     FUNCTION_URI = "function_uri"
     FUNCTION = "function"

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -117,6 +117,10 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
     endpoint_type: EndpointType = EndpointType.NODE_EP
     uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
 
+    @classmethod
+    def mutable_fields(cls):
+        return ["labels"]
+
 
 class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
     model_uid: Optional[str] = ""
@@ -136,6 +140,21 @@ class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
     children_uids: Optional[list[str]] = []
     monitoring_feature_set_uri: Optional[str] = ""
 
+    @classmethod
+    def mutable_fields(cls):
+        return [
+            "model_uid",
+            "model_name",
+            "model_db_key",
+            "model_tag",
+            "model_class",
+            "function_uid",
+            "feature_names",
+            "label_names",
+            "children",
+            "children_uids",
+        ]
+
 
 class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
     state: Optional[str] = "unknown"  # will be updated according to the function state
@@ -152,6 +171,14 @@ class ModelEndpointStatus(ObjectStatus, ModelEndpointParser):
     drift_measures: Optional[dict] = {}
     drift_measures_timestamp: Optional[datetime] = None
 
+    @classmethod
+    def mutable_fields(cls):
+        return [
+            "monitoring_mode",
+            "first_request",
+            "last_request",
+        ]
+
 
 class ModelEndpoint(BaseModel):
     kind: ObjectKind = Field(ObjectKind.model_endpoint, const=True)
@@ -159,6 +186,14 @@ class ModelEndpoint(BaseModel):
     spec: ModelEndpointSpec
     status: ModelEndpointStatus
 
+    @classmethod
+    def mutable_fields(cls):
+        return (
+            ModelEndpointMetadata.mutable_fields()
+            + ModelEndpointSpec.mutable_fields()
+            + ModelEndpointStatus.mutable_fields()
+        )
+
     def flat_dict(self) -> dict[str, Any]:
         """Generate a flattened `ModelEndpoint` dictionary. The flattened dictionary result is important for storing
         the model endpoint object in the database.

mlrun/db/base.py

@@ -666,6 +666,7 @@ class RunDBInterface(ABC):
     def create_model_endpoint(
         self,
         model_endpoint: mlrun.common.schemas.ModelEndpoint,
+        creation_strategy: mlrun.common.schemas.ModelEndpointCreationStrategy = "inplace",
     ) -> mlrun.common.schemas.ModelEndpoint:
         pass
 
@@ -688,6 +689,7 @@ class RunDBInterface(ABC):
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         model_name: Optional[str] = None,
+        model_tag: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,

mlrun/db/httpdb.py

@@ -3582,11 +3582,17 @@ class HTTPRunDB(RunDBInterface):
     def create_model_endpoint(
         self,
         model_endpoint: mlrun.common.schemas.ModelEndpoint,
+        creation_strategy: mlrun.common.schemas.ModelEndpointCreationStrategy = "inplace",
     ) -> mlrun.common.schemas.ModelEndpoint:
         """
         Creates a DB record with the given model_endpoint record.
 
         :param model_endpoint: An object representing the model endpoint.
+        :param creation_strategy: model endpoint creation strategy :
+                            * overwrite - Create a new model endpoint and delete the last old one if it exists.
+                            * inplace - Use the existing model endpoint if it already exists (default).
+                            * archive - Preserve the old model endpoint and create a new one,
+                            tagging it as the latest.
 
         :return: The created model endpoint object.
         """
@@ -3596,6 +3602,9 @@ class HTTPRunDB(RunDBInterface):
             method=mlrun.common.types.HTTPMethod.POST,
             path=path,
             body=model_endpoint.json(),
+            params={
+                "creation_strategy": creation_strategy,
+            },
         )
         return mlrun.common.schemas.ModelEndpoint(**response.json())
 
@@ -3637,6 +3646,7 @@ class HTTPRunDB(RunDBInterface):
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         model_name: Optional[str] = None,
+        model_tag: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime] = None,
         end: Optional[datetime] = None,
@@ -3653,6 +3663,7 @@ class HTTPRunDB(RunDBInterface):
         :param function_name:   The name of the function
         :param function_tag:    The tag of the function
         :param model_name:      The name of the model
+        :param model_tag:       The tag of the model
         :param labels:          A list of labels to filter by. (see mlrun.common.schemas.LabelsModel)
         :param start:           The start time to filter by.Corresponding to the `created` field.
         :param end:             The end time to filter by. Corresponding to the `created` field.
@@ -3671,6 +3682,7 @@ class HTTPRunDB(RunDBInterface):
             params={
                 "name": name,
                 "model_name": model_name,
+                "model_tag": model_tag,
                 "function_name": function_name,
                 "function_tag": function_tag,
                 "label": labels,

mlrun/db/nopdb.py

@@ -575,6 +575,7 @@ class NopDB(RunDBInterface):
     def create_model_endpoint(
         self,
         model_endpoint: mlrun.common.schemas.ModelEndpoint,
+        creation_strategy: mlrun.common.schemas.ModelEndpointCreationStrategy = "inplace",
     ) -> mlrun.common.schemas.ModelEndpoint:
         pass
 
@@ -595,6 +596,7 @@ class NopDB(RunDBInterface):
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         model_name: Optional[str] = None,
+        model_tag: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,

mlrun/feature_store/steps.py

@@ -671,7 +671,7 @@ class SetEventMetadata(MapClass):
 
         self._tagging_funcs = []
 
-    def post_init(self, mode="sync"):
+    def post_init(self, mode="sync", **kwargs):
         def add_metadata(name, path, operator=str):
             def _add_meta(event):
                 value = get_in(event.body, path)

mlrun/model_monitoring/api.py

@@ -54,9 +54,10 @@ def get_or_create_model_endpoint(
     model_endpoint_name: str = "",
     endpoint_id: str = "",
     function_name: str = "",
+    function_tag: str = "latest",
     context: typing.Optional["mlrun.MLClientCtx"] = None,
     sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
-    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
     db_session=None,
 ) -> ModelEndpoint:
     """
@@ -70,8 +71,8 @@ def get_or_create_model_endpoint(
                                      under this endpoint (applicable only to new endpoint_id).
     :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record based
                                      on the provided `endpoint_id`.
-    :param function_name:            If a new model endpoint is created, use this function name for generating the
-                                     function URI (applicable only to new endpoint_id).
+    :param function_name:            If a new model endpoint is created, use this function name.
+    :param function_tag:             If a new model endpoint is created, use this function tag.
     :param context:                  MLRun context. If `function_name` not provided, use the context to generate the
                                      full function hash.
     :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
@@ -86,28 +87,32 @@ def get_or_create_model_endpoint(
     if not db_session:
         # Generate a runtime database
         db_session = mlrun.get_run_db()
+    model_endpoint = None
     try:
-        model_endpoint = db_session.get_model_endpoint(
-            project=project,
-            name=model_endpoint_name,
-            endpoint_id=endpoint_id,
-            function_name=function_name,
-        )
-        # If other fields provided, validate that they are correspond to the existing model endpoint data
-        _model_endpoint_validations(
-            model_endpoint=model_endpoint,
-            model_path=model_path,
-            sample_set_statistics=sample_set_statistics,
-        )
+        if endpoint_id:
+            model_endpoint = db_session.get_model_endpoint(
+                project=project,
+                name=model_endpoint_name,
+                endpoint_id=endpoint_id,
+            )
+            # If other fields provided, validate that they are correspond to the existing model endpoint data
+            _model_endpoint_validations(
+                model_endpoint=model_endpoint,
+                model_path=model_path,
+                sample_set_statistics=sample_set_statistics,
+            )
 
     except mlrun.errors.MLRunNotFoundError:
         # Create a new model endpoint with the provided details
+        pass
+    if not model_endpoint:
         model_endpoint = _generate_model_endpoint(
             project=project,
             db_session=db_session,
             model_path=model_path,
             model_endpoint_name=model_endpoint_name,
             function_name=function_name,
+            function_tag=function_tag,
             context=context,
             sample_set_statistics=sample_set_statistics,
             monitoring_mode=monitoring_mode,
@@ -333,9 +338,10 @@ def _generate_model_endpoint(
     model_path: str,
     model_endpoint_name: str,
     function_name: str,
+    function_tag: str,
     context: "mlrun.MLClientCtx",
     sample_set_statistics: dict[str, typing.Any],
-    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
 ) -> ModelEndpoint:
     """
     Write a new model endpoint record.
@@ -345,8 +351,8 @@ def _generate_model_endpoint(
     :param db_session:               A session that manages the current dialog with the database.
     :param model_path:               The model Store path.
     :param model_endpoint_name:      Model endpoint name will be presented under the new model endpoint.
-    :param function_name:            If a new model endpoint is created, use this function name for generating the
-                                     function URI.
+    :param function_name:            If a new model endpoint is created, use this function name.
+    :param function_tag:             If a new model endpoint is created, use this function tag.
     :param context:                  MLRun context. If function_name not provided, use the context to generate the
                                      full function hash.
     :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
@@ -374,7 +380,8 @@ def _generate_model_endpoint(
             endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.BATCH_EP,
         ),
         spec=mlrun.common.schemas.ModelEndpointSpec(
-            function_name=function_name,
+            function_name=function_name or "function",
+            function_tag=function_tag or "latest",
             model_name=model_obj.metadata.key if model_obj else None,
             model_uid=model_obj.metadata.uid if model_obj else None,
             model_tag=model_obj.metadata.tag if model_obj else None,

mlrun/model_monitoring/applications/base.py

@@ -148,6 +148,44 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             )
         return start, end
 
+    @classmethod
+    def deploy(
+        cls,
+        func_name: str,
+        func_path: Optional[str] = None,
+        image: Optional[str] = None,
+        handler: Optional[str] = None,
+        with_repo: Optional[bool] = False,
+        tag: Optional[str] = None,
+        requirements: Optional[Union[str, list[str]]] = None,
+        requirements_file: str = "",
+        **application_kwargs,
+    ) -> None:
+        """
+        Set the application to the current project and deploy it as a Nuclio serving function.
+        Required for your model monitoring application to work as a part of the model monitoring framework.
+
+        :param func_name: The name of the function.
+        :param func_path: The path of the function, :code:`None` refers to the current Jupyter notebook.
+
+        For the other arguments, refer to
+        :py:meth:`~mlrun.projects.MlrunProject.set_model_monitoring_function`.
+        """
+        project = cast("mlrun.MlrunProject", mlrun.get_current_project())
+        function = project.set_model_monitoring_function(
+            name=func_name,
+            func=func_path,
+            application_class=cls.__name__,
+            handler=handler,
+            image=image,
+            with_repo=with_repo,
+            requirements=requirements,
+            requirements_file=requirements_file,
+            tag=tag,
+            **application_kwargs,
+        )
+        function.deploy()
+
     @classmethod
     def evaluate(
         cls,
@@ -175,10 +213,10 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :param func_name: The name of the function. If not passed, the class name is used.
         :param tag:       An optional tag for the function.
         :param run_local: Whether to run the function locally or remotely.
-        :param sample_df: Optional - pandas data-frame as the current dataset.
-                          When set, it replaces the data read from the model endpoint's offline source.
-        :param feature_stats: Optional - statistics dictionary of the reference data.
-                              When set, it overrides the model endpoint's feature stats.
+        :param sample_data:       Optional - pandas data-frame as the current dataset.
+                                  When set, it replaces the data read from the model endpoint's offline source.
+        :param reference_data:    Optional - pandas data-frame of the reference dataset.
+                                  When set, its statistics override the model endpoint's feature statistics.
         :param image:             Docker image to run the job on.
         :param with_repo:         Whether to clone the current repo to the build source.
         :param requirements:      List of Python requirements to be installed in the image.

mlrun/projects/project.py

@@ -2117,13 +2117,13 @@ class MlrunProject(ModelObj):
 
     def set_model_monitoring_function(
         self,
+        name: str,
         func: typing.Union[str, mlrun.runtimes.RemoteRuntime, None] = None,
         application_class: typing.Union[
             str, mm_app.ModelMonitoringApplicationBase, None
         ] = None,
-        name: Optional[str] = None,
         image: Optional[str] = None,
-        handler=None,
+        handler: Optional[str] = None,
         with_repo: Optional[bool] = None,
         tag: Optional[str] = None,
         requirements: Optional[typing.Union[str, list[str]]] = None,
@@ -2135,7 +2135,7 @@ class MlrunProject(ModelObj):
         Note: to deploy the function after linking it to the project,
         call `fn.deploy()` where `fn` is the object returned by this method.
 
-        examples::
+        Example::
 
             project.set_model_monitoring_function(
                 name="myApp", application_class="MyApp", image="mlrun/mlrun"
@@ -2144,8 +2144,7 @@ class MlrunProject(ModelObj):
         :param func:                    Remote function object or spec/code URL. :code:`None` refers to the current
                                         notebook.
         :param name:                    Name of the function (under the project), can be specified with a tag to support
-                                        versions (e.g. myfunc:v1)
-                                        Default: job
+                                        versions (e.g. myfunc:v1).
         :param image:                   Docker image to be used, can also be specified in
                                         the function object/yaml
         :param handler:                 Default function handler to invoke (can only be set with .py/.ipynb files)
@@ -2183,12 +2182,13 @@ class MlrunProject(ModelObj):
 
     def create_model_monitoring_function(
         self,
+        name: str,
         func: Optional[str] = None,
         application_class: typing.Union[
             str,
             mm_app.ModelMonitoringApplicationBase,
+            None,
         ] = None,
-        name: Optional[str] = None,
         image: Optional[str] = None,
         handler: Optional[str] = None,
         with_repo: Optional[bool] = None,
@@ -2200,16 +2200,15 @@ class MlrunProject(ModelObj):
         """
         Create a monitoring function object without setting it to the project
 
-        examples::
+        Example::
 
             project.create_model_monitoring_function(
-                application_class_name="MyApp", image="mlrun/mlrun", name="myApp"
+                name="myApp", application_class="MyApp", image="mlrun/mlrun"
             )
 
         :param func:                    The function's code URL. :code:`None` refers to the current notebook.
         :param name:                    Name of the function, can be specified with a tag to support
-                                        versions (e.g. myfunc:v1)
-                                        Default: job
+                                        versions (e.g. myfunc:v1).
         :param image:                   Docker image to be used, can also be specified in
                                         the function object/yaml
         :param handler:                 Default function handler to invoke (can only be set with .py/.ipynb files)
@@ -3551,6 +3550,7 @@ class MlrunProject(ModelObj):
         self,
         name: Optional[str] = None,
         model_name: Optional[str] = None,
+        model_tag: Optional[str] = None,
         function_name: Optional[str] = None,
         function_tag: Optional[str] = None,
         labels: Optional[list[str]] = None,
@@ -3565,12 +3565,13 @@ class MlrunProject(ModelObj):
         model endpoint. This functions supports filtering by the following parameters:
         1) name
         2) model_name
-        3) function_name
-        4) function_tag
-        5) labels
-        6) top level
-        7) uids
-        8) start and end time, corresponding to the `created` field.
+        3) model_tag
+        4) function_name
+        5) function_tag
+        6) labels
+        7) top level
+        8) uids
+        9) start and end time, corresponding to the `created` field.
         By default, when no filters are applied, all available endpoints for the given project will be listed.
 
         In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
@@ -3599,6 +3600,7 @@ class MlrunProject(ModelObj):
             project=self.name,
             name=name,
             model_name=model_name,
+            model_tag=model_tag,
             function_name=function_name,
             function_tag=function_tag,
             labels=labels,

mlrun/runtimes/nuclio/serving.py

@@ -22,7 +22,7 @@ import nuclio
 from nuclio import KafkaTrigger
 
 import mlrun
-import mlrun.common.schemas
+import mlrun.common.schemas as schemas
 from mlrun.datastore import get_kafka_brokers_from_dict, parse_kafka_url
 from mlrun.model import ObjectList
 from mlrun.runtimes.function_reference import FunctionReference
@@ -362,6 +362,9 @@ class ServingRuntime(RemoteRuntime):
         handler: Optional[str] = None,
         router_step: Optional[str] = None,
         child_function: Optional[str] = None,
+        creation_strategy: Optional[
+            schemas.ModelEndpointCreationStrategy
+        ] = schemas.ModelEndpointCreationStrategy.INPLACE,
         **class_args,
     ):
         """add ml model and/or route to the function.
@@ -384,6 +387,11 @@ class ServingRuntime(RemoteRuntime):
         :param router_step: router step name (to determine which router we add the model to in graphs
                             with multiple router steps)
         :param child_function: child function name, when the model runs in a child function
+        :param creation_strategy: model endpoint creation strategy :
+                                    * overwrite - Create a new model endpoint and delete the last old one if it exists.
+                                    * inplace - Use the existing model endpoint if it already exists (default).
+                                    * archive - Preserve the old model endpoint and create a new one,
+                                    tagging it as the latest.
         :param class_args:  extra kwargs to pass to the model serving class __init__
                             (can be read in the model using .get_param(key) method)
         """
@@ -419,7 +427,12 @@ class ServingRuntime(RemoteRuntime):
         if class_name and hasattr(class_name, "to_dict"):
             if model_path:
                 class_name.model_path = model_path
-            key, state = params_to_step(class_name, key)
+            key, state = params_to_step(
+                class_name,
+                key,
+                model_endpoint_creation_strategy=creation_strategy,
+                endpoint_type=schemas.EndpointType.LEAF_EP,
+            )
         else:
             class_name = class_name or self.spec.default_class
             if class_name and not isinstance(class_name, str):
@@ -432,12 +445,22 @@ class ServingRuntime(RemoteRuntime):
                 model_path = str(model_path)
 
             if model_url:
-                state = new_remote_endpoint(model_url, **class_args)
+                state = new_remote_endpoint(
+                    model_url,
+                    creation_strategy=creation_strategy,
+                    endpoint_type=schemas.EndpointType.LEAF_EP,
+                    **class_args,
+                )
             else:
                 class_args = deepcopy(class_args)
                 class_args["model_path"] = model_path
                 state = TaskStep(
-                    class_name, class_args, handler=handler, function=child_function
+                    class_name,
+                    class_args,
+                    handler=handler,
+                    function=child_function,
+                    model_endpoint_creation_strategy=creation_strategy,
+                    endpoint_type=schemas.EndpointType.LEAF_EP,
                 )
 
         return graph.add_route(key, state)
@@ -581,7 +604,7 @@ class ServingRuntime(RemoteRuntime):
         project="",
         tag="",
         verbose=False,
-        auth_info: mlrun.common.schemas.AuthInfo = None,
+        auth_info: schemas.AuthInfo = None,
         builder_env: Optional[dict] = None,
         force_build: bool = False,
     ):

mlrun/serving/__init__.py

@@ -23,6 +23,10 @@ __all__ = [
     "QueueStep",
     "ErrorStep",
     "MonitoringApplicationStep",
+    "ModelRunnerStep",
+    "ModelRunner",
+    "Model",
+    "ModelSelector",
 ]
 
 from .routers import ModelRouter, VotingEnsemble  # noqa
@@ -33,6 +37,10 @@ from .states import (
     RouterStep,
     TaskStep,
     MonitoringApplicationStep,
+    ModelRunnerStep,
+    ModelRunner,
+    Model,
+    ModelSelector,
 )  # noqa
 from .v1_serving import MLModelServer, new_v1_model_server  # noqa
 from .v2_serving import V2ModelServer  # noqa

mlrun/serving/merger.py

@@ -74,7 +74,7 @@ class Merge(storey.Flow):
         self._queue_len = max_behind or 64  # default queue is 64 entries
         self._keys_queue = []
 
-    def post_init(self, mode="sync"):
+    def post_init(self, mode="sync", **kwargs):
         # auto detect number of uplinks or use user specified value
         self._uplinks = self.expected_num_events or (
             len(self._graph_step.after) if self._graph_step else 0

mlrun/serving/remote.py

@@ -14,6 +14,7 @@
 #
 import asyncio
 import json
+from copy import copy
 from typing import Optional
 
 import aiohttp
@@ -53,6 +54,7 @@ class RemoteStep(storey.SendToHttp):
         retries=None,
         backoff_factor=None,
         timeout=None,
+        headers_expression: Optional[str] = None,
         **kwargs,
     ):
         """class for calling remote endpoints
@@ -86,6 +88,7 @@ class RemoteStep(storey.SendToHttp):
         :param retries:     number of retries (in exponential backoff)
         :param backoff_factor: A backoff factor in seconds to apply between attempts after the second try
         :param timeout:     How long to wait for the server to send data before giving up, float in seconds
+        :param headers_expression: an expression for getting the request headers from the event, e.g. "event['headers']"
         """
         # init retry args for storey
         retries = default_retries if retries is None else retries
@@ -102,6 +105,7 @@ class RemoteStep(storey.SendToHttp):
         self.url = url
         self.url_expression = url_expression
         self.body_expression = body_expression
+        self.headers_expression = headers_expression
         self.headers = headers
         self.method = method
         self.return_json = return_json
@@ -114,8 +118,9 @@ class RemoteStep(storey.SendToHttp):
         self._session = None
         self._url_function_handler = None
         self._body_function_handler = None
+        self._headers_function_handler = None
 
-    def post_init(self, mode="sync"):
+    def post_init(self, mode="sync", **kwargs):
         self._endpoint = self.url
         if self.url and self.context:
             self._endpoint = self.context.get_remote_endpoint(self.url).strip("/")
@@ -131,6 +136,12 @@ class RemoteStep(storey.SendToHttp):
                 {"endpoint": self._endpoint, "context": self.context},
                 {},
             )
+        if self.headers_expression:
+            self._headers_function_handler = eval(
+                "lambda event: " + self.headers_expression,
+                {"context": self.context},
+                {},
+            )
         elif self.subpath:
             self._append_event_path = self.subpath == "$path"
             if not self._append_event_path:
@@ -205,7 +216,10 @@ class RemoteStep(storey.SendToHttp):
 
     def _generate_request(self, event, body):
         method = self.method or event.method or "POST"
-        headers = self.headers or {}
+        if self._headers_function_handler:
+            headers = self._headers_function_handler(body)
+        else:
+            headers = copy(self.headers) or {}
 
         if self._url_function_handler:
             url = self._url_function_handler(body)
@@ -216,10 +230,8 @@ class RemoteStep(storey.SendToHttp):
                 url = url + "/" + striped_path
             if striped_path:
                 headers[event_path_key] = event.path
-
         if event.id:
             headers[event_id_key] = event.id
-
         if method == "GET":
             body = None
         elif body is not None and not isinstance(body, (str, bytes)):
@@ -334,7 +346,7 @@ class BatchHttpRequests(_ConcurrentJobExecution):
     async def _cleanup(self):
         await self._client_session.close()
 
-    def post_init(self, mode="sync"):
+    def post_init(self, mode="sync", **kwargs):
         self._endpoint = self.url
         if self.url and self.context:
             self._endpoint = self.context.get_remote_endpoint(self.url).strip("/")