mlrun 1.8.0rc10__py3-none-any.whl → 1.8.0rc13__py3-none-any.whl

mlrun/execution.py

@@ -880,7 +880,7 @@ class MLClientCtx:
         tag: str = "",
         local_path: str = "",
         artifact_path: Optional[str] = None,
-        document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
+        document_loader_spec: DocumentLoaderSpec = DocumentLoaderSpec(),
         upload: Optional[bool] = False,
         labels: Optional[dict[str, str]] = None,
         target_path: Optional[str] = None,
@@ -891,22 +891,48 @@ class MLClientCtx:
 
         :param key: Artifact key
         :param tag: Version tag
-        :param local_path:    path to the local file we upload, will also be use
-                              as the destination subpath (under "artifact_path")
-        :param artifact_path:   Target artifact path (when not using the default)
-                                to define a subpath under the default location use:
-                                `artifact_path=context.artifact_subpath('data')`
-        :param document_loader: Spec to use to load the artifact as langchain document
+        :param local_path: path to the local file we upload, will also be use
+                        as the destination subpath (under "artifact_path")
+        :param artifact_path: Target artifact path (when not using the default)
+                            to define a subpath under the default location use:
+                            `artifact_path=context.artifact_subpath('data')`
+        :param document_loader_spec: Spec to use to load the artifact as langchain document.
+
+            By default, uses DocumentLoaderSpec() which initializes with:
+
+            * loader_class_name="langchain_community.document_loaders.TextLoader"
+            * src_name="file_path"
+            * kwargs=None
+
+            Can be customized for different document types, e.g.::
+
+                DocumentLoaderSpec(
+                    loader_class_name="langchain_community.document_loaders.PDFLoader",
+                    src_name="file_path",
+                    kwargs={"extract_images": True}
+                )
         :param upload: Whether to upload the artifact
         :param labels: Key-value labels
         :param target_path: Path to the local file
         :param kwargs: Additional keyword arguments
         :return: DocumentArtifact object
+
+        Example:
+            >>> # Log a PDF document with custom loader
+            >>> project.log_document(
+            ...     key="my_doc",
+            ...     local_path="path/to/doc.pdf",
+            ...     document_loader_spec=DocumentLoaderSpec(
+            ...         loader_class_name="langchain_community.document_loaders.PDFLoader",
+            ...         src_name="file_path",
+            ...         kwargs={"extract_images": True},
+            ...     ),
+            ... )
         """
         doc_artifact = DocumentArtifact(
             key=key,
             original_source=local_path or target_path,
-            document_loader=document_loader,
+            document_loader_spec=document_loader_spec,
             **kwargs,
         )
 
@@ -929,9 +955,15 @@ class MLClientCtx:
         )
         return self.get_artifact(key)
 
-    def get_artifact(self, key: str) -> Artifact:
-        artifact_uri = self._artifacts_manager.artifact_uris[key]
-        return self.get_store_resource(artifact_uri)
+    def get_artifact(
+        self, key, tag=None, iter=None, tree=None, uid=None
+    ) -> Optional[Artifact]:
+        if tag or iter or tree or uid:
+            project = self.get_project_object()
+            return project.get_artifact(key=key, tag=tag, iter=iter, tree=tree, uid=uid)
+        else:
+            artifact_uri = self._artifacts_manager.artifact_uris[key]
+            return self.get_store_resource(artifact_uri)
 
     def update_artifact(self, artifact_object: Artifact):
         """Update an artifact object in the DB and the cached uri"""

mlrun/feature_store/steps.py

@@ -671,7 +671,7 @@ class SetEventMetadata(MapClass):
 
         self._tagging_funcs = []
 
-    def post_init(self, mode="sync"):
+    def post_init(self, mode="sync", **kwargs):
         def add_metadata(name, path, operator=str):
             def _add_meta(event):
                 value = get_in(event.body, path)

mlrun/model_monitoring/api.py

@@ -54,9 +54,10 @@ def get_or_create_model_endpoint(
     model_endpoint_name: str = "",
     endpoint_id: str = "",
     function_name: str = "",
+    function_tag: str = "latest",
     context: typing.Optional["mlrun.MLClientCtx"] = None,
     sample_set_statistics: typing.Optional[dict[str, typing.Any]] = None,
-    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
     db_session=None,
 ) -> ModelEndpoint:
     """
@@ -70,8 +71,8 @@ def get_or_create_model_endpoint(
                                      under this endpoint (applicable only to new endpoint_id).
     :param endpoint_id:              Model endpoint unique ID. If not exist in DB, will generate a new record based
                                      on the provided `endpoint_id`.
-    :param function_name:            If a new model endpoint is created, use this function name for generating the
-                                     function URI (applicable only to new endpoint_id).
+    :param function_name:            If a new model endpoint is created, use this function name.
+    :param function_tag:             If a new model endpoint is created, use this function tag.
     :param context:                  MLRun context. If `function_name` not provided, use the context to generate the
                                      full function hash.
     :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
@@ -86,28 +87,32 @@ def get_or_create_model_endpoint(
     if not db_session:
         # Generate a runtime database
         db_session = mlrun.get_run_db()
+    model_endpoint = None
     try:
-        model_endpoint = db_session.get_model_endpoint(
-            project=project,
-            name=model_endpoint_name,
-            endpoint_id=endpoint_id,
-            function_name=function_name,
-        )
-        # If other fields provided, validate that they are correspond to the existing model endpoint data
-        _model_endpoint_validations(
-            model_endpoint=model_endpoint,
-            model_path=model_path,
-            sample_set_statistics=sample_set_statistics,
-        )
+        if endpoint_id:
+            model_endpoint = db_session.get_model_endpoint(
+                project=project,
+                name=model_endpoint_name,
+                endpoint_id=endpoint_id,
+            )
+            # If other fields provided, validate that they are correspond to the existing model endpoint data
+            _model_endpoint_validations(
+                model_endpoint=model_endpoint,
+                model_path=model_path,
+                sample_set_statistics=sample_set_statistics,
+            )
 
     except mlrun.errors.MLRunNotFoundError:
         # Create a new model endpoint with the provided details
+        pass
+    if not model_endpoint:
         model_endpoint = _generate_model_endpoint(
             project=project,
             db_session=db_session,
             model_path=model_path,
             model_endpoint_name=model_endpoint_name,
             function_name=function_name,
+            function_tag=function_tag,
             context=context,
             sample_set_statistics=sample_set_statistics,
             monitoring_mode=monitoring_mode,
@@ -333,9 +338,10 @@ def _generate_model_endpoint(
     model_path: str,
     model_endpoint_name: str,
     function_name: str,
+    function_tag: str,
     context: "mlrun.MLClientCtx",
     sample_set_statistics: dict[str, typing.Any],
-    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.disabled,
+    monitoring_mode: mm_constants.ModelMonitoringMode = mm_constants.ModelMonitoringMode.enabled,
 ) -> ModelEndpoint:
     """
     Write a new model endpoint record.
@@ -345,8 +351,8 @@ def _generate_model_endpoint(
     :param db_session:               A session that manages the current dialog with the database.
     :param model_path:               The model Store path.
     :param model_endpoint_name:      Model endpoint name will be presented under the new model endpoint.
-    :param function_name:            If a new model endpoint is created, use this function name for generating the
-                                     function URI.
+    :param function_name:            If a new model endpoint is created, use this function name.
+    :param function_tag:             If a new model endpoint is created, use this function tag.
     :param context:                  MLRun context. If function_name not provided, use the context to generate the
                                      full function hash.
     :param sample_set_statistics:    Dictionary of sample set statistics that will be used as a reference data for
@@ -374,7 +380,8 @@ def _generate_model_endpoint(
             endpoint_type=mlrun.common.schemas.model_monitoring.EndpointType.BATCH_EP,
         ),
         spec=mlrun.common.schemas.ModelEndpointSpec(
-            function_name=function_name,
+            function_name=function_name or "function",
+            function_tag=function_tag or "latest",
             model_name=model_obj.metadata.key if model_obj else None,
             model_uid=model_obj.metadata.uid if model_obj else None,
             model_tag=model_obj.metadata.tag if model_obj else None,

mlrun/model_monitoring/applications/_application_steps.py

@@ -136,7 +136,7 @@ class _PrepareMonitoringEvent(StepToDict):
         :param event: Application event.
         :return: Application context.
         """
-        application_context = MonitoringApplicationContext(
+        application_context = MonitoringApplicationContext._from_graph_ctx(
             application_name=self.application_name,
             event=event,
             model_endpoint_dict=self.model_endpoints,

mlrun/model_monitoring/applications/base.py

@@ -112,11 +112,10 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         def call_do_tracking(event: Optional[dict] = None):
             if event is None:
                 event = {}
-            monitoring_context = mm_context.MonitoringApplicationContext(
+            monitoring_context = mm_context.MonitoringApplicationContext._from_ml_ctx(
                 event=event,
                 application_name=self.__class__.__name__,
-                logger=context.logger,
-                artifacts_logger=context,
+                context=context,
                 sample_df=sample_data,
                 feature_stats=feature_stats,
             )
@@ -148,6 +147,44 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             )
         return start, end
 
+    @classmethod
+    def deploy(
+        cls,
+        func_name: str,
+        func_path: Optional[str] = None,
+        image: Optional[str] = None,
+        handler: Optional[str] = None,
+        with_repo: Optional[bool] = False,
+        tag: Optional[str] = None,
+        requirements: Optional[Union[str, list[str]]] = None,
+        requirements_file: str = "",
+        **application_kwargs,
+    ) -> None:
+        """
+        Set the application to the current project and deploy it as a Nuclio serving function.
+        Required for your model monitoring application to work as a part of the model monitoring framework.
+
+        :param func_name: The name of the function.
+        :param func_path: The path of the function, :code:`None` refers to the current Jupyter notebook.
+
+        For the other arguments, refer to
+        :py:meth:`~mlrun.projects.MlrunProject.set_model_monitoring_function`.
+        """
+        project = cast("mlrun.MlrunProject", mlrun.get_current_project())
+        function = project.set_model_monitoring_function(
+            name=func_name,
+            func=func_path,
+            application_class=cls.__name__,
+            handler=handler,
+            image=image,
+            with_repo=with_repo,
+            requirements=requirements,
+            requirements_file=requirements_file,
+            tag=tag,
+            **application_kwargs,
+        )
+        function.deploy()
+
     @classmethod
     def evaluate(
         cls,
@@ -175,10 +212,10 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :param func_name: The name of the function. If not passed, the class name is used.
         :param tag:       An optional tag for the function.
         :param run_local: Whether to run the function locally or remotely.
-        :param sample_df: Optional - pandas data-frame as the current dataset.
-                          When set, it replaces the data read from the model endpoint's offline source.
-        :param feature_stats: Optional - statistics dictionary of the reference data.
-                              When set, it overrides the model endpoint's feature stats.
+        :param sample_data:       Optional - pandas data-frame as the current dataset.
+                                  When set, it replaces the data read from the model endpoint's offline source.
+        :param reference_data:    Optional - pandas data-frame of the reference dataset.
+                                  When set, its statistics override the model endpoint's feature statistics.
         :param image:             Docker image to run the job on.
         :param with_repo:         Whether to clone the current repo to the build source.
         :param requirements:      List of Python requirements to be installed in the image.

mlrun/model_monitoring/applications/context.py

@@ -45,32 +45,6 @@ class _ArtifactsLogger(Protocol):
 
 
 class MonitoringApplicationContext:
-    """
-    The monitoring context holds all the relevant information for the monitoring application,
-    and also it can be used for logging artifacts and results.
-    The monitoring context has the following attributes:
-
-    :param application_name:        (str) The model monitoring application name.
-    :param project_name:            (str) The project name.
-    :param project:                 (MlrunProject) The project object.
-    :param logger:                  (mlrun.utils.Logger) MLRun logger.
-    :param nuclio_logger:           (nuclio.request.Logger) Nuclio logger.
-    :param sample_df_stats:         (FeatureStats) The new sample distribution dictionary.
-    :param feature_stats:           (FeatureStats) The train sample distribution dictionary.
-    :param sample_df:               (pd.DataFrame) The new sample DataFrame.
-    :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
-    :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
-    :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
-    :param endpoint_id:             (str) ID of the monitored model endpoint
-    :param endpoint_name:           (str) Name of the monitored model endpoint
-    :param output_stream_uri:       (str) URI of the output stream for results
-    :param model_endpoint:          (ModelEndpoint) The model endpoint object.
-    :param feature_names:           (list[str]) List of models feature names.
-    :param label_names:             (list[str]) List of models label names.
-    :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object,
-                                    and a list of extra data items.
-    """
-
     _logger_name = "monitoring-application"
 
     def __init__(
@@ -78,64 +52,51 @@ class MonitoringApplicationContext:
         *,
         application_name: str,
         event: dict[str, Any],
+        project: "mlrun.MlrunProject",
+        artifacts_logger: _ArtifactsLogger,
+        logger: mlrun.utils.Logger,
+        nuclio_logger: nuclio.request.Logger,
         model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
-        logger: Optional[mlrun.utils.Logger] = None,
-        graph_context: Optional[mlrun.serving.GraphContext] = None,
-        context: Optional["mlrun.MLClientCtx"] = None,
-        artifacts_logger: Optional[_ArtifactsLogger] = None,
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
     ) -> None:
         """
-        The :code:`__init__` method initializes a :code:`MonitoringApplicationContext` object
-        and has the following attributes.
-        Note: this object should not be instantiated manually.
-
-        :param application_name:    The application name.
-        :param event:               The instance data dictionary.
-        :param model_endpoint_dict: Optional - dictionary of model endpoints.
-        :param logger:              Optional - MLRun logger instance.
-        :param graph_context:       Optional - GraphContext instance.
-        :param context:             Optional - MLClientCtx instance.
-        :param artifacts_logger:    Optional - an object that can log artifacts,
-                                    typically :py:class:`~mlrun.projects.MlrunProject` or
-                                    :py:class:`~mlrun.execution.MLClientCtx`.
-        :param sample_df:           Optional - pandas data-frame as the current dataset.
-                                    When set, it replaces the data read from the offline source.
-        :param feature_stats:       Optional - statistics dictionary of the reference data.
-                                    When set, it overrides the model endpoint's feature stats.
+        The :code:`MonitoringApplicationContext` object holds all the relevant information for the
+        model monitoring application, and can be used for logging artifacts and messages.
+        The monitoring context has the following attributes:
+
+        :param application_name:        (str) The model monitoring application name.
+        :param project:                 (:py:class:`~mlrun.projects.MlrunProject`) The current MLRun project object.
+        :param project_name:            (str) The project name.
+        :param logger:                  (:py:class:`~mlrun.utils.Logger`) MLRun logger.
+        :param nuclio_logger:           (nuclio.request.Logger) Nuclio logger.
+        :param sample_df_stats:         (FeatureStats) The new sample distribution dictionary.
+        :param feature_stats:           (FeatureStats) The train sample distribution dictionary.
+        :param sample_df:               (pd.DataFrame) The new sample DataFrame.
+        :param start_infer_time:        (pd.Timestamp) Start time of the monitoring schedule.
+        :param end_infer_time:          (pd.Timestamp) End time of the monitoring schedule.
+        :param latest_request:          (pd.Timestamp) Timestamp of the latest request on this endpoint_id.
+        :param endpoint_id:             (str) ID of the monitored model endpoint
+        :param endpoint_name:           (str) Name of the monitored model endpoint
+        :param output_stream_uri:       (str) URI of the output stream for results
+        :param model_endpoint:          (ModelEndpoint) The model endpoint object.
+        :param feature_names:           (list[str]) List of models feature names.
+        :param label_names:             (list[str]) List of models label names.
+        :param model:                   (tuple[str, ModelArtifact, dict]) The model file, model spec object,
+                                        and a list of extra data items.
         """
         self.application_name = application_name
 
-        if graph_context:
-            self.project_name = graph_context.project
-            self.project = mlrun.load_project(url=self.project_name)
-        elif context:
-            potential_project = context.get_project_object()
-            if not potential_project:
-                raise mlrun.errors.MLRunValueError(
-                    "Could not load project from context"
-                )
-            self.project = potential_project
-            self.project_name = self.project.name
+        self.project = project
+        self.project_name = project.name
 
-        self._artifacts_logger: _ArtifactsLogger = artifacts_logger or self.project
+        self._artifacts_logger = artifacts_logger
 
         # MLRun Logger
-        self.logger = logger or mlrun.utils.create_logger(
-            level=mlrun.mlconf.log_level,
-            formatter_kind=mlrun.mlconf.log_formatter,
-            name=self._logger_name,
-        )
+        self.logger = logger
         # Nuclio logger - `nuclio.request.Logger`.
         # Note: this logger accepts keyword arguments only in its `_with` methods, e.g. `info_with`.
-        self.nuclio_logger = (
-            graph_context.logger
-            if graph_context
-            else nuclio.request.Logger(
-                level=mlrun.mlconf.log_level, name=self._logger_name
-            )
-        )
+        self.nuclio_logger = nuclio_logger
 
         # event data
         self.start_infer_time = pd.Timestamp(
@@ -166,6 +127,68 @@ class MonitoringApplicationContext:
             model_endpoint_dict.get(self.endpoint_id) if model_endpoint_dict else None
         )
 
+    @classmethod
+    def _from_ml_ctx(
+        cls,
+        context: "mlrun.MLClientCtx",
+        *,
+        application_name: str,
+        event: dict[str, Any],
+        model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
+        sample_df: Optional[pd.DataFrame] = None,
+        feature_stats: Optional[FeatureStats] = None,
+    ) -> "MonitoringApplicationContext":
+        project = context.get_project_object()
+        if not project:
+            raise mlrun.errors.MLRunValueError("Could not load project from context")
+        logger = context.logger
+        artifacts_logger = context
+        nuclio_logger = nuclio.request.Logger(
+            level=mlrun.mlconf.log_level, name=cls._logger_name
+        )
+        return cls(
+            application_name=application_name,
+            event=event,
+            model_endpoint_dict=model_endpoint_dict,
+            project=project,
+            logger=logger,
+            nuclio_logger=nuclio_logger,
+            artifacts_logger=artifacts_logger,
+            sample_df=sample_df,
+            feature_stats=feature_stats,
+        )
+
+    @classmethod
+    def _from_graph_ctx(
+        cls,
+        graph_context: mlrun.serving.GraphContext,
+        *,
+        application_name: str,
+        event: dict[str, Any],
+        model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
+        sample_df: Optional[pd.DataFrame] = None,
+        feature_stats: Optional[FeatureStats] = None,
+    ) -> "MonitoringApplicationContext":
+        project = mlrun.load_project(url=graph_context.project)
+        nuclio_logger = graph_context.logger
+        artifacts_logger = project
+        logger = mlrun.utils.create_logger(
+            level=mlrun.mlconf.log_level,
+            formatter_kind=mlrun.mlconf.log_formatter,
+            name=cls._logger_name,
+        )
+        return cls(
+            application_name=application_name,
+            event=event,
+            project=project,
+            model_endpoint_dict=model_endpoint_dict,
+            logger=logger,
+            nuclio_logger=nuclio_logger,
+            artifacts_logger=artifacts_logger,
+            sample_df=sample_df,
+            feature_stats=feature_stats,
+        )
+
     def _get_default_labels(self) -> dict[str, str]:
         labels = {
             mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),

mlrun/projects/pipelines.py

@@ -471,6 +471,7 @@ class _PipelineRunner(abc.ABC):
         namespace=None,
         source=None,
         notifications: typing.Optional[list[mlrun.model.Notification]] = None,
+        context: typing.Optional[mlrun.execution.MLClientCtx] = None,
     ) -> _PipelineRunStatus:
         pass
 
@@ -595,6 +596,7 @@ class _KFPRunner(_PipelineRunner):
         namespace=None,
         source=None,
         notifications: typing.Optional[list[mlrun.model.Notification]] = None,
+        context: typing.Optional[mlrun.execution.MLClientCtx] = None,
     ) -> _PipelineRunStatus:
         pipeline_context.set(project, workflow_spec)
         workflow_handler = _PipelineRunner._get_handler(
@@ -646,9 +648,7 @@ class _KFPRunner(_PipelineRunner):
                 )
         project.notifiers.push_pipeline_start_message(
             project.metadata.name,
-            project.get_param("commit_id", None),
-            run_id,
-            True,
+            context.uid,
         )
         pipeline_context.clear()
         return _PipelineRunStatus(run_id, cls, project=project, workflow=workflow_spec)
@@ -722,6 +722,7 @@ class _LocalRunner(_PipelineRunner):
         namespace=None,
         source=None,
         notifications: typing.Optional[list[mlrun.model.Notification]] = None,
+        context: typing.Optional[mlrun.execution.MLClientCtx] = None,
     ) -> _PipelineRunStatus:
         pipeline_context.set(project, workflow_spec)
         workflow_handler = _PipelineRunner._get_handler(
@@ -805,6 +806,7 @@ class _RemoteRunner(_PipelineRunner):
         namespace: typing.Optional[str] = None,
         source: typing.Optional[str] = None,
         notifications: typing.Optional[list[mlrun.model.Notification]] = None,
+        context: typing.Optional[mlrun.execution.MLClientCtx] = None,
     ) -> typing.Optional[_PipelineRunStatus]:
         workflow_name = normalize_workflow_name(name=name, project_name=project.name)
         workflow_id = None
@@ -1127,6 +1129,7 @@ def load_and_run_workflow(
         engine=engine,
         local=local,
         notifications=start_notifications,
+        context=context,
     )
     context.log_result(key="workflow_id", value=run.run_id)
     context.log_result(key="engine", value=run._engine.engine, commit=True)