mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc3__py3-none-any.whl

mlrun/projects/project.py

@@ -41,6 +41,7 @@ import mlrun.artifacts.model
 import mlrun.common.formatters
 import mlrun.common.helpers
 import mlrun.common.runtimes.constants
+import mlrun.common.schemas.alert
 import mlrun.common.schemas.artifact
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
@@ -49,6 +50,7 @@ import mlrun.k8s_utils
 import mlrun.lists
 import mlrun.model_monitoring.applications as mm_app
 import mlrun.runtimes
+import mlrun.runtimes.mounts
 import mlrun.runtimes.nuclio.api_gateway
 import mlrun.runtimes.pod
 import mlrun.runtimes.utils
@@ -56,14 +58,25 @@ import mlrun.serving
 import mlrun.utils
 import mlrun.utils.regex
 import mlrun_pipelines.common.models
-import mlrun_pipelines.mounts
 from mlrun.alerts.alert import AlertConfig
-from mlrun.common.schemas.alert import AlertTemplate
-from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
+from mlrun.datastore.datastore_profile import (
+    DatastoreProfile,
+    DatastoreProfile2Json,
+    VectorStoreProfile,
+    datastore_profile_read,
+)
+from mlrun.datastore.vectorstore import VectorStoreCollection
 from mlrun.runtimes.nuclio.function import RemoteRuntime
 from mlrun_pipelines.models import PipelineNodeWrapper
 
-from ..artifacts import Artifact, ArtifactProducer, DatasetArtifact, ModelArtifact
+from ..artifacts import (
+    Artifact,
+    ArtifactProducer,
+    DatasetArtifact,
+    DocumentArtifact,
+    DocumentLoaderSpec,
+    ModelArtifact,
+)
 from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifact_path
 from ..datastore import store_manager
 from ..features import Feature
@@ -1520,6 +1533,10 @@ class MlrunProject(ModelObj):
                     is_retained_producer=is_retained_producer,
                 )
 
+    def update_artifact(self, artifact_object: Artifact):
+        artifacts_manager = self._get_artifact_manager()
+        artifacts_manager.update_artifact(artifact_object, artifact_object)
+
     def _get_artifact_manager(self):
         if self._artifact_manager:
             return self._artifact_manager
@@ -1839,6 +1856,72 @@ class MlrunProject(ModelObj):
         )
         return item
 
+    def get_or_create_vector_store_collection(
+        self,
+        collection_name: str,
+        profile: Union[str, VectorStoreProfile],
+        **kwargs,
+    ) -> VectorStoreCollection:
+        """
+        Create or retrieve a VectorStoreCollection.
+
+        :param collection_name: Name of the collection
+        :param profile: Name of the VectorStoreProfile or a VectorStoreProfile object
+        :param kwargs: Additional arguments for the VectorStoreCollection
+        :return: VectorStoreCollection object
+        """
+        if isinstance(profile, str):
+            profile = datastore_profile_read(f"ds://{profile}")
+
+        if not isinstance(profile, VectorStoreProfile):
+            raise ValueError(
+                "Profile must be a VectorStoreProfile object or a profile name"
+            )
+        return VectorStoreCollection(
+            profile.vector_store_class,
+            self,
+            profile.name,
+            collection_name,
+            **profile.attributes(kwargs),
+        )
+
+    def log_document(
+        self,
+        key: str,
+        artifact_path: Optional[str] = None,
+        document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
+        tag: str = "",
+        upload: Optional[bool] = False,
+        labels: Optional[dict[str, str]] = None,
+        **kwargs,
+    ) -> DocumentArtifact:
+        """
+        Log a document as an artifact.
+
+        :param key: Artifact key
+        :param target_path: Path to the local file
+        :param artifact_path: Target path for artifact storage
+        :param document_loader: Spec to use to load the artifact as langchain document
+        :param tag: Version tag
+        :param upload: Whether to upload the artifact
+        :param labels: Key-value labels
+        :param kwargs: Additional keyword arguments
+        :return: DocumentArtifact object
+        """
+        doc_artifact = DocumentArtifact(
+            key=key,
+            document_loader=document_loader,
+            **kwargs,
+        )
+
+        return self.log_artifact(
+            doc_artifact,
+            artifact_path=artifact_path,
+            tag=tag,
+            upload=upload,
+            labels=labels,
+        )
+
     def import_artifact(
         self, item_path: str, new_key=None, artifact_path=None, tag=None
     ):
@@ -1995,8 +2078,6 @@ class MlrunProject(ModelObj):
         :param application_kwargs:      Additional keyword arguments to be passed to the
                                         monitoring application's constructor.
         """
-
-        function_object: RemoteRuntime = None
         (
             resolved_function_name,
             function_object,
@@ -2094,7 +2175,6 @@ class MlrunProject(ModelObj):
     ) -> tuple[str, mlrun.runtimes.BaseRuntime, dict]:
         import mlrun.model_monitoring.api
 
-        function_object: RemoteRuntime = None
         kind = None
         if (isinstance(func, str) or func is None) and application_class is not None:
             kind = mlrun.run.RuntimeKinds.serving
@@ -2133,9 +2213,6 @@ class MlrunProject(ModelObj):
             mm_constants.ModelMonitoringAppLabel.VAL,
         )
 
-        if not mlrun.mlconf.is_ce_mode():
-            function_object.apply(mlrun.mount_v3io())
-
         return resolved_function_name, function_object, func
 
     def _wait_for_functions_deployment(self, function_names: list[str]) -> None:
@@ -3390,6 +3467,61 @@ class MlrunProject(ModelObj):
                 "and set `rebuild_images=True`"
             )
 
+    def list_model_endpoints(
+        self,
+        model: Optional[str] = None,
+        function: Optional[str] = None,
+        labels: Optional[list[str]] = None,
+        start: str = "now-1h",
+        end: str = "now",
+        top_level: bool = False,
+        uids: Optional[list[str]] = None,
+    ) -> list[mlrun.model_monitoring.model_endpoint.ModelEndpoint]:
+        """
+        Returns a list of `ModelEndpoint` objects. Each `ModelEndpoint` object represents the current state of a
+        model endpoint. This functions supports filtering by the following parameters:
+        1) model
+        2) function
+        3) labels
+        4) top level
+        5) uids
+        By default, when no filters are applied, all available endpoints for the given project will be listed.
+
+        In addition, this functions provides a facade for listing endpoint related metrics. This facade is time-based
+        and depends on the 'start' and 'end' parameters.
+
+        :param model: The name of the model to filter by
+        :param function: The name of the function to filter by
+        :param labels: Filter model endpoints by label key-value pairs or key existence. This can be provided as:
+            - A dictionary in the format `{"label": "value"}` to match specific label key-value pairs,
+            or `{"label": None}` to check for key existence.
+            - A list of strings formatted as `"label=value"` to match specific label key-value pairs,
+            or just `"label"` for key existence.
+            - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
+            the specified key-value pairs or key existence.
+        :param start: The start time of the metrics. Can be represented by a string containing an RFC 3339 time, a
+                      Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
+                      `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
+        :param end: The end time of the metrics. Can be represented by a string containing an RFC 3339 time, a
+                      Unix timestamp in milliseconds, a relative time (`'now'` or `'now-[0-9]+[mhd]'`, where
+                      `m` = minutes, `h` = hours, `'d'` = days, and `'s'` = seconds), or 0 for the earliest time.
+        :param top_level: if true will return only routers and endpoint that are NOT children of any router
+        :param uids: if passed will return a list `ModelEndpoint` object with uid in uids
+
+        :returns: Returns a list of `ModelEndpoint` objects.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.list_model_endpoints(
+            project=self.name,
+            model=model,
+            function=function,
+            labels=labels,
+            start=start,
+            end=end,
+            top_level=top_level,
+            uids=uids,
+        )
+
     def run_function(
         self,
         function: typing.Union[str, mlrun.runtimes.BaseRuntime],
@@ -3809,6 +3941,16 @@ class MlrunProject(ModelObj):
         format_: Optional[
             mlrun.common.formatters.ArtifactFormat
         ] = mlrun.common.formatters.ArtifactFormat.full,
+        partition_by: Optional[
+            Union[mlrun.common.schemas.ArtifactPartitionByField, str]
+        ] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Optional[
+            Union[mlrun.common.schemas.SortField, str]
+        ] = mlrun.common.schemas.SortField.updated,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
     ) -> mlrun.lists.ArtifactList:
         """List artifacts filtered by various parameters.
 
@@ -3845,6 +3987,13 @@ class MlrunProject(ModelObj):
         :param tree: Return artifacts of the requested tree.
         :param limit: Maximum number of artifacts to return.
         :param format_: The format in which to return the artifacts. Default is 'full'.
+        :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
+            parameter must be provided as well.
+        :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
+            to return per group. Default value is 1.
+        :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
+            Currently the only allowed values are `created` and `updated`.
+        :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_artifacts(
@@ -3861,6 +4010,10 @@ class MlrunProject(ModelObj):
             tree=tree,
             format_=format_,
             limit=limit,
+            partition_by=partition_by,
+            rows_per_partition=rows_per_partition,
+            partition_sort_by=partition_sort_by,
+            partition_order=partition_order,
         )
 
     def paginated_list_artifacts(
@@ -4051,6 +4204,8 @@ class MlrunProject(ModelObj):
         name: Optional[str] = None,
         tag: Optional[str] = None,
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
+        kind: Optional[str] = None,
+        format_: Optional[str] = None,
     ):
         """Retrieve a list of functions, filtered by specific criteria.
 
@@ -4068,10 +4223,19 @@ class MlrunProject(ModelObj):
             or just `"label"` for key existence.
             - A comma-separated string formatted as `"label1=value1,label2"` to match entities with
             the specified key-value pairs or key existence.
+        :param kind: Return functions of the specified kind. If not provided, all function kinds will be returned.
+        :param format_: The format in which to return the functions. Default is 'full'.
         :returns: List of function objects.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
-        functions = db.list_functions(name, self.metadata.name, tag=tag, labels=labels)
+        functions = db.list_functions(
+            name,
+            project=self.metadata.name,
+            tag=tag,
+            kind=kind,
+            labels=labels,
+            format_=format_,
+        )
         if functions:
             # convert dict to function objects
             return [mlrun.new_function(runtime=func) for func in functions]
@@ -4133,9 +4297,8 @@ class MlrunProject(ModelObj):
             page_token=page_token,
             **kwargs,
         )
-        if functions:
-            # convert dict to function objects
-            return [mlrun.new_function(runtime=func) for func in functions], token
+        # convert dict to function objects
+        return [mlrun.new_function(runtime=func) for func in functions], token
 
     def list_model_monitoring_functions(
         self,
@@ -4241,9 +4404,11 @@ class MlrunProject(ModelObj):
             uid,
             self.metadata.name,
             labels=labels,
-            states=mlrun.utils.helpers.as_list(state)
-            if state is not None
-            else states or None,
+            states=(
+                mlrun.utils.helpers.as_list(state)
+                if state is not None
+                else states or None
+            ),
             sort=sort,
             last=last,
             iter=iter,
@@ -4555,7 +4720,9 @@ class MlrunProject(ModelObj):
             alert_name = alert_data.name
         db.reset_alert_config(alert_name, self.metadata.name)
 
-    def get_alert_template(self, template_name: str) -> AlertTemplate:
+    def get_alert_template(
+        self, template_name: str
+    ) -> mlrun.common.schemas.alert.AlertTemplate:
         """
         Retrieve a specific alert template.
 
@@ -4565,7 +4732,7 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.get_alert_template(template_name)
 
-    def list_alert_templates(self) -> list[AlertTemplate]:
+    def list_alert_templates(self) -> list[mlrun.common.schemas.alert.AlertTemplate]:
         """
         Retrieve list of all alert templates.
 
@@ -4574,6 +4741,109 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_alert_templates()
 
+    def list_alert_activations(
+        self,
+        name: Optional[str] = None,
+        since: Optional[datetime.datetime] = None,
+        until: Optional[datetime.datetime] = None,
+        entity: Optional[str] = None,
+        severity: Optional[
+            list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
+        ] = None,
+        entity_kind: Optional[
+            Union[mlrun.common.schemas.alert.EventEntityKind, str]
+        ] = None,
+        event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
+    ) -> list[mlrun.common.schemas.alert.AlertActivation]:
+        """
+        Retrieve a list of alert activations for a project.
+
+        :param name: The alert name to filter by. Supports exact matching or partial matching if prefixed with `~`.
+        :param since: Filters for alert activations occurring after this timestamp.
+        :param until: Filters for alert activations occurring before this timestamp.
+        :param entity: The entity ID to filter by. Supports wildcard matching if prefixed with `~`.
+        :param severity: A list of severity levels to filter by (e.g., ["high", "low"]).
+        :param entity_kind: The kind of entity (e.g., "job", "endpoint") to filter by.
+        :param event_kind: The kind of event (e.g., ""data-drift-detected"", "failed") to filter by.
+
+        :returns: A list of alert activations matching the provided filters.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.list_alert_activations(
+            project=self.metadata.name,
+            name=name,
+            since=since,
+            until=until,
+            entity=entity,
+            severity=severity,
+            entity_kind=entity_kind,
+            event_kind=event_kind,
+        )
+
+    def paginated_list_alert_activations(
+        self,
+        *args,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
+        page_token: Optional[str] = None,
+        **kwargs,
+    ) -> tuple[mlrun.common.schemas.alert.AlertActivation, Optional[str]]:
+        """
+        List alerts activations with support for pagination and various filtering options.
+
+        This method retrieves a paginated list of alert activations based on the specified filter parameters.
+        Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
+        will return a list of alert activations that match the filtering criteria provided.
+
+        For detailed information about the parameters, refer to the list_alert_activations method:
+            See :py:func:`~list_alert_activations` for more details.
+
+        Examples::
+
+            # Fetch first page of alert activations with page size of 5
+            alert_activations, token = project.paginated_list_alert_activations(page_size=5)
+            # Fetch next page using the pagination token from the previous response
+            alert_activations, token = project.paginated_list_alert_activations(
+                page_token=token
+            )
+            # Fetch alert activations for a specific page (e.g., page 3)
+            alert_activations, token = project.paginated_list_alert_activations(
+                page=3, page_size=5
+            )
+
+            # Automatically iterate over all pages without explicitly specifying the page number
+            alert_activations = []
+            token = None
+            while True:
+                page_alert_activations, token = project.paginated_list_alert_activations(
+                    page_token=token, page_size=5
+                )
+                alert_activations.extend(page_alert_activations)
+
+                # If token is None and page_alert_activations is empty, we've reached the end (no more activations).
+                # If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
+                if not token:
+                    break
+            print(f"Total alert activations retrieved: {len(alert_activations)}")
+
+        :param page: The page number to retrieve. If not provided, the next page will be retrieved.
+        :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
+        :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
+            for the first request.
+
+        :returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.paginated_list_alert_activations(
+            *args,
+            project=self.metadata.name,
+            page=page,
+            page_size=page_size,
+            page_token=page_token,
+            return_all=False,
+            **kwargs,
+        )
+
     def _run_authenticated_git_action(
         self,
         action: Callable,
@@ -4686,23 +4956,29 @@ class MlrunProject(ModelObj):
                 )
 
             if producer_dict.get("kind", "") == "run":
-                return ArtifactProducer(
-                    name=producer_dict.get("name", ""),
-                    kind=producer_dict.get("kind", ""),
-                    project=producer_project,
-                    tag=producer_tag,
-                    owner=producer_dict.get("owner", ""),
-                ), True
+                return (
+                    ArtifactProducer(
+                        name=producer_dict.get("name", ""),
+                        kind=producer_dict.get("kind", ""),
+                        project=producer_project,
+                        tag=producer_tag,
+                        owner=producer_dict.get("owner", ""),
+                    ),
+                    True,
+                )
 
         # do not retain the artifact's producer, replace it with the project as the producer
         project_producer_tag = project_producer_tag or self._get_project_tag()
-        return ArtifactProducer(
-            kind="project",
-            name=self.metadata.name,
-            project=self.metadata.name,
-            tag=project_producer_tag,
-            owner=self._resolve_artifact_owner(),
-        ), False
+        return (
+            ArtifactProducer(
+                kind="project",
+                name=self.metadata.name,
+                project=self.metadata.name,
+                tag=project_producer_tag,
+                owner=self._resolve_artifact_owner(),
+            ),
+            False,
+        )
 
     def _resolve_existing_artifact(
         self,

mlrun/run.py

@@ -73,6 +73,9 @@ from .utils import (
     update_in,
 )
 
+if typing.TYPE_CHECKING:
+    from mlrun.datastore import DataItem
+
 
 def function_to_module(code="", workdir=None, secrets=None, silent=False):
     """Load code, notebook or mlrun function as .py module
@@ -1089,7 +1092,7 @@ def get_object(url, secrets=None, size=None, offset=0, db=None):
     return stores.object(url=url).get(size, offset)
 
 
-def get_dataitem(url, secrets=None, db=None) -> mlrun.datastore.DataItem:
+def get_dataitem(url, secrets=None, db=None) -> "DataItem":
     """get mlrun dataitem object (from path/url)"""
     stores = store_manager.set(secrets, db=db)
     return stores.object(url=url)

mlrun/runtimes/base.py

@@ -25,9 +25,10 @@ from nuclio.build import mlrun_footer
 
 import mlrun.common.constants
 import mlrun.common.constants as mlrun_constants
+import mlrun.common.formatters
+import mlrun.common.runtimes
 import mlrun.common.schemas
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
-import mlrun.db
 import mlrun.errors
 import mlrun.launcher.factory
 import mlrun.utils.helpers