mlrun 1.8.0rc4__py3-none-any.whl → 1.8.0rc7__py3-none-any.whl

mlrun/__init__.py

@@ -39,6 +39,7 @@ from .execution import MLClientCtx
 from .model import RunObject, RunTemplate, new_task
 from .package import ArtifactType, DefaultPackager, Packager, handler
 from .projects import (
+    MlrunProject,
     ProjectMetadata,
     build_function,
     deploy_function,
@@ -59,6 +60,7 @@ from .run import (
     get_pipeline,
     import_function,
     new_function,
+    retry_pipeline,
     wait_for_pipeline_completion,
 )
 from .runtimes import mounts, new_model_server
@@ -162,10 +164,10 @@ def set_environment(
     return mlconf.default_project, mlconf.artifact_path
 
 
-def get_current_project(silent=False):
+def get_current_project(silent: bool = False) -> Optional[MlrunProject]:
     if not pipeline_context.project and not silent:
         raise MLRunInvalidArgumentError(
-            "current project is not initialized, use new, get or load project methods first"
+            "No current project is initialized. Use new, get or load project functions first."
         )
     return pipeline_context.project
 
@@ -182,7 +184,7 @@ def get_sample_path(subpath=""):
     return samples_path
 
 
-def set_env_from_file(env_file: str, return_dict: bool = False):
+def set_env_from_file(env_file: str, return_dict: bool = False) -> Optional[dict]:
     """Read and set and/or return environment variables from a file
     the env file should have lines in the form KEY=VALUE, comment line start with "#"
 

mlrun/alerts/alert.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from datetime import datetime
 from typing import Optional, Union
 
 import mlrun
@@ -30,6 +30,7 @@ class AlertConfig(ModelObj):
         "state",
         "count",
         "created",
+        "updated",
     ]
     _fields_to_serialize = ModelObj._fields_to_serialize + [
         "entities",
@@ -55,6 +56,7 @@ class AlertConfig(ModelObj):
         state: alert_objects.AlertActiveState = None,
         created: Optional[str] = None,
         count: Optional[int] = None,
+        updated: Optional[str] = None,
     ):
         """Alert config object
 
@@ -118,6 +120,7 @@ class AlertConfig(ModelObj):
         :param state:          State of the alert, may be active/inactive (user should not supply it)
         :param created:        When the alert is created (user should not supply it)
         :param count:          Internal counter of the alert (user should not supply it)
+        :param updated:        The last update time of the alert (user should not supply it)
         """
         self.project = project
         self.name = name
@@ -131,12 +134,39 @@ class AlertConfig(ModelObj):
         self.entities = entities
         self.id = id
         self.state = state
-        self.created = created
+        self._created = created
         self.count = count
+        self._updated = updated
 
         if template:
             self._apply_template(template)
 
+    @property
+    def created(self) -> datetime:
+        """
+        Get the `created` field as a datetime object.
+        """
+        if isinstance(self._created, str):
+            return datetime.fromisoformat(self._created)
+        return self._created
+
+    @created.setter
+    def created(self, created):
+        self._created = created
+
+    @property
+    def updated(self) -> datetime:
+        """
+        Get the `updated` field as a datetime object.
+        """
+        if isinstance(self._updated, str):
+            return datetime.fromisoformat(self._updated)
+        return self._updated
+
+    @updated.setter
+    def updated(self, updated):
+        self._updated = updated
+
     def validate_required_fields(self):
         if not self.name:
             raise mlrun.errors.MLRunInvalidArgumentError("Alert name must be provided")
@@ -253,3 +283,100 @@ class AlertConfig(ModelObj):
         self.criteria = self.criteria or template.criteria
         self.trigger = self.trigger or template.trigger
         self.reset_policy = self.reset_policy or template.reset_policy
+
+    def list_activations(
+        self,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        from_last_update: bool = False,
+    ) -> list[mlrun.common.schemas.alert.AlertActivation]:
+        """
+        Retrieve a list of all alert activations.
+
+        :param since: Filters for alert activations occurring after this timestamp.
+        :param until: Filters for alert activations occurring before this timestamp.
+        :param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
+                                 if both since and from_last_update=True are provided, from_last_update takes precedence
+                                 and the since value will be overridden by the alert's last update timestamp.
+
+        :returns: A list of alert activations matching the provided filters.
+        """
+        db = mlrun.get_run_db()
+        if from_last_update and self._updated:
+            since = self.updated
+
+        return db.list_alert_activations(
+            project=self.project,
+            name=self.name,
+            since=since,
+            until=until,
+        )
+
+    def paginated_list_activations(
+        self,
+        *args,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
+        page_token: Optional[str] = None,
+        from_last_update: bool = False,
+        **kwargs,
+    ) -> tuple[mlrun.common.schemas.alert.AlertActivation, Optional[str]]:
+        """
+        List alerts activations with support for pagination and various filtering options.
+
+        This method retrieves a paginated list of alert activations based on the specified filter parameters.
+        Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
+        will return a list of alert activations that match the filtering criteria provided.
+
+        For detailed information about the parameters, refer to the list_activations method:
+            See :py:func:`~list_activations` for more details.
+
+        Examples::
+
+            # Fetch first page of alert activations with page size of 5
+            alert_activations, token = alert_config.paginated_list_activations(page_size=5)
+            # Fetch next page using the pagination token from the previous response
+            alert_activations, token = alert_config.paginated_list_activations(
+                page_token=token
+            )
+            # Fetch alert activations for a specific page (e.g., page 3)
+            alert_activations, token = alert_config.paginated_list_activations(
+                page=3, page_size=5
+            )
+
+            # Automatically iterate over all pages without explicitly specifying the page number
+            alert_activations = []
+            token = None
+            while True:
+                page_alert_activations, token = alert_config.paginated_list_activations(
+                    page_token=token, page_size=5
+                )
+                alert_activations.extend(page_alert_activations)
+
+                # If token is None and page_alert_activations is empty, we've reached the end (no more activations).
+                # If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
+                if not token:
+                    break
+            print(f"Total alert activations retrieved: {len(alert_activations)}")
+
+        :param page: The page number to retrieve. If not provided, the next page will be retrieved.
+        :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
+        :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
+            for the first request.
+        :param from_last_update: If set to True, retrieves alert activations since the alert's last update time.
+
+        :returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
+        """
+        if from_last_update and self._updated:
+            kwargs["since"] = self.updated
+
+        db = mlrun.get_run_db()
+        return db.paginated_list_alert_activations(
+            *args,
+            project=self.project,
+            name=self.name,
+            page=page,
+            page_size=page_size,
+            page_token=page_token,
+            **kwargs,
+        )

mlrun/artifacts/__init__.py

@@ -23,7 +23,7 @@ from .base import (
     get_artifact_meta,
 )
 from .dataset import DatasetArtifact, TableArtifact, update_dataset_meta
-from .document import DocumentArtifact, DocumentLoader, DocumentLoaderSpec
+from .document import DocumentArtifact, DocumentLoaderSpec, MLRunLoader
 from .manager import (
     ArtifactManager,
     ArtifactProducer,

mlrun/artifacts/base.py

@@ -36,7 +36,16 @@ from ..utils import (
 
 
 class ArtifactMetadata(ModelObj):
-    _dict_fields = ["key", "project", "iter", "tree", "description", "hash", "tag"]
+    _dict_fields = [
+        "key",
+        "project",
+        "iter",
+        "tree",
+        "description",
+        "hash",
+        "tag",
+        "uid",
+    ]
     _extra_fields = ["updated", "labels"]
 
     def __init__(
@@ -48,6 +57,7 @@ class ArtifactMetadata(ModelObj):
         description=None,
         hash=None,
         tag=None,
+        uid=None,
     ):
         self.key = key
         self.project = project
@@ -58,6 +68,7 @@ class ArtifactMetadata(ModelObj):
         self.labels = {}
         self.updated = None
         self.tag = tag  # temp store of the tag
+        self.uid = uid
 
     def base_dict(self):
         return super().to_dict()

mlrun/artifacts/document.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import ast
 import re
 import tempfile
 from collections.abc import Iterator
@@ -74,14 +73,14 @@ class DocumentLoaderSpec(ModelObj):
         return loader
 
 
-class DocumentLoader:
+class MLRunLoader:
     """
     A factory class for creating instances of a dynamically defined document loader.
 
     Args:
         artifact_key (str): The key for the artifact to be logged.It can include '%%' which will be replaced
         by a hex-encoded version of the source path.
-        source_path (str): The source path of the document to be loaded.
+        local_path (str): The source path of the document to be loaded.
         loader_spec (DocumentLoaderSpec): Specification for the document loader.
         producer (Optional[Union[MlrunProject, str, MLClientCtx]], optional): The producer of the document
         upload (bool, optional): Flag indicating whether to upload the document.
@@ -104,7 +103,7 @@ class DocumentLoader:
         class DynamicDocumentLoader(BaseLoader):
             def __init__(
                 self,
-                source_path,
+                local_path,
                 loader_spec,
                 artifact_key,
                 producer,
@@ -112,12 +111,12 @@ class DocumentLoader:
             ):
                 self.producer = producer
                 self.artifact_key = (
-                    DocumentLoader.artifact_key_instance(artifact_key, source_path)
+                    MLRunLoader.artifact_key_instance(artifact_key, local_path)
                     if "%%" in artifact_key
                     else artifact_key
                 )
                 self.loader_spec = loader_spec
-                self.source_path = source_path
+                self.local_path = local_path
                 self.upload = upload
 
                 # Resolve the producer
@@ -129,16 +128,17 @@ class DocumentLoader:
             def lazy_load(self) -> Iterator["Document"]:  # noqa: F821
                 artifact = self.producer.log_document(
                     key=self.artifact_key,
-                    document_loader=self.loader_spec,
-                    src_path=self.source_path,
+                    document_loader_spec=self.loader_spec,
+                    local_path=self.local_path,
                     upload=self.upload,
                 )
-                yield artifact.to_langchain_documents()
+                res = artifact.to_langchain_documents()
+                yield res[0]
 
         # Return an instance of the dynamically defined subclass
         instance = DynamicDocumentLoader(
             artifact_key=artifact_key,
-            source_path=source_path,
+            local_path=source_path,
             loader_spec=loader_spec,
             producer=producer,
             upload=upload,
@@ -195,12 +195,15 @@ class DocumentArtifact(Artifact):
         def __init__(
             self,
             *args,
+            document_loader: Optional[DocumentLoaderSpec] = None,
+            collections: Optional[dict] = None,
+            original_source: Optional[str] = None,
             **kwargs,
         ):
             super().__init__(*args, **kwargs)
-            self.document_loader = None
-            self.collections = set()
-            self.original_source = None
+            self.document_loader = document_loader
+            self.collections = collections if collections is not None else {}
+            self.original_source = original_source
 
     """
     A specific artifact class inheriting from generic artifact, used to maintain Document meta-data.
@@ -216,14 +219,17 @@ class DocumentArtifact(Artifact):
 
     def __init__(
         self,
-        key=None,
-        document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
+        original_source: Optional[str] = None,
+        document_loader_spec: Optional[DocumentLoaderSpec] = None,
         **kwargs,
     ):
-        super().__init__(key, **kwargs)
-        self.spec.document_loader = document_loader.to_str()
-        if "src_path" in kwargs:
-            self.spec.original_source = kwargs["src_path"]
+        super().__init__(**kwargs)
+        self.spec.document_loader = (
+            document_loader_spec.to_dict()
+            if document_loader_spec
+            else self.spec.document_loader
+        )
+        self.spec.original_source = original_source or self.spec.original_source
 
     @property
     def spec(self) -> DocumentArtifactSpec:
@@ -234,17 +240,8 @@ class DocumentArtifact(Artifact):
         self._spec = self._verify_dict(
             spec, "spec", DocumentArtifact.DocumentArtifactSpec
         )
-        # _verify_dict doesn't handle set, so we need to convert it back
-        if isinstance(self._spec.collections, str):
-            self._spec.collections = ast.literal_eval(self._spec.collections)
-
-    @property
-    def inputs(self):
-        # To keep the interface consistent with the project.update_artifact() when we update the artifact
-        return None
 
-    @property
-    def source(self):
+    def get_source(self):
         return generate_artifact_uri(self.metadata.project, self.spec.db_key)
 
     def to_langchain_documents(
@@ -262,9 +259,8 @@ class DocumentArtifact(Artifact):
         Returns:
             list[Document]: A list of LangChain Document objects.
         """
-        dictionary = ast.literal_eval(self.spec.document_loader)
-        loader_spec = DocumentLoaderSpec.from_dict(dictionary)
 
+        loader_spec = DocumentLoaderSpec.from_dict(self.spec.document_loader)
         if self.get_target_path():
             with tempfile.NamedTemporaryFile() as tmp_file:
                 mlrun.datastore.store_manager.object(
@@ -272,8 +268,8 @@ class DocumentArtifact(Artifact):
                 ).download(tmp_file.name)
                 loader = loader_spec.make_loader(tmp_file.name)
                 documents = loader.load()
-        elif self.src_path:
-            loader = loader_spec.make_loader(self.src_path)
+        elif self.spec.original_source:
+            loader = loader_spec.make_loader(self.spec.original_source)
             documents = loader.load()
         else:
             raise ValueError(
@@ -289,8 +285,8 @@ class DocumentArtifact(Artifact):
 
             metadata = document.metadata
 
-            metadata[self.METADATA_ORIGINAL_SOURCE_KEY] = self.src_path
-            metadata[self.METADATA_SOURCE_KEY] = self.source
+            metadata[self.METADATA_ORIGINAL_SOURCE_KEY] = self.spec.original_source
+            metadata[self.METADATA_SOURCE_KEY] = self.get_source()
             metadata[self.METADATA_ARTIFACT_URI_KEY] = self.uri
             if self.get_target_path():
                 metadata[self.METADATA_ARTIFACT_TARGET_PATH_KEY] = (
@@ -301,13 +297,38 @@ class DocumentArtifact(Artifact):
                 metadata[self.METADATA_CHUNK_KEY] = str(idx)
                 doc = Document(
                     page_content=text,
-                    metadata=metadata,
+                    metadata=metadata.copy(),
                 )
                 results.append(doc)
         return results
 
     def collection_add(self, collection_id: str) -> None:
-        self.spec.collections.add(collection_id)
+        """
+        Add a collection ID to the artifact's collection list.
+
+        Adds the specified collection ID to the artifact's collection mapping if it
+        doesn't already exist.
+        This method only modifies the client-side artifact object and does not persist
+        the changes to the MLRun DB. To save the changes permanently, you must call
+        project.update_artifact() after this method.
+
+        Args:
+            collection_id (str): The ID of the collection to add
+        """
+        if collection_id not in self.spec.collections:
+            self.spec.collections[collection_id] = "1"
 
     def collection_remove(self, collection_id: str) -> None:
-        return self.spec.collections.discard(collection_id)
+        """
+        Remove a collection ID from the artifact's collection list.
+
+        Removes the specified collection ID from the artifact's local collection mapping.
+        This method only modifies the client-side artifact object and does not persist
+        the changes to the MLRun DB. To save the changes permanently, you must call
+        project.update_artifact() or context.update_artifact() after this method.
+
+        Args:
+            collection_id (str): The ID of the collection to remove
+        """
+        if collection_id in self.spec.collections:
+            self.spec.collections.pop(collection_id)

mlrun/common/constants.py

@@ -69,6 +69,7 @@ class MLRunInternalLabels:
     producer_type = f"{MLRUN_LABEL_PREFIX}producer-type"
     app_name = f"{MLRUN_LABEL_PREFIX}app-name"
     endpoint_id = f"{MLRUN_LABEL_PREFIX}endpoint-id"
+    endpoint_name = f"{MLRUN_LABEL_PREFIX}endpoint-name"
     host = "host"
     job_type = "job-type"
     kind = "kind"

mlrun/common/model_monitoring/__init__.py

@@ -11,5 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from .helpers import create_model_endpoint_uid

mlrun/common/model_monitoring/helpers.py

@@ -17,11 +17,6 @@ import typing
 
 import mlrun.common
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
-from mlrun.common.schemas.model_monitoring import (
-    EndpointUID,
-    FunctionURI,
-    VersionedModel,
-)
 
 FeatureStats = typing.NewType("FeatureStats", dict[str, dict[str, typing.Any]])
 Histogram = typing.NewType("Histogram", list[list])
@@ -31,29 +26,6 @@ BinEdges = typing.NewType("BinEdges", list[float])
 _MAX_FLOAT = sys.float_info.max
 
 
-def create_model_endpoint_uid(function_uri: str, versioned_model: str):
-    function_uri = FunctionURI.from_string(function_uri)
-    versioned_model = VersionedModel.from_string(versioned_model)
-
-    if (
-        not function_uri.project
-        or not function_uri.function
-        or not versioned_model.model
-    ):
-        raise ValueError("Both function_uri and versioned_model have to be initialized")
-
-    uid = EndpointUID(
-        function_uri.project,
-        function_uri.function,
-        function_uri.tag,
-        function_uri.hash_key,
-        versioned_model.model,
-        versioned_model.version,
-    )
-
-    return uid
-
-
 def parse_model_endpoint_project_prefix(path: str, project_name: str):
     return path.split(project_name, 1)[0] + project_name
 

mlrun/common/schemas/__init__.py

@@ -14,6 +14,7 @@
 
 from .alert import (
     AlertActivation,
+    AlertActivations,
     AlertActiveState,
     AlertConfig,
     AlertNotification,
@@ -149,10 +150,6 @@ from .model_monitoring import (
     ModelEndpointMetadata,
     ModelEndpointSpec,
     ModelEndpointStatus,
-    ModelEndpointV2,
-    ModelEndpointV2Metadata,
-    ModelEndpointV2Spec,
-    ModelEndpointV2Status,
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
@@ -165,6 +162,7 @@ from .notification import (
     NotificationSeverity,
     NotificationState,
     NotificationStatus,
+    NotificationSummary,
     SetNotificationRequest,
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus

mlrun/common/schemas/alert.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from collections import defaultdict
+from collections.abc import Iterator
 from datetime import datetime
-from typing import Annotated, Optional, Union
+from typing import Annotated, Any, Callable, Optional, Union
 
 import pydantic.v1
 
@@ -156,6 +158,7 @@ class AlertConfig(pydantic.v1.BaseModel):
     notifications: pydantic.v1.conlist(AlertNotification, min_items=1)
     state: AlertActiveState = AlertActiveState.INACTIVE
     count: Optional[int] = 0
+    updated: datetime = None
 
     def get_raw_notifications(self) -> list[notification_objects.Notification]:
         return [
@@ -203,6 +206,7 @@ class AlertTemplate(
 
 
 class AlertActivation(pydantic.v1.BaseModel):
+    id: int
     name: str
     project: str
     severity: AlertSeverity
@@ -213,3 +217,78 @@ class AlertActivation(pydantic.v1.BaseModel):
     event_kind: EventKind
     number_of_events: int
     notifications: list[notification_objects.NotificationState]
+    reset_time: Optional[datetime] = None
+
+    def group_key(self, attributes: list[str]) -> Union[Any, tuple]:
+        """
+        Dynamically create a key for grouping based on the provided attributes.
+        - If there's only one attribute, return the value directly (not a single-element tuple).
+        - If there are multiple attributes, return them as a tuple for grouping.
+
+        This ensures grouping behaves intuitively without redundant tuple representations.
+        """
+        if len(attributes) == 1:
+            # Avoid single-element tuple like (high,) when only one grouping attribute is used
+            return getattr(self, attributes[0])
+        # Otherwise, return a tuple of all specified attributes
+        return tuple(getattr(self, attr) for attr in attributes)
+
+
+class AlertActivations(pydantic.v1.BaseModel):
+    activations: list[AlertActivation]
+    pagination: Optional[dict]
+
+    def __iter__(self) -> Iterator[AlertActivation]:
+        return iter(self.activations)
+
+    def __getitem__(self, index: int) -> AlertActivation:
+        return self.activations[index]
+
+    def __len__(self) -> int:
+        return len(self.activations)
+
+    def group_by(self, *attributes: str) -> dict:
+        """
+        Group alert activations by specified attributes.
+
+        Args:
+        :param attributes: Attributes to group by.
+
+        :returns: A dictionary where keys are tuples of attribute values and values are lists of
+            AlertActivation objects.
+
+        Example:
+            # Group by project and severity
+            grouped = activations.group_by("project", "severity")
+        """
+        grouped = defaultdict(list)
+        for activation in self.activations:
+            key = activation.group_key(attributes)
+            grouped[key].append(activation)
+        return dict(grouped)
+
+    def aggregate_by(
+        self,
+        group_by_attrs: list[str],
+        aggregation_function: Callable[[list[AlertActivation]], Any],
+    ) -> dict:
+        """
+        Aggregate alert activations by specified attributes using a given aggregation function.
+
+        Args:
+        :param group_by_attrs: Attributes to group by.
+        :param aggregation_function: Function to aggregate grouped activations.
+
+        :returns: A dictionary where keys are tuples of attribute values and values are the result
+            of the aggregation function.
+
+        Example:
+            # Aggregate by name and entity_id and count number of activations in each group
+            activations.aggregate_by(["name", "entity_id"], lambda activations: len(activations))
+        """
+        grouped = self.group_by(*group_by_attrs)
+        aggregated = {
+            key: aggregation_function(activations)
+            for key, activations in grouped.items()
+        }
+        return aggregated

mlrun/common/schemas/artifact.py

@@ -25,6 +25,7 @@ from .object import ObjectStatus
 class ArtifactCategories(mlrun.common.types.StrEnum):
     model = "model"
     dataset = "dataset"
+    document = "document"
     other = "other"
 
     # we define the link as a category to prevent import cycles, but it's not a real category
@@ -38,11 +39,14 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
             return [ArtifactCategories.model.value, link_kind], False
         if self.value == ArtifactCategories.dataset.value:
             return [ArtifactCategories.dataset.value, link_kind], False
+        if self.value == ArtifactCategories.document.value:
+            return [ArtifactCategories.document.value, link_kind], False
         if self.value == ArtifactCategories.other.value:
             return (
                 [
                     ArtifactCategories.model.value,
                     ArtifactCategories.dataset.value,
+                    ArtifactCategories.document.value,
                 ],
                 True,
             )

mlrun/common/schemas/client_spec.py

@@ -57,7 +57,6 @@ class ClientSpec(pydantic.v1.BaseModel):
     redis_url: typing.Optional[str]
     redis_type: typing.Optional[str]
     sql_url: typing.Optional[str]
-    model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
     model_monitoring_tsdb_connection: typing.Optional[str]
     ce: typing.Optional[dict]
     # not passing them as one object as it possible client user would like to override only one of the params