mlrun 1.8.0rc2__py3-none-any.whl → 1.8.0rc4__py3-none-any.whl

mlrun/db/httpdb.py

@@ -858,8 +858,8 @@ class HTTPRunDB(RunDBInterface):
         :param last_update_time_from: Filter by run last update time in ``(last_update_time_from,
             last_update_time_to)``.
         :param last_update_time_to: Filter by run last update time in ``(last_update_time_from, last_update_time_to)``.
-        :param partition_by: Field to group results by. Only allowed value is `name`. When `partition_by` is specified,
-            the `partition_sort_by` parameter must be provided as well.
+        :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
+            parameter must be provided as well.
         :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
             to return per group. Default value is 1.
         :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
@@ -1137,6 +1137,16 @@ class HTTPRunDB(RunDBInterface):
             mlrun.common.formatters.ArtifactFormat
         ] = mlrun.common.formatters.ArtifactFormat.full,
         limit: Optional[int] = None,
+        partition_by: Optional[
+            Union[mlrun.common.schemas.ArtifactPartitionByField, str]
+        ] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Optional[
+            Union[mlrun.common.schemas.SortField, str]
+        ] = mlrun.common.schemas.SortField.updated,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
     ) -> ArtifactList:
         """List artifacts filtered by various parameters.
 
@@ -1178,6 +1188,13 @@ class HTTPRunDB(RunDBInterface):
             is a workflow id (artifact was created as part of a workflow).
         :param format_:         The format in which to return the artifacts. Default is 'full'.
         :param limit:           Maximum number of artifacts to return.
+        :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
+            parameter must be provided as well.
+        :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
+            to return per group. Default value is 1.
+        :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
+            Currently the only allowed values are `created` and `updated`.
+        :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
         """
 
         artifacts, _ = self._list_artifacts(
@@ -1195,6 +1212,10 @@ class HTTPRunDB(RunDBInterface):
             producer_uri=producer_uri,
             format_=format_,
             limit=limit,
+            partition_by=partition_by,
+            rows_per_partition=rows_per_partition,
+            partition_sort_by=partition_sort_by,
+            partition_order=partition_order,
             return_all=True,
         )
         return artifacts
@@ -2471,7 +2492,6 @@ class HTTPRunDB(RunDBInterface):
 
     @staticmethod
     def _generate_partition_by_params(
-        partition_by_cls,
         partition_by,
         rows_per_partition,
         sort_by,
@@ -2551,7 +2571,6 @@ class HTTPRunDB(RunDBInterface):
         if partition_by:
             params.update(
                 self._generate_partition_by_params(
-                    mlrun.common.schemas.FeatureStorePartitionByField,
                     partition_by,
                     rows_per_partition,
                     partition_sort_by,
@@ -2776,7 +2795,6 @@ class HTTPRunDB(RunDBInterface):
         if partition_by:
             params.update(
                 self._generate_partition_by_params(
-                    mlrun.common.schemas.FeatureStorePartitionByField,
                     partition_by,
                     rows_per_partition,
                     partition_sort_by,
@@ -4821,6 +4839,16 @@ class HTTPRunDB(RunDBInterface):
             mlrun.common.formatters.ArtifactFormat
         ] = mlrun.common.formatters.ArtifactFormat.full,
         limit: Optional[int] = None,
+        partition_by: Optional[
+            Union[mlrun.common.schemas.ArtifactPartitionByField, str]
+        ] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Optional[
+            Union[mlrun.common.schemas.SortField, str]
+        ] = mlrun.common.schemas.SortField.updated,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
         page: Optional[int] = None,
         page_size: Optional[int] = None,
         page_token: Optional[str] = None,
@@ -4850,6 +4878,15 @@ class HTTPRunDB(RunDBInterface):
             "page-token": page_token,
         }
 
+        if partition_by:
+            params.update(
+                self._generate_partition_by_params(
+                    partition_by,
+                    rows_per_partition,
+                    partition_sort_by,
+                    partition_order,
+                )
+            )
         error = "list artifacts"
         endpoint_path = f"projects/{project}/artifacts"
 
@@ -5012,7 +5049,6 @@ class HTTPRunDB(RunDBInterface):
         if partition_by:
             params.update(
                 self._generate_partition_by_params(
-                    mlrun.common.schemas.RunPartitionByField,
                     partition_by,
                     rows_per_partition,
                     partition_sort_by,
@@ -5072,7 +5108,11 @@ class HTTPRunDB(RunDBInterface):
         paginated_responses, token = self.process_paginated_responses(
             responses, "activations"
         )
-        return paginated_responses, token
+        paginated_results = [
+            mlrun.common.schemas.AlertActivation(**item) for item in paginated_responses
+        ]
+
+        return paginated_results, token
 
     def _wait_for_background_task_from_response(self, response):
         if response.status_code == http.HTTPStatus.ACCEPTED:

mlrun/db/nopdb.py

@@ -182,6 +182,16 @@ class NopDB(RunDBInterface):
         tree: Optional[str] = None,
         format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
         limit: Optional[int] = None,
+        partition_by: Optional[
+            Union[mlrun.common.schemas.ArtifactPartitionByField, str]
+        ] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Optional[
+            Union[mlrun.common.schemas.SortField, str]
+        ] = mlrun.common.schemas.SortField.updated,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
     ):
         return mlrun.lists.ArtifactList()
 

mlrun/execution.py

@@ -26,7 +26,13 @@ from dateutil import parser
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.formatters
-from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact
+from mlrun.artifacts import (
+    Artifact,
+    DatasetArtifact,
+    DocumentArtifact,
+    DocumentLoaderSpec,
+    ModelArtifact,
+)
 from mlrun.datastore.store_resources import get_store_resource
 from mlrun.errors import MLRunInvalidArgumentError
 
@@ -861,6 +867,46 @@ class MLClientCtx:
         self._update_run()
         return item
 
+    def log_document(
+        self,
+        key: str,
+        artifact_path: Optional[str] = None,
+        document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
+        tag: str = "",
+        upload: Optional[bool] = False,
+        labels: Optional[dict[str, str]] = None,
+        **kwargs,
+    ) -> DocumentArtifact:
+        """
+        Log a document as an artifact.
+
+        :param key: Artifact key
+        :param target_path: Path to the local file
+        :param artifact_path: Target path for artifact storage
+        :param document_loader: Spec to use to load the artifact as langchain document
+        :param tag: Version tag
+        :param upload: Whether to upload the artifact
+        :param labels: Key-value labels
+        :param kwargs: Additional keyword arguments
+        :return: DocumentArtifact object
+        """
+        doc_artifact = DocumentArtifact(
+            key=key,
+            document_loader=document_loader,
+            **kwargs,
+        )
+
+        item = self._artifacts_manager.log_artifact(
+            self,
+            doc_artifact,
+            artifact_path=extend_artifact_path(artifact_path, self.artifact_path),
+            tag=tag,
+            upload=upload,
+            labels=labels,
+        )
+        self._update_run()
+        return item
+
     def get_cached_artifact(self, key):
         """Return a logged artifact from cache (for potential updates)"""
         warnings.warn(

mlrun/platforms/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import warnings
 from pprint import pprint
 from time import sleep
 from typing import Optional
@@ -24,6 +25,49 @@ from .iguazio import (
 )
 
 
+class _DeprecationHelper:
+    """A helper class to deprecate old schemas"""
+
+    def __init__(self, new_target: str, version="1.8.0"):
+        self._new_target = new_target
+        self._version = version
+
+    def __call__(self, *args, **kwargs):
+        self._warn()
+        return self._lazy_load()(*args, **kwargs)
+
+    def __getattr__(self, attr):
+        self._warn()
+        return getattr(self._lazy_load(), attr)
+
+    def _lazy_load(self, *args, **kwargs):
+        import mlrun.runtimes.mounts as mlrun_mounts
+
+        return getattr(mlrun_mounts, self._new_target)
+
+    def _warn(self):
+        warnings.warn(
+            f"mlrun.platforms.{self._new_target} is deprecated since version {self._version}, "
+            f"and will be removed in 1.10. Use mlrun.runtimes.mounts.{self._new_target} instead.",
+            FutureWarning,
+        )
+
+
+# TODO: Remove in 1.10
+# For backwards compatibility
+VolumeMount = _DeprecationHelper("VolumeMount")
+auto_mount = _DeprecationHelper("auto_mount")
+mount_configmap = _DeprecationHelper("mount_configmap")
+mount_hostpath = _DeprecationHelper("mount_hostpath")
+mount_pvc = _DeprecationHelper("mount_pvc")
+mount_s3 = _DeprecationHelper("mount_s3")
+mount_secret = _DeprecationHelper("mount_secret")
+mount_v3io = _DeprecationHelper("mount_v3io")
+set_env_variables = _DeprecationHelper("set_env_variables")
+v3io_cred = _DeprecationHelper("v3io_cred")
+# eof 'For backwards compatibility'
+
+
 def watch_stream(
     url,
     shard_ids: Optional[list] = None,

mlrun/projects/project.py

@@ -41,6 +41,7 @@ import mlrun.artifacts.model
 import mlrun.common.formatters
 import mlrun.common.helpers
 import mlrun.common.runtimes.constants
+import mlrun.common.schemas.alert
 import mlrun.common.schemas.artifact
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.db
@@ -58,12 +59,24 @@ import mlrun.utils
 import mlrun.utils.regex
 import mlrun_pipelines.common.models
 from mlrun.alerts.alert import AlertConfig
-from mlrun.common.schemas.alert import AlertTemplate
-from mlrun.datastore.datastore_profile import DatastoreProfile, DatastoreProfile2Json
+from mlrun.datastore.datastore_profile import (
+    DatastoreProfile,
+    DatastoreProfile2Json,
+    VectorStoreProfile,
+    datastore_profile_read,
+)
+from mlrun.datastore.vectorstore import VectorStoreCollection
 from mlrun.runtimes.nuclio.function import RemoteRuntime
 from mlrun_pipelines.models import PipelineNodeWrapper
 
-from ..artifacts import Artifact, ArtifactProducer, DatasetArtifact, ModelArtifact
+from ..artifacts import (
+    Artifact,
+    ArtifactProducer,
+    DatasetArtifact,
+    DocumentArtifact,
+    DocumentLoaderSpec,
+    ModelArtifact,
+)
 from ..artifacts.manager import ArtifactManager, dict_to_artifact, extend_artifact_path
 from ..datastore import store_manager
 from ..features import Feature
@@ -1520,6 +1533,10 @@ class MlrunProject(ModelObj):
                     is_retained_producer=is_retained_producer,
                 )
 
+    def update_artifact(self, artifact_object: Artifact):
+        artifacts_manager = self._get_artifact_manager()
+        artifacts_manager.update_artifact(artifact_object, artifact_object)
+
     def _get_artifact_manager(self):
         if self._artifact_manager:
             return self._artifact_manager
@@ -1839,6 +1856,72 @@ class MlrunProject(ModelObj):
         )
         return item
 
+    def get_or_create_vector_store_collection(
+        self,
+        collection_name: str,
+        profile: Union[str, VectorStoreProfile],
+        **kwargs,
+    ) -> VectorStoreCollection:
+        """
+        Create or retrieve a VectorStoreCollection.
+
+        :param collection_name: Name of the collection
+        :param profile: Name of the VectorStoreProfile or a VectorStoreProfile object
+        :param kwargs: Additional arguments for the VectorStoreCollection
+        :return: VectorStoreCollection object
+        """
+        if isinstance(profile, str):
+            profile = datastore_profile_read(f"ds://{profile}")
+
+        if not isinstance(profile, VectorStoreProfile):
+            raise ValueError(
+                "Profile must be a VectorStoreProfile object or a profile name"
+            )
+        return VectorStoreCollection(
+            profile.vector_store_class,
+            self,
+            profile.name,
+            collection_name,
+            **profile.attributes(kwargs),
+        )
+
+    def log_document(
+        self,
+        key: str,
+        artifact_path: Optional[str] = None,
+        document_loader: DocumentLoaderSpec = DocumentLoaderSpec(),
+        tag: str = "",
+        upload: Optional[bool] = False,
+        labels: Optional[dict[str, str]] = None,
+        **kwargs,
+    ) -> DocumentArtifact:
+        """
+        Log a document as an artifact.
+
+        :param key: Artifact key
+        :param target_path: Path to the local file
+        :param artifact_path: Target path for artifact storage
+        :param document_loader: Spec to use to load the artifact as langchain document
+        :param tag: Version tag
+        :param upload: Whether to upload the artifact
+        :param labels: Key-value labels
+        :param kwargs: Additional keyword arguments
+        :return: DocumentArtifact object
+        """
+        doc_artifact = DocumentArtifact(
+            key=key,
+            document_loader=document_loader,
+            **kwargs,
+        )
+
+        return self.log_artifact(
+            doc_artifact,
+            artifact_path=artifact_path,
+            tag=tag,
+            upload=upload,
+            labels=labels,
+        )
+
     def import_artifact(
         self, item_path: str, new_key=None, artifact_path=None, tag=None
     ):
@@ -3858,6 +3941,16 @@ class MlrunProject(ModelObj):
         format_: Optional[
             mlrun.common.formatters.ArtifactFormat
         ] = mlrun.common.formatters.ArtifactFormat.full,
+        partition_by: Optional[
+            Union[mlrun.common.schemas.ArtifactPartitionByField, str]
+        ] = None,
+        rows_per_partition: int = 1,
+        partition_sort_by: Optional[
+            Union[mlrun.common.schemas.SortField, str]
+        ] = mlrun.common.schemas.SortField.updated,
+        partition_order: Union[
+            mlrun.common.schemas.OrderType, str
+        ] = mlrun.common.schemas.OrderType.desc,
     ) -> mlrun.lists.ArtifactList:
         """List artifacts filtered by various parameters.
 
@@ -3894,6 +3987,13 @@ class MlrunProject(ModelObj):
         :param tree: Return artifacts of the requested tree.
         :param limit: Maximum number of artifacts to return.
         :param format_: The format in which to return the artifacts. Default is 'full'.
+        :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
+            parameter must be provided as well.
+        :param rows_per_partition: How many top rows (per sorting defined by `partition_sort_by` and `partition_order`)
+            to return per group. Default value is 1.
+        :param partition_sort_by: What field to sort the results by, within each partition defined by `partition_by`.
+            Currently the only allowed values are `created` and `updated`.
+        :param partition_order: Order of sorting within partitions - `asc` or `desc`. Default is `desc`.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_artifacts(
@@ -3910,6 +4010,10 @@ class MlrunProject(ModelObj):
             tree=tree,
             format_=format_,
             limit=limit,
+            partition_by=partition_by,
+            rows_per_partition=rows_per_partition,
+            partition_sort_by=partition_sort_by,
+            partition_order=partition_order,
         )
 
     def paginated_list_artifacts(
@@ -4616,7 +4720,9 @@ class MlrunProject(ModelObj):
             alert_name = alert_data.name
         db.reset_alert_config(alert_name, self.metadata.name)
 
-    def get_alert_template(self, template_name: str) -> AlertTemplate:
+    def get_alert_template(
+        self, template_name: str
+    ) -> mlrun.common.schemas.alert.AlertTemplate:
         """
         Retrieve a specific alert template.
 
@@ -4626,7 +4732,7 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.get_alert_template(template_name)
 
-    def list_alert_templates(self) -> list[AlertTemplate]:
+    def list_alert_templates(self) -> list[mlrun.common.schemas.alert.AlertTemplate]:
         """
         Retrieve list of all alert templates.
 
@@ -4635,6 +4741,109 @@ class MlrunProject(ModelObj):
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.list_alert_templates()
 
+    def list_alert_activations(
+        self,
+        name: Optional[str] = None,
+        since: Optional[datetime.datetime] = None,
+        until: Optional[datetime.datetime] = None,
+        entity: Optional[str] = None,
+        severity: Optional[
+            list[Union[mlrun.common.schemas.alert.AlertSeverity, str]]
+        ] = None,
+        entity_kind: Optional[
+            Union[mlrun.common.schemas.alert.EventEntityKind, str]
+        ] = None,
+        event_kind: Optional[Union[mlrun.common.schemas.alert.EventKind, str]] = None,
+    ) -> list[mlrun.common.schemas.alert.AlertActivation]:
+        """
+        Retrieve a list of alert activations for a project.
+
+        :param name: The alert name to filter by. Supports exact matching or partial matching if prefixed with `~`.
+        :param since: Filters for alert activations occurring after this timestamp.
+        :param until: Filters for alert activations occurring before this timestamp.
+        :param entity: The entity ID to filter by. Supports wildcard matching if prefixed with `~`.
+        :param severity: A list of severity levels to filter by (e.g., ["high", "low"]).
+        :param entity_kind: The kind of entity (e.g., "job", "endpoint") to filter by.
+        :param event_kind: The kind of event (e.g., ""data-drift-detected"", "failed") to filter by.
+
+        :returns: A list of alert activations matching the provided filters.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.list_alert_activations(
+            project=self.metadata.name,
+            name=name,
+            since=since,
+            until=until,
+            entity=entity,
+            severity=severity,
+            entity_kind=entity_kind,
+            event_kind=event_kind,
+        )
+
+    def paginated_list_alert_activations(
+        self,
+        *args,
+        page: Optional[int] = None,
+        page_size: Optional[int] = None,
+        page_token: Optional[str] = None,
+        **kwargs,
+    ) -> tuple[mlrun.common.schemas.alert.AlertActivation, Optional[str]]:
+        """
+        List alerts activations with support for pagination and various filtering options.
+
+        This method retrieves a paginated list of alert activations based on the specified filter parameters.
+        Pagination is controlled using the `page`, `page_size`, and `page_token` parameters. The method
+        will return a list of alert activations that match the filtering criteria provided.
+
+        For detailed information about the parameters, refer to the list_alert_activations method:
+            See :py:func:`~list_alert_activations` for more details.
+
+        Examples::
+
+            # Fetch first page of alert activations with page size of 5
+            alert_activations, token = project.paginated_list_alert_activations(page_size=5)
+            # Fetch next page using the pagination token from the previous response
+            alert_activations, token = project.paginated_list_alert_activations(
+                page_token=token
+            )
+            # Fetch alert activations for a specific page (e.g., page 3)
+            alert_activations, token = project.paginated_list_alert_activations(
+                page=3, page_size=5
+            )
+
+            # Automatically iterate over all pages without explicitly specifying the page number
+            alert_activations = []
+            token = None
+            while True:
+                page_alert_activations, token = project.paginated_list_alert_activations(
+                    page_token=token, page_size=5
+                )
+                alert_activations.extend(page_alert_activations)
+
+                # If token is None and page_alert_activations is empty, we've reached the end (no more activations).
+                # If token is None and page_alert_activations is not empty, we've fetched the last page of activations.
+                if not token:
+                    break
+            print(f"Total alert activations retrieved: {len(alert_activations)}")
+
+        :param page: The page number to retrieve. If not provided, the next page will be retrieved.
+        :param page_size: The number of items per page to retrieve. Up to `page_size` responses are expected.
+        :param page_token: A pagination token used to retrieve the next page of results. Should not be provided
+            for the first request.
+
+        :returns: A tuple containing the list of alert activations and an optional `page_token` for pagination.
+        """
+        db = mlrun.db.get_run_db(secrets=self._secrets)
+        return db.paginated_list_alert_activations(
+            *args,
+            project=self.metadata.name,
+            page=page,
+            page_size=page_size,
+            page_token=page_token,
+            return_all=False,
+            **kwargs,
+        )
+
     def _run_authenticated_git_action(
         self,
         action: Callable,

mlrun/utils/clones.py

@@ -122,7 +122,7 @@ def add_credentials_git_remote_url(url: str, secrets=None) -> tuple[str, bool]:
         username, password = get_git_username_password_from_token(token)
 
     if username:
-        return f"https://{username}:{password}@{url_obj.hostname}{url_obj.path}", True
+        return f"https://{username}:{password}@{url_obj.netloc}{url_obj.path}", True
     return url, False
 
 

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "d1bac86097b49ad1b459421723c455622e88e5b3",
-  "version": "1.8.0-rc2"
+  "git_commit": "ae4d3819d9263a61389338d8d2c57c4ea3d65b6b",
+  "version": "1.8.0-rc4"
 }

{mlrun-1.8.0rc2.dist-info → mlrun-1.8.0rc4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.8.0rc2
+Version: 1.8.0rc4
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -50,8 +50,8 @@ Requires-Dist: setuptools>=75.2
 Requires-Dist: deprecated~=1.2
 Requires-Dist: jinja2>=3.1.3,~=3.1
 Requires-Dist: orjson<4,>=3.9.15
-Requires-Dist: mlrun-pipelines-kfp-common~=0.2.2
-Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.2.1
+Requires-Dist: mlrun-pipelines-kfp-common~=0.2.3
+Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.2.2
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
 Provides-Extra: s3
@@ -104,7 +104,7 @@ Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: api
 Requires-Dist: uvicorn~=0.27.1; extra == "api"
 Requires-Dist: dask-kubernetes~=0.11.0; extra == "api"
-Requires-Dist: apscheduler<4,>=3.10.3; extra == "api"
+Requires-Dist: apscheduler<4,>=3.11; extra == "api"
 Requires-Dist: objgraph~=3.6; extra == "api"
 Requires-Dist: igz-mgmt~=0.4.1; extra == "api"
 Requires-Dist: humanfriendly~=10.0; extra == "api"
@@ -181,7 +181,7 @@ Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
 Requires-Dist: aiosmtplib~=3.0; extra == "complete-api"
 Requires-Dist: alembic~=1.9; extra == "complete-api"
-Requires-Dist: apscheduler<4,>=3.10.3; extra == "complete-api"
+Requires-Dist: apscheduler<4,>=3.11; extra == "complete-api"
 Requires-Dist: avro~=1.11; extra == "complete-api"
 Requires-Dist: azure-core~=1.24; extra == "complete-api"
 Requires-Dist: azure-identity~=1.5; extra == "complete-api"