mlrun 1.10.0rc40__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/projects/project.py

@@ -26,11 +26,11 @@ import typing
 import uuid
 import warnings
 import zipfile
+from collections.abc import Callable
 from copy import deepcopy
 from os import environ, makedirs, path
-from typing import Callable, Optional, Union, cast
+from typing import Optional, Union, cast
 
-import deprecated
 import dotenv
 import git
 import git.exc
@@ -45,6 +45,7 @@ import mlrun.common.runtimes.constants
 import mlrun.common.schemas.alert
 import mlrun.common.schemas.artifact
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
+import mlrun.common.secrets
 import mlrun.datastore.datastore_profile
 import mlrun.db
 import mlrun.errors
@@ -166,7 +167,7 @@ def new_project(
     in the project root dir, it will be executed upon project creation or loading.
 
 
-    example::
+    Example::
 
         # create a project with local and hub functions, a workflow, and an artifact
         project = mlrun.new_project(
@@ -183,7 +184,7 @@ def new_project(
         # run the "main" workflow (watch=True to wait for run completion)
         project.run("main", watch=True)
 
-    example (load from template)::
+    Example (load from template)::
 
         # create a new project from a zip template (can also use yaml/git templates)
         # initialize a local git, and register the git remote path
@@ -197,7 +198,7 @@ def new_project(
         project.run("main", watch=True)
 
 
-    example using project_setup.py to init the project objects::
+    Example using project_setup.py to init the project objects::
 
             def setup(project):
                 project.set_function(
@@ -1281,7 +1282,7 @@ class MlrunProject(ModelObj):
     ) -> str:
         """return the project artifact uri (store://..) from the artifact key
 
-        example::
+        Example::
 
             uri = project.get_artifact_uri("my_model", category="model", tag="prod", iter=0)
 
@@ -1459,7 +1460,7 @@ class MlrunProject(ModelObj):
     ):
         """add/set an artifact in the project spec (will be registered on load)
 
-        example::
+        Example::
 
             # register a simple file artifact
             project.set_artifact("data", target_path=data_url)
@@ -1610,7 +1611,7 @@ class MlrunProject(ModelObj):
 
         If the artifact already exists with the same key and tag, it will be overwritten.
 
-        example::
+        Example::
 
             project.log_artifact(
                 "some-data",
@@ -1714,7 +1715,7 @@ class MlrunProject(ModelObj):
 
         If the dataset already exists with the same key and tag, it will be overwritten.
 
-        example::
+        Example::
 
             raw_data = {
                 "first_name": ["Jason", "Molly", "Tina", "Jake", "Amy"],
@@ -1801,7 +1802,7 @@ class MlrunProject(ModelObj):
 
         If the model already exists with the same key and tag, it will be overwritten.
 
-        example::
+        Example::
 
             project.log_model(
                 "model",
@@ -2043,11 +2044,12 @@ class MlrunProject(ModelObj):
                                 This wrapper provides both access to the original vector
                                 store's capabilities and additional MLRun functionality.
 
-        Example:
-            >>> vector_store = Chroma(embedding_function=embeddings)
-            >>> collection = project.get_vector_store_collection(
-            ...     vector_store, collection_name="my_collection"
-            ... )
+        Example::
+
+            vector_store = Chroma(embedding_function=embeddings)
+            collection = project.get_vector_store_collection(
+                vector_store, collection_name="my_collection"
+            )
         """
         return VectorStoreCollection(
             self,
@@ -2098,16 +2100,17 @@ class MlrunProject(ModelObj):
         :param kwargs: Additional keyword arguments
         :return: DocumentArtifact object
 
-        Example:
-            >>> # Log a PDF document with custom loader
-            >>> project.log_document(
-            ...     local_path="path/to/doc.pdf",
-            ...     document_loader=DocumentLoaderSpec(
-            ...         loader_class_name="langchain_community.document_loaders.PDFLoader",
-            ...         src_name="file_path",
-            ...         kwargs={"extract_images": True},
-            ...     ),
-            ... )
+        Example::
+
+            # Log a PDF document with custom loader
+            project.log_document(
+                local_path="path/to/doc.pdf",
+                document_loader=DocumentLoaderSpec(
+                    loader_class_name="langchain_community.document_loaders.PDFLoader",
+                    src_name="file_path",
+                    kwargs={"extract_images": True},
+                ),
+            )
 
         """
         if not key and not local_path and not target_path:
@@ -2764,9 +2767,9 @@ class MlrunProject(ModelObj):
 
         Supported URL prefixes:
 
-        - Object: s3://, v3io://, etc.
-        - MLRun DB: e.g db://project/func:ver
-        - Function hub/market: e.g. hub://auto-trainer:master
+            | Object (s3://, v3io://, ..)
+            | MLRun DB e.g. db://project/func:ver
+            | Hub/market: e.g. hub://auto-trainer:master
 
         Examples::
 
@@ -3015,20 +3018,6 @@ class MlrunProject(ModelObj):
 
         self.spec.set_function(name, function_object, func)
 
-    # TODO: Remove this in 1.11.0
-    @deprecated.deprecated(
-        version="1.8.0",
-        reason="'remove_function' is deprecated and will be removed in 1.11.0. "
-        "Please use `delete_function` instead.",
-        category=FutureWarning,
-    )
-    def remove_function(self, name):
-        """remove the specified function from the project
-
-        :param name:    name of the function (under the project)
-        """
-        self.spec.remove_function(name)
-
     def delete_function(self, name, delete_from_db=False):
         """deletes the specified function from the project
 
@@ -3418,7 +3407,12 @@ class MlrunProject(ModelObj):
         self._initialized = True
         return self.spec._function_objects
 
-    def with_secrets(self, kind, source, prefix=""):
+    def with_secrets(
+        self,
+        kind,
+        source,
+        prefix="",
+    ):
         """register a secrets source (file, env or dict)
 
         read secrets from a source provider to be used in workflows, example::
@@ -3440,12 +3434,19 @@ class MlrunProject(ModelObj):
 
         This will enable access to all secrets in vault registered to the current project.
 
-        :param kind:   secret type (file, inline, env, vault)
+        :param kind:   secret type (file, inline, env, vault, azure_vault)
         :param source: secret data or link (see example)
         :param prefix: add a prefix to the keys in this source
 
         :returns: project object
         """
+        # Block using mlrun-auth-secrets.* via azure_vault's k8s_secret param (client-side only)
+        if kind == "azure_vault" and isinstance(source, dict):
+            candidate_secret_name = (source.get("k8s_secret") or "").strip()
+            if candidate_secret_name:
+                mlrun.common.secrets.validate_not_forbidden_secret(
+                    candidate_secret_name
+                )
 
         if kind == "vault" and isinstance(source, list):
             source = {"project": self.metadata.name, "secrets": source}
@@ -3472,7 +3473,7 @@ class MlrunProject(ModelObj):
         when using a secrets file it should have lines in the form KEY=VALUE, comment line start with "#"
         V3IO paths/credentials and MLrun service API address are dropped from the secrets
 
-        example secrets file:
+        Example secrets file:
 
         .. code-block:: shell
 
@@ -3823,25 +3824,26 @@ class MlrunProject(ModelObj):
         Please note that you have to set the credentials before deploying any model monitoring application
         or a tracked serving function.
 
-        For example, the full flow for enabling model monitoring infrastructure with **TDEngine** and **Kafka**, is:
+        For example, the full flow for enabling model monitoring infrastructure with **TimescaleDB** and **Kafka**, is:
 
         .. code-block:: python
 
             import mlrun
             from mlrun.datastore.datastore_profile import (
                 DatastoreProfileKafkaStream,
-                DatastoreProfileTDEngine,
+                DatastoreProfilePostgreSQL,
             )
 
             project = mlrun.get_or_create_project("mm-infra-setup")
 
             # Create and register TSDB profile
-            tsdb_profile = DatastoreProfileTDEngine(
-                name="my-tdengine",
-                host="<tdengine-server-ip-address>",
-                port=6041,
-                user="username",
-                password="<tdengine-password>",
+            tsdb_profile = DatastoreProfilePostgreSQL(
+                name="my-timescaledb",
+                host="<timescaledb-server-ip-address>",
+                port=5432,
+                user="postgres",
+                password="<timescaledb-password>",
+                database="mlrun",
             )
             project.register_datastore_profile(tsdb_profile)
 
@@ -3912,7 +3914,7 @@ class MlrunProject(ModelObj):
                                           monitoring. The supported profiles are:
 
                                           * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileV3io`
-                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfileTDEngine`
+                                          * :py:class:`~mlrun.datastore.datastore_profile.DatastoreProfilePostgreSQL`
 
                                           You need to register one of them, and pass the profile's name.
         :param stream_profile_name:       The datastore profile name of the stream to be used in model monitoring.
@@ -4058,7 +4060,7 @@ class MlrunProject(ModelObj):
     ) -> typing.Union[mlrun.model.RunObject, PipelineNodeWrapper]:
         """Run a local or remote task as part of a local/kubeflow pipeline
 
-        example (use with project)::
+        Example (use with project)::
 
             # create a project with two functions (local and from hub)
             project = mlrun.new_project(project_name, "./proj")
@@ -4462,7 +4464,6 @@ class MlrunProject(ModelObj):
         kind: Optional[str] = None,
         category: typing.Union[str, mlrun.common.schemas.ArtifactCategories] = None,
         tree: Optional[str] = None,
-        limit: Optional[int] = None,
         format_: Optional[
             mlrun.common.formatters.ArtifactFormat
         ] = mlrun.common.formatters.ArtifactFormat.full,
@@ -4512,7 +4513,6 @@ class MlrunProject(ModelObj):
         :param kind: Return artifacts of the requested kind.
         :param category: Return artifacts of the requested category.
         :param tree: Return artifacts of the requested tree.
-        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
         :param format_: The format in which to return the artifacts. Default is 'full'.
         :param partition_by: Field to group results by. When `partition_by` is specified, the `partition_sort_by`
             parameter must be provided as well.
@@ -4524,13 +4524,6 @@ class MlrunProject(ModelObj):
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
 
-        if limit:
-            # TODO: Remove this in 1.11.0
-            warnings.warn(
-                "'limit' is deprecated and will be removed in 1.11.0. Use 'page' and 'page_size' instead.",
-                FutureWarning,
-            )
-
         return db.list_artifacts(
             name,
             self.metadata.name,
@@ -4544,7 +4537,6 @@ class MlrunProject(ModelObj):
             category=category,
             tree=tree,
             format_=format_,
-            limit=limit,
             partition_by=partition_by,
             rows_per_partition=rows_per_partition,
             partition_sort_by=partition_sort_by,
@@ -4625,7 +4617,6 @@ class MlrunProject(ModelObj):
         iter: Optional[int] = None,
         best_iteration: bool = False,
         tree: Optional[str] = None,
-        limit: Optional[int] = None,
         format_: Optional[
             mlrun.common.formatters.ArtifactFormat
         ] = mlrun.common.formatters.ArtifactFormat.full,
@@ -4659,7 +4650,6 @@ class MlrunProject(ModelObj):
             artifacts generated from a hyper-param run. If only a single iteration exists, will return the artifact
             from that iteration. If using ``best_iter``, the ``iter`` parameter must not be used.
         :param tree: Return artifacts of the requested tree.
-        :param limit: Deprecated - Maximum number of artifacts to return (will be removed in 1.11.0).
         :param format_: The format in which to return the artifacts. Default is 'full'.
         """
         db = mlrun.db.get_run_db(secrets=self._secrets)
@@ -4674,7 +4664,6 @@ class MlrunProject(ModelObj):
             best_iteration=best_iteration,
             kind=mlrun.artifacts.model.ModelArtifact.kind,
             tree=tree,
-            limit=limit,
             format_=format_,
         ).to_objects()
 
@@ -4897,7 +4886,7 @@ class MlrunProject(ModelObj):
     ):
         """Retrieve a list of functions, filtered by specific criteria.
 
-        example::
+        Example::
 
             functions = project.list_functions(tag="latest")
 
@@ -5035,21 +5024,27 @@ class MlrunProject(ModelObj):
         include_stats: bool = False,
         include_infra: bool = True,
     ) -> list[mlrun.common.schemas.model_monitoring.FunctionSummary]:
-        """Get monitoring function summaries for the specified project.
-        :param start: Start time for filtering the results (optional).
-        :param end: End time for filtering the results (optional).
+        """
+        Get monitoring function summaries for the specified project.
+
+        :param start: The start time of the monitoring applications’ statistics.
+            If not defined, the default is 24 hours ago.
+            Required timezone, applicable only when `include_stats` is set to True.
+        :param end: The end time of the monitoring applications’ statistics.
+            If not defined, the default is now.
+            Required timezone, applicable only when `include_stats` is set to True.
         :param names: List of function names to filter by (optional).
         :param labels: Labels to filter by (optional).
         :param include_stats: Whether to include statistics in the response (default is False).
-        :param include_infra: whether to include model monitoring infrastructure functions (default is True).
+        :param include_infra: Whether to include model monitoring infrastructure functions (default is True).
+
         :return: A list of FunctionSummary objects containing information about the monitoring functions.
         """
 
-        if start is not None and end is not None:
-            if start.tzinfo is None or end.tzinfo is None:
-                raise mlrun.errors.MLRunInvalidArgumentTypeError(
-                    "Custom start and end times must contain the timezone."
-                )
+        if (start and start.tzinfo is None) or (end and end.tzinfo is None):
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Custom start and end times must contain the timezone."
+            )
 
         db = mlrun.db.get_run_db(secrets=self._secrets)
         return db.get_monitoring_function_summaries(
@@ -5069,10 +5064,14 @@ class MlrunProject(ModelObj):
         end: Optional[datetime.datetime] = None,
         include_latest_metrics: bool = False,
     ) -> mlrun.common.schemas.model_monitoring.FunctionSummary:
-        """Get a monitoring function summary for the specified project and function name.
-        :param name:                   Name of the monitoring function to retrieve the summary for.
-        :param start:                  Start time for filtering the results (optional).
-        :param end:                    End time for filtering the results (optional).
+        """
+        Get a monitoring function summary for the specified project and function name.
+
+        :param name: Name of the monitoring function to retrieve the summary for.
+        :param start: The start time of the monitoring applications’ statistics.
+            If not defined, the default is 24 hours ago. Required timezone.
+        :param end: The end time of the monitoring applications’ statistics.
+            If not defined, the default is now. Required timezone.
         :param include_latest_metrics: Whether to include the latest metrics in the response (default is False).
 
         :return: A FunctionSummary object containing information about the monitoring function.

mlrun/run.py

@@ -290,7 +290,7 @@ def get_or_create_ctx(
     elif with_env and config:
         newspec = config
 
-    if isinstance(newspec, (RunObject, RunTemplate)):
+    if isinstance(newspec, RunObject | RunTemplate):
         newspec = newspec.to_dict()
 
     if newspec and not isinstance(newspec, dict):
@@ -345,7 +345,7 @@ def get_or_create_ctx(
 def import_function(url="", secrets=None, db="", project=None, new_name=None):
     """Create function object from DB or local/remote YAML file
 
-    Functions can be imported from function repositories (mlrun Function Hub (formerly Marketplace) or local db),
+    Functions can be imported from function repositories (MLRun Hub) or local db),
     or be read from a remote URL (http(s), s3, git, v3io, ..) containing the function YAML
 
     special URLs::
@@ -361,7 +361,7 @@ def import_function(url="", secrets=None, db="", project=None, new_name=None):
             "https://raw.githubusercontent.com/org/repo/func.yaml"
         )
 
-    :param url: path/url to Function Hub, db or function YAML file
+    :param url: path/url to MLRun Hub, db or function YAML file
     :param secrets: optional, credentials dict for DB or URL (s3, v3io, ...)
     :param db: optional, mlrun api/db path
     :param project: optional, target project for the function
@@ -692,7 +692,7 @@ def code_to_function(
     :param description:  short function description, defaults to ''
     :param requirements: a list of python packages
     :param requirements_file: path to a python requirements file
-    :param categories:   list of categories for mlrun Function Hub, defaults to None
+    :param categories:   list of categories for MLRun Hub, defaults to None
     :param labels:       name/value pairs dict to tag the function with useful metadata, defaults to None
     :param with_doc:     indicates whether to document the function parameters, defaults to True
     :param ignored_tags: notebook cells to ignore when converting notebooks to py code (separated by ';')
@@ -735,7 +735,6 @@ def code_to_function(
         )
 
     """
-    filebase, _ = path.splitext(path.basename(filename))
     ignored_tags = ignored_tags or mlconf.ignored_notebook_tags
 
     def add_name(origin, name=""):
@@ -801,8 +800,6 @@ def code_to_function(
         ignored_tags=ignored_tags,
     )
 
-    mlrun.utils.helpers.validate_function_name(name)
-
     spec["spec"]["env"].append(
         {
             "name": "MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK",

mlrun/runtimes/__init__.py

@@ -26,6 +26,7 @@ __all__ = [
     "KubeResource",
     "ApplicationRuntime",
     "MpiRuntimeV1",
+    "RuntimeKinds",
 ]
 
 import typing
@@ -34,6 +35,7 @@ from mlrun.runtimes.utils import resolve_spark_operator_version
 
 from ..common.runtimes.constants import MPIJobCRDVersions
 from .base import BaseRuntime, RunError, RuntimeClassMode  # noqa
+from .constants import RuntimeKinds
 from .daskjob import DaskCluster  # noqa
 from .databricks_job.databricks_runtime import DatabricksRuntime
 from .kubejob import KubejobRuntime, KubeResource  # noqa
@@ -94,210 +96,6 @@ def new_model_server(
         )
 
 
-class RuntimeKinds:
-    remote = "remote"
-    nuclio = "nuclio"
-    dask = "dask"
-    job = "job"
-    spark = "spark"
-    remotespark = "remote-spark"
-    mpijob = "mpijob"
-    serving = "serving"
-    local = "local"
-    handler = "handler"
-    databricks = "databricks"
-    application = "application"
-
-    @staticmethod
-    def all():
-        return [
-            RuntimeKinds.remote,
-            RuntimeKinds.nuclio,
-            RuntimeKinds.serving,
-            RuntimeKinds.dask,
-            RuntimeKinds.job,
-            RuntimeKinds.spark,
-            RuntimeKinds.remotespark,
-            RuntimeKinds.mpijob,
-            RuntimeKinds.local,
-            RuntimeKinds.databricks,
-            RuntimeKinds.application,
-        ]
-
-    @staticmethod
-    def runtime_with_handlers():
-        return [
-            RuntimeKinds.dask,
-            RuntimeKinds.job,
-            RuntimeKinds.spark,
-            RuntimeKinds.remotespark,
-            RuntimeKinds.mpijob,
-            RuntimeKinds.databricks,
-        ]
-
-    @staticmethod
-    def abortable_runtimes():
-        return [
-            RuntimeKinds.job,
-            RuntimeKinds.spark,
-            RuntimeKinds.remotespark,
-            RuntimeKinds.mpijob,
-            RuntimeKinds.databricks,
-            RuntimeKinds.local,
-            RuntimeKinds.handler,
-            "",
-        ]
-
-    @staticmethod
-    def retriable_runtimes():
-        return [
-            RuntimeKinds.job,
-        ]
-
-    @staticmethod
-    def nuclio_runtimes():
-        return [
-            RuntimeKinds.remote,
-            RuntimeKinds.nuclio,
-            RuntimeKinds.serving,
-            RuntimeKinds.application,
-        ]
-
-    @staticmethod
-    def pure_nuclio_deployed_runtimes():
-        return [
-            RuntimeKinds.remote,
-            RuntimeKinds.nuclio,
-            RuntimeKinds.serving,
-        ]
-
-    @staticmethod
-    def handlerless_runtimes():
-        return [
-            RuntimeKinds.serving,
-            # Application runtime handler is internal reverse proxy
-            RuntimeKinds.application,
-        ]
-
-    @staticmethod
-    def local_runtimes():
-        return [
-            RuntimeKinds.local,
-            RuntimeKinds.handler,
-        ]
-
-    @staticmethod
-    def is_log_collectable_runtime(kind: typing.Optional[str]):
-        """
-        whether log collector can collect logs for that runtime
-        :param kind: kind name
-        :return: whether log collector can collect logs for that runtime
-        """
-        # if local run, the log collector doesn't support it as it is only supports k8s resources
-        # when runtime is local the client is responsible for logging the stdout of the run by using `log_std`
-        if RuntimeKinds.is_local_runtime(kind):
-            return False
-
-        if (
-            kind
-            not in [
-                # dask implementation is different from other runtimes, because few runs can be run against the same
-                # runtime resource, so collecting logs on that runtime resource won't be correct, the way we collect
-                # logs for dask is by using `log_std` on client side after we execute the code against the cluster,
-                # as submitting the run with the dask client will return the run stdout.
-                # For more information head to `DaskCluster._run`.
-                RuntimeKinds.dask
-            ]
-            + RuntimeKinds.nuclio_runtimes()
-        ):
-            return True
-
-        return False
-
-    @staticmethod
-    def is_local_runtime(kind):
-        # "" or None counted as local
-        if not kind or kind in RuntimeKinds.local_runtimes():
-            return True
-        return False
-
-    @staticmethod
-    def requires_k8s_name_validation(kind: str) -> bool:
-        """
-        Returns True if the runtime kind creates Kubernetes resources that use the function name.
-
-        Function names for k8s-deployed runtimes must conform to DNS-1123 label requirements:
-        - Lowercase alphanumeric characters or '-'
-        - Start and end with an alphanumeric character
-        - Maximum 63 characters
-
-        Local runtimes (local, handler) run on the local machine and don't create k8s resources,
-        so they don't require k8s naming validation.
-
-        :param kind: Runtime kind string (job, spark, serving, local, etc.)
-        :return: True if function name needs k8s DNS-1123 validation, False otherwise
-        """
-        return not RuntimeKinds.is_local_runtime(kind)
-
-    @staticmethod
-    def requires_absolute_artifacts_path(kind):
-        """
-        Returns True if the runtime kind requires absolute artifacts' path (i.e. is local), False otherwise.
-        """
-        if RuntimeKinds.is_local_runtime(kind):
-            return False
-
-        if kind not in [
-            # logging artifacts is done externally to the dask cluster by a client that can either run locally (in which
-            # case the path can be relative) or remotely (in which case the path must be absolute and will be passed
-            # to another run)
-            RuntimeKinds.dask
-        ]:
-            return True
-        return False
-
-    @staticmethod
-    def requires_image_name_for_execution(kind):
-        if RuntimeKinds.is_local_runtime(kind):
-            return False
-
-        # both spark and remote spark uses different mechanism for assigning images
-        return kind not in [RuntimeKinds.spark, RuntimeKinds.remotespark]
-
-    @staticmethod
-    def supports_from_notebook(kind):
-        return kind not in [RuntimeKinds.application]
-
-    @staticmethod
-    def resolve_nuclio_runtime(kind: str, sub_kind: str):
-        kind = kind.split(":")[0]
-        if kind not in RuntimeKinds.nuclio_runtimes():
-            raise ValueError(
-                f"Kind {kind} is not a nuclio runtime, available runtimes are {RuntimeKinds.nuclio_runtimes()}"
-            )
-
-        if sub_kind == serving_subkind:
-            return ServingRuntime()
-
-        if kind == RuntimeKinds.application:
-            return ApplicationRuntime()
-
-        runtime = RemoteRuntime()
-        runtime.spec.function_kind = sub_kind
-        return runtime
-
-    @staticmethod
-    def resolve_nuclio_sub_kind(kind):
-        is_nuclio = kind.startswith("nuclio")
-        sub_kind = kind[kind.find(":") + 1 :] if is_nuclio and ":" in kind else None
-        if kind == RuntimeKinds.serving:
-            is_nuclio = True
-            sub_kind = serving_subkind
-        elif kind == RuntimeKinds.application:
-            is_nuclio = True
-        return is_nuclio, sub_kind
-
-
 def get_runtime_class(kind: str):
     if kind == RuntimeKinds.mpijob:
         return MpiRuntimeV1