mlrun 1.5.0rc1__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/api/utils/scheduler.py

@@ -142,14 +142,14 @@ class Scheduler:
         self._enrich_schedule_notifications(project, name, scheduled_object)
 
         get_db().create_schedule(
-            db_session,
-            project,
-            name,
-            kind,
-            scheduled_object,
-            cron_trigger,
-            concurrency_limit,
-            labels,
+            session=db_session,
+            project=project,
+            name=name,
+            kind=kind,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            concurrency_limit=concurrency_limit,
+            labels=labels,
         )
         job = self._create_schedule_in_scheduler(
             project,
@@ -217,13 +217,13 @@ class Scheduler:
         self._enrich_schedule_notifications(project, name, scheduled_object)
 
         get_db().update_schedule(
-            db_session,
-            project,
-            name,
-            scheduled_object,
-            cron_trigger,
-            labels,
-            concurrency_limit,
+            session=db_session,
+            project=project,
+            name=name,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
         )
         db_schedule = get_db().get_schedule(db_session, project, name)
 
@@ -303,6 +303,89 @@ class Scheduler:
             )
         get_db().delete_schedules(db_session, project)
 
+    @mlrun.api.utils.helpers.ensure_running_on_chief
+    def store_schedule(
+        self,
+        db_session: Session,
+        auth_info: mlrun.common.schemas.AuthInfo,
+        project: str,
+        name: str,
+        scheduled_object: Union[Dict, Callable] = None,
+        cron_trigger: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
+        labels: Dict = None,
+        concurrency_limit: int = None,
+        kind: mlrun.common.schemas.ScheduleKinds = None,
+    ):
+        if isinstance(cron_trigger, str):
+            cron_trigger = mlrun.common.schemas.ScheduleCronTrigger.from_crontab(
+                cron_trigger
+            )
+
+        if cron_trigger is not None:
+            self._validate_cron_trigger(cron_trigger)
+
+        logger.debug(
+            "Storing schedule",
+            project=project,
+            name=name,
+            kind=kind,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
+        )
+
+        if not kind:
+            # TODO: Need to think of a way to not use `get_schedule`
+            #  in this function or in `get_db().store_function()` in this flow
+            #  because we must have kind to ensure that auth info has access key.
+            db_schedule = get_db().get_schedule(
+                db_session, project, name, raise_on_not_found=False
+            )
+            kind = db_schedule.kind
+
+        self._ensure_auth_info_has_access_key(auth_info, kind)
+        secret_name = self._store_schedule_secrets_using_auth_secret(auth_info)
+        labels = self._append_access_key_secret_to_labels(labels, secret_name)
+        self._enrich_schedule_notifications(project, name, scheduled_object)
+
+        db_schedule = get_db().store_schedule(
+            session=db_session,
+            project=project,
+            name=name,
+            kind=kind,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
+        )
+        if db_schedule:
+            updated_schedule = self._transform_and_enrich_db_schedule(
+                db_session, db_schedule
+            )
+
+            job = self._update_schedule_in_scheduler(
+                project,
+                name,
+                updated_schedule.kind,
+                updated_schedule.scheduled_object,
+                updated_schedule.cron_trigger,
+                updated_schedule.concurrency_limit,
+                auth_info,
+            )
+        else:
+            job = self._create_schedule_in_scheduler(
+                project,
+                name,
+                kind,
+                scheduled_object,
+                cron_trigger,
+                concurrency_limit,
+                auth_info,
+            )
+
+        self.update_schedule_next_run_time(db_session, name, project, job)
+
     def _remove_schedule_scheduler_resources(self, db_session: Session, project, name):
         self._remove_schedule_from_scheduler(project, name)
         # This is kept for backwards compatibility - if schedule was using the "old" format of storing secrets, then

mlrun/api/utils/singletons/db.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import mlrun.db
 from mlrun.api.db.base import DBInterface
 from mlrun.api.db.sqldb.db import SQLDB
 from mlrun.common.db.sql_session import create_session
@@ -34,6 +35,9 @@ def initialize_db(override_db=None):
         return
     logger.info("Creating sql db")
     db = SQLDB(config.httpdb.dsn)
+    # set the run db path to the sql db dsn
+    mlrun.db.get_or_set_dburl(config.httpdb.dsn)
+
     db_session = None
     try:
         db_session = create_session()

mlrun/artifacts/manager.py

@@ -17,7 +17,6 @@ from os.path import isdir
 
 import mlrun.config
 
-from ..db import RunDBInterface
 from ..utils import (
     is_legacy_artifact,
     is_relative_path,
@@ -111,7 +110,7 @@ def dict_to_artifact(struct: dict) -> Artifact:
 class ArtifactManager:
     def __init__(
         self,
-        db: RunDBInterface = None,
+        db: "mlrun.db.RunDBInterface" = None,
         calc_hash=True,
     ):
         self.calc_hash = calc_hash

mlrun/common/schemas/__init__.py

@@ -171,3 +171,9 @@ from .secret import (
     UserSecretCreationRequest,
 )
 from .tag import Tag, TagObjects
+from .workflow import (
+    GetWorkflowResponse,
+    WorkflowRequest,
+    WorkflowResponse,
+    WorkflowSpec,
+)

mlrun/common/schemas/auth.py

@@ -57,6 +57,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
     model_endpoint = "model-endpoint"
     pipeline = "pipeline"
     hub_source = "hub-source"
+    workflow = "workflow"
 
     def to_resource_string(
         self,
@@ -87,8 +88,10 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             AuthorizationResourceTypes.pipeline: "/projects/{project_name}/pipelines/{resource_name}",
             # Hub sources are not project-scoped, and auth is globally on the sources endpoint.
             # TODO - this was reverted to /marketplace since MLRun needs to be able to run with old igz versions. Once
-            # we only have support for igz versions that support /hub (>=3.5.4), change this to "/hub/sources".
+            #  we only have support for igz versions that support /hub (>=3.5.4), change this to "/hub/sources".
             AuthorizationResourceTypes.hub_source: "/marketplace/sources",
+            # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
+            AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
         }[self].format(project_name=project_name, resource_name=resource_name)
 
 

mlrun/common/schemas/client_spec.py

@@ -57,7 +57,7 @@ class ClientSpec(pydantic.BaseModel):
     redis_url: typing.Optional[str]
     redis_type: typing.Optional[str]
     sql_url: typing.Optional[str]
-
+    model_endpoint_monitoring_store_type: typing.Optional[str]
     # ce_mode is deprecated, we will use the full ce config instead and ce_mode will be removed in 1.6.0
     ce_mode: typing.Optional[str]
     ce: typing.Optional[dict]

mlrun/common/schemas/model_monitoring/__init__.py

@@ -25,6 +25,7 @@ from .constants import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     ProjectSecretKeys,
+    PrometheusMetric,
     TimeSeriesTarget,
     VersionedModel,
 )

mlrun/common/schemas/model_monitoring/constants.py

@@ -49,6 +49,7 @@ class EventFieldType:
     ENTITIES = "entities"
     FIRST_REQUEST = "first_request"
     LAST_REQUEST = "last_request"
+    METRIC = "metric"
     METRICS = "metrics"
     TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
     BATCH_INTERVALS_DICT = "batch_intervals_dict"
@@ -71,6 +72,7 @@ class EventFieldType:
     MONITOR_CONFIGURATION = "monitor_configuration"
     FEATURE_SET_URI = "monitoring_feature_set_uri"
     ALGORITHM = "algorithm"
+    VALUE = "value"
 
 
 class EventLiveStats:
@@ -129,6 +131,15 @@ class EndpointType(enum.IntEnum):
     LEAF_EP = 3  # end point that is a child of a router
 
 
+class PrometheusMetric:
+    PREDICTIONS_TOTAL = "predictions_total"
+    MODEL_LATENCY_SECONDS = "model_latency_seconds"
+    INCOME_FEATURES = "income_features"
+    ERRORS_TOTAL = "errors_total"
+    DRIFT_METRICS = "drift_metrics"
+    DRIFT_STATUS = "drift_status"
+
+
 @dataclass
 class FunctionURI:
     project: str

mlrun/common/schemas/project.py

@@ -84,6 +84,7 @@ class ProjectSpec(pydantic.BaseModel):
     origin_url: typing.Optional[str] = None
     desired_state: typing.Optional[ProjectDesiredState] = ProjectDesiredState.online
     custom_packagers: typing.Optional[typing.List[typing.Tuple[str, bool]]] = None
+    default_image: typing.Optional[str] = None
 
     class Config:
         extra = pydantic.Extra.allow

mlrun/common/schemas/runs.py

@@ -14,17 +14,10 @@
 
 import typing
 
-# TODO: When we remove support for python 3.7, we can use Literal from the typing package.
-#       Remove the following try/except block with import from typing_extensions.
-try:
-    from typing import Literal
-except ImportError:
-    from typing_extensions import Literal
-
 import pydantic
 
 
 class RunIdentifier(pydantic.BaseModel):
-    kind: Literal["run"] = "run"
+    kind: typing.Literal["run"] = "run"
     uid: typing.Optional[str]
     iter: typing.Optional[int]

mlrun/common/schemas/schedule.py

@@ -13,14 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime
-from typing import Any, List, Optional, Union
-
-# TODO: When we remove support for python 3.7, we can use Literal from the typing package.
-#       Remove the following try/except block with import from typing_extensions.
-try:
-    from typing import Literal
-except ImportError:
-    from typing_extensions import Literal
+from typing import Any, List, Literal, Optional, Union
 
 from pydantic import BaseModel
 

mlrun/common/schemas/workflow.py

@@ -0,0 +1,54 @@
+# Copyright 2018 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+
+import pydantic
+
+from .schedule import ScheduleCronTrigger
+
+
+class WorkflowSpec(pydantic.BaseModel):
+    name: str
+    engine: typing.Optional[str] = None
+    code: typing.Optional[str] = None
+    path: typing.Optional[str] = None
+    args: typing.Optional[dict] = None
+    handler: typing.Optional[str] = None
+    ttl: typing.Optional[int] = None
+    args_schema: typing.Optional[list] = None
+    schedule: typing.Union[str, ScheduleCronTrigger] = None
+    run_local: typing.Optional[bool] = None
+    image: typing.Optional[str] = None
+
+
+class WorkflowRequest(pydantic.BaseModel):
+    spec: typing.Optional[WorkflowSpec] = None
+    arguments: typing.Optional[typing.Dict] = None
+    artifact_path: typing.Optional[str] = None
+    source: typing.Optional[str] = None
+    run_name: typing.Optional[str] = None
+    namespace: typing.Optional[str] = None
+
+
+class WorkflowResponse(pydantic.BaseModel):
+    project: str = None
+    name: str = None
+    status: str = None
+    run_id: typing.Optional[str] = None
+    schedule: typing.Union[str, ScheduleCronTrigger] = None
+
+
+class GetWorkflowResponse(pydantic.BaseModel):
+    workflow_id: str = None

mlrun/config.py

@@ -27,8 +27,8 @@ import copy
 import json
 import os
 import typing
-import urllib.parse
 from collections.abc import Mapping
+from datetime import timedelta
 from distutils.util import strtobool
 from os.path import expanduser
 from threading import Lock
@@ -149,7 +149,7 @@ default_config = {
         "timeout_mode": "enabled",
         # timeout in seconds to wait for background task to be updated / finished by the worker responsible for the task
         "default_timeouts": {
-            "operations": {"migrations": "3600"},
+            "operations": {"migrations": "3600", "load_project": "60"},
             "runtimes": {"dask": "600"},
         },
     },
@@ -417,6 +417,7 @@ default_config = {
         "default_http_sink": "http://nuclio-{project}-model-monitoring-stream.mlrun.svc.cluster.local:8080",
         "batch_processing_function_branch": "master",
         "parquet_batching_max_events": 10000,
+        "parquet_batching_timeout_secs": timedelta(minutes=30).total_seconds(),
         # See mlrun.model_monitoring.stores.ModelEndpointStoreType for available options
         "store_type": "v3io-nosql",
         "endpoint_store_connection": "",
@@ -516,7 +517,11 @@ default_config = {
     "debug": {
         "expose_internal_api_endpoints": False,
     },
-    "default_workflow_runner_name": "workflow-runner-{}",
+    "workflows": {
+        "default_workflow_runner_name": "workflow-runner-{}",
+        # Default timeout seconds for retrieving workflow id after execution:
+        "timeouts": {"local": 120, "kfp": 30},
+    },
     "log_collector": {
         "address": "localhost:8282",
         # log collection mode can be one of: "sidecar", "legacy", "best-effort"
@@ -776,7 +781,6 @@ class Config:
             return semver.VersionInfo.parse(f"{semver_compatible_igz_version}.0")
 
     def verify_security_context_enrichment_mode_is_allowed(self):
-
         # TODO: move SecurityContextEnrichmentModes to a different package so that we could use it here without
         #  importing mlrun.api
         if config.function.spec.security_context.enrichment_mode == "disabled":
@@ -933,36 +937,6 @@ class Config:
             # when dbpath is set we want to connect to it which will sync configuration from it to the client
             mlrun.db.get_run_db(value, force_reconnect=True)
 
-    @property
-    def iguazio_api_url(self):
-        """
-        we want to be able to run with old versions of the service who runs the API (which doesn't configure this
-        value) so we're doing best effort to try and resolve it from other configurations
-        TODO: Remove this hack when 0.6.x is old enough
-        """
-        if not self._iguazio_api_url:
-            if self.httpdb.builder.docker_registry and self.igz_version:
-                return self._extract_iguazio_api_from_docker_registry_url()
-        return self._iguazio_api_url
-
-    def _extract_iguazio_api_from_docker_registry_url(self):
-        docker_registry_url = self.httpdb.builder.docker_registry
-        # add schema otherwise parsing go wrong
-        if "://" not in docker_registry_url:
-            docker_registry_url = f"http://{docker_registry_url}"
-        parsed_registry_url = urllib.parse.urlparse(docker_registry_url)
-        registry_hostname = parsed_registry_url.hostname
-        # replace the first domain section (app service name) with dashboard
-        first_dot_index = registry_hostname.find(".")
-        if first_dot_index < 0:
-            # if not found it's not the format we know - can't resolve the api url from the registry url
-            return ""
-        return f"https://dashboard{registry_hostname[first_dot_index:]}"
-
-    @iguazio_api_url.setter
-    def iguazio_api_url(self, value):
-        self._iguazio_api_url = value
-
     def is_api_running_on_k8s(self):
         # determine if the API service is attached to K8s cluster
         # when there is a cluster the .namespace is set
@@ -1045,6 +1019,40 @@ class Config:
             ver in mlrun.mlconf.ce.mode for ver in ["lite", "full"]
         )
 
+    def get_s3_storage_options(self) -> typing.Dict[str, typing.Any]:
+        """
+        Generate storage options dictionary as required for handling S3 path in fsspec. The model monitoring stream
+        graph uses this method for generating the storage options for S3 parquet target path.
+        :return: A storage options dictionary in which each key-value pair  represents a particular configuration,
+        such as endpoint_url or aws access key.
+        """
+        key = mlrun.get_secret_or_env("AWS_ACCESS_KEY_ID")
+        secret = mlrun.get_secret_or_env("AWS_SECRET_ACCESS_KEY")
+
+        force_non_anonymous = mlrun.get_secret_or_env("S3_NON_ANONYMOUS")
+        profile = mlrun.get_secret_or_env("AWS_PROFILE")
+
+        storage_options = dict(
+            anon=not (force_non_anonymous or (key and secret)),
+            key=key,
+            secret=secret,
+        )
+
+        endpoint_url = mlrun.get_secret_or_env("S3_ENDPOINT_URL")
+        if endpoint_url:
+            client_kwargs = {"endpoint_url": endpoint_url}
+            storage_options["client_kwargs"] = client_kwargs
+
+        if profile:
+            storage_options["profile"] = profile
+
+        return storage_options
+
+    def is_explicit_ack(self) -> bool:
+        return self.httpdb.nuclio.explicit_ack == "enabled" and (
+            not self.nuclio_version or self.nuclio_version >= "1.11.20"
+        )
+
 
 # Global configuration
 config = Config.from_dict(default_config)
@@ -1092,12 +1100,6 @@ def _do_populate(env=None, skip_errors=False):
     if data:
         config.update(data, skip_errors=skip_errors)
 
-    # HACK to enable config property to both have dynamic default and to use the value from dict/env like other
-    # configurations - we just need a key in the dict that is different than the property name, so simply adding prefix
-    # underscore
-    config._cfg["_iguazio_api_url"] = config._cfg["iguazio_api_url"]
-    del config._cfg["iguazio_api_url"]
-
     _validate_config(config)
 
 

mlrun/datastore/sources.py

@@ -793,7 +793,7 @@ class OnlineSource(BaseSourceDriver):
             context=context,
             key_field=self.key_field,
             full_event=True,
-            explicit_ack=mlrun.mlconf.httpdb.nuclio.explicit_ack == "enabled",
+            explicit_ack=mlrun.mlconf.is_explicit_ack(),
             **source_args,
         )
 

mlrun/db/__init__.py

@@ -12,14 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from os import environ
-from urllib.parse import urlparse
 
 from ..config import config
-from ..platforms import add_or_refresh_credentials
-from ..utils import logger
 from .base import RunDBError, RunDBInterface  # noqa
-from .nopdb import NopDB
-from .sqldb import SQLDB
 
 
 def get_or_set_dburl(default=""):
@@ -29,69 +24,10 @@ def get_or_set_dburl(default=""):
     return config.dbpath
 
 
-def get_httpdb_kwargs(host, username, password):
-    username = username or config.httpdb.user
-    password = password or config.httpdb.password
-
-    username, password, token = add_or_refresh_credentials(
-        host, username, password, config.httpdb.token
-    )
-
-    return {
-        "user": username,
-        "password": password,
-        "token": token,
-    }
-
-
-_run_db = None
-_last_db_url = None
-
-
 def get_run_db(url="", secrets=None, force_reconnect=False):
     """Returns the runtime database"""
-    global _run_db, _last_db_url
-
-    if not url:
-        url = get_or_set_dburl("./")
-
-    if (
-        _last_db_url is not None
-        and url == _last_db_url
-        and _run_db
-        and not force_reconnect
-    ):
-        return _run_db
-    _last_db_url = url
-
-    parsed_url = urlparse(url)
-    scheme = parsed_url.scheme.lower()
-    kwargs = {}
-    if "://" not in str(url) or scheme in ["file", "s3", "v3io", "v3ios"]:
-        logger.warning(
-            "Could not detect path to API server, not connected to API server!"
-        )
-        logger.warning(
-            "MLRUN_DBPATH is not set. Set this environment variable to the URL of the API server"
-            " in order to connect"
-        )
-        cls = NopDB
-
-    elif scheme in ("http", "https"):
-        # import here to avoid circular imports
-        from .httpdb import HTTPRunDB
-
-        cls = HTTPRunDB
-        kwargs = get_httpdb_kwargs(
-            parsed_url.hostname, parsed_url.username, parsed_url.password
-        )
-        endpoint = parsed_url.hostname
-        if parsed_url.port:
-            endpoint += f":{parsed_url.port}"
-        url = f"{parsed_url.scheme}://{endpoint}{parsed_url.path}"
-    else:
-        cls = SQLDB
+    # import here to avoid circular import
+    import mlrun.db.factory
 
-    _run_db = cls(url, **kwargs)
-    _run_db.connect(secrets=secrets)
-    return _run_db
+    run_db_factory = mlrun.db.factory.RunDBFactory()
+    return run_db_factory.create_run_db(url, secrets, force_reconnect)

mlrun/db/base.py

@@ -621,3 +621,15 @@ class RunDBInterface(ABC):
         notifications: typing.List[mlrun.model.Notification],
     ):
         pass
+
+    def store_run_notifications(
+        self,
+        notification_objects: typing.List[mlrun.model.Notification],
+        run_uid: str,
+        project: str = None,
+        mask_params: bool = True,
+    ):
+        pass
+
+    def watch_log(self, uid, project="", watch=True, offset=0):
+        pass

mlrun/db/factory.py

@@ -0,0 +1,65 @@
+# Copyright 2023 MLRun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from dependency_injector import containers, providers
+
+import mlrun.db
+import mlrun.db.httpdb
+import mlrun.db.nopdb
+import mlrun.utils.singleton
+from mlrun.utils import logger
+
+
+class RunDBFactory(
+    metaclass=mlrun.utils.singleton.AbstractSingleton,
+):
+    def __init__(self):
+        self._run_db = None
+        self._last_db_url = None
+        self._rundb_container = RunDBContainer()
+
+    def create_run_db(self, url="", secrets=None, force_reconnect=False):
+        """Returns the runtime database"""
+        if not url:
+            url = mlrun.db.get_or_set_dburl("./")
+
+        if (
+            self._last_db_url is not None
+            and url == self._last_db_url
+            and self._run_db
+            and not force_reconnect
+        ):
+            return self._run_db
+
+        self._last_db_url = url
+
+        if "://" not in str(url):
+            logger.warning(
+                "Could not detect path to API server, not connected to API server!"
+            )
+            logger.warning(
+                "MLRUN_DBPATH is misconfigured. Set this environment variable to the URL of the API server"
+                " in order to connect"
+            )
+            self._run_db = self._rundb_container.nop(url)
+
+        else:
+            self._run_db = self._rundb_container.run_db(url)
+
+        self._run_db.connect(secrets=secrets)
+        return self._run_db
+
+
+class RunDBContainer(containers.DeclarativeContainer):
+    nop = providers.Factory(mlrun.db.nopdb.NopDB)
+    run_db = providers.Factory(mlrun.db.httpdb.HTTPRunDB)