mlrun 1.4.0rc25__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/api/utils/clients/iguazio.py

@@ -28,7 +28,7 @@ import igz_mgmt.schemas.manual_events
 import requests.adapters
 from fastapi.concurrency import run_in_threadpool
 
-import mlrun.api.utils.projects.remotes.leader
+import mlrun.api.utils.projects.remotes.leader as project_leader
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.helpers
@@ -73,7 +73,7 @@ class SessionPlanes:
 
 
 class Client(
-    mlrun.api.utils.projects.remotes.leader.Member,
+    project_leader.Member,
     metaclass=mlrun.utils.singleton.AbstractSingleton,
 ):
     def __init__(self, *args, **kwargs) -> None:
@@ -93,37 +93,6 @@ class Client(
         self._logger = logger.get_child("iguazio-client")
         self._igz_clients = {}
 
-    def try_get_grafana_service_url(self, session: str) -> typing.Optional[str]:
-        """
-        Try to find a ready grafana app service, and return its URL
-        If nothing found, returns None
-        """
-        self._logger.debug("Getting grafana service url from Iguazio")
-        response = self._send_request_to_api(
-            "GET",
-            "app_services_manifests",
-            "Failed getting app services manifests from Iguazio",
-            session,
-        )
-        response_body = response.json()
-        for app_services_manifest in response_body.get("data", []):
-            for app_service in app_services_manifest.get("attributes", {}).get(
-                "app_services", []
-            ):
-                if (
-                    app_service.get("spec", {}).get("kind") == "grafana"
-                    and app_service.get("status", {}).get("state") == "ready"
-                    and len(app_service.get("status", {}).get("urls", [])) > 0
-                ):
-                    url_kind_to_url = {}
-                    for url in app_service["status"]["urls"]:
-                        url_kind_to_url[url["kind"]] = url["url"]
-                    # precedence for https
-                    for kind in ["https", "http"]:
-                        if kind in url_kind_to_url:
-                            return url_kind_to_url[kind]
-        return None
-
     def verify_request_session(
         self, request: fastapi.Request
     ) -> mlrun.common.schemas.AuthInfo:
@@ -326,6 +295,37 @@ class Client(
         """
         return True
 
+    def try_get_grafana_service_url(self, session: str) -> typing.Optional[str]:
+        """
+        Try to find a ready grafana app service, and return its URL
+        If nothing found, returns None
+        """
+        self._logger.debug("Getting grafana service url from Iguazio")
+        response = self._send_request_to_api(
+            "GET",
+            "app_services_manifests",
+            "Failed getting app services manifests from Iguazio",
+            session,
+        )
+        response_body = response.json()
+        for app_services_manifest in response_body.get("data", []):
+            for app_service in app_services_manifest.get("attributes", {}).get(
+                "app_services", []
+            ):
+                if (
+                    app_service.get("spec", {}).get("kind") == "grafana"
+                    and app_service.get("status", {}).get("state") == "ready"
+                    and len(app_service.get("status", {}).get("urls", [])) > 0
+                ):
+                    url_kind_to_url = {}
+                    for url in app_service["status"]["urls"]:
+                        url_kind_to_url[url["kind"]] = url["url"]
+                    # precedence for https
+                    for kind in ["https", "http"]:
+                        if kind in url_kind_to_url:
+                            return url_kind_to_url[kind]
+        return None
+
     def emit_manual_event(self, access_key: str, event: igz_mgmt.Event):
         """
         Emit a manual event to Iguazio

mlrun/api/utils/clients/nuclio.py

@@ -20,7 +20,7 @@ import typing
 import requests.adapters
 import sqlalchemy.orm
 
-import mlrun.api.utils.projects.remotes.follower
+import mlrun.api.utils.projects.remotes.follower as project_follower
 import mlrun.common.schemas
 import mlrun.errors
 import mlrun.utils.singleton
@@ -28,7 +28,7 @@ from mlrun.utils import logger
 
 
 class Client(
-    mlrun.api.utils.projects.remotes.follower.Member,
+    project_follower.Member,
     metaclass=mlrun.utils.singleton.AbstractSingleton,
 ):
     def __init__(self) -> None:

mlrun/api/utils/periodic.py

@@ -27,7 +27,9 @@ tasks: typing.Dict = {}
 # This module is different from mlrun.db.periodic in that this module's functions aren't supposed to persist
 # also this module supports asyncio while the other currently not
 # TODO: merge the modules
-async def _periodic_function_wrapper(interval: int, function, *args, **kwargs):
+async def _periodic_function_wrapper(
+    interval: typing.Union[int, float], function, *args, **kwargs
+):
     while True:
         try:
             if asyncio.iscoroutinefunction(function):
@@ -45,7 +47,12 @@ async def _periodic_function_wrapper(interval: int, function, *args, **kwargs):
 
 
 def run_function_periodically(
-    interval: int, name: str, replace: bool, function, *args, **kwargs
+    interval: typing.Union[float, int],
+    name: str,
+    replace: bool,
+    function,
+    *args,
+    **kwargs
 ):
     global tasks
     logger.debug("Submitting function to run periodically", name=name)

mlrun/api/utils/projects/follower.py

@@ -27,7 +27,7 @@ import mlrun.api.utils.auth.verifier
 import mlrun.api.utils.clients.iguazio
 import mlrun.api.utils.clients.nuclio
 import mlrun.api.utils.periodic
-import mlrun.api.utils.projects.member
+import mlrun.api.utils.projects.member as project_member
 import mlrun.api.utils.projects.remotes.leader
 import mlrun.api.utils.projects.remotes.nop_leader
 import mlrun.common.schemas
@@ -42,7 +42,7 @@ from mlrun.utils import logger
 
 
 class Member(
-    mlrun.api.utils.projects.member.Member,
+    project_member.Member,
     metaclass=mlrun.utils.singleton.AbstractSingleton,
 ):
     def initialize(self):
@@ -346,11 +346,18 @@ class Member(
                         "Found project in the DB that is not in leader. Removing",
                         name=project_to_remove,
                     )
-                    mlrun.api.crud.Projects().delete_project(
-                        db_session,
-                        project_to_remove,
-                        mlrun.common.schemas.DeletionStrategy.cascading,
-                    )
+                    try:
+                        mlrun.api.crud.Projects().delete_project(
+                            db_session,
+                            project_to_remove,
+                            mlrun.common.schemas.DeletionStrategy.cascading,
+                        )
+                    except Exception as exc:
+                        logger.warning(
+                            "Failed to delete project from DB, continuing...",
+                            name=project_to_remove,
+                            exc=err_to_str(exc),
+                        )
             if latest_updated_at:
 
                 # sanity and defensive programming - if the leader returned a latest_updated_at that is older

mlrun/api/utils/projects/leader.py

@@ -23,6 +23,7 @@ import mlrun.api.db.session
 import mlrun.api.utils.clients.nuclio
 import mlrun.api.utils.periodic
 import mlrun.api.utils.projects.member
+import mlrun.api.utils.projects.member as project_member
 import mlrun.api.utils.projects.remotes.follower
 import mlrun.api.utils.projects.remotes.nop_follower
 import mlrun.common.schemas
@@ -37,7 +38,7 @@ from mlrun.utils import logger
 
 
 class Member(
-    mlrun.api.utils.projects.member.Member,
+    project_member.Member,
     metaclass=mlrun.utils.singleton.AbstractSingleton,
 ):
     def initialize(self):

mlrun/api/utils/projects/remotes/nop_follower.py

@@ -18,12 +18,12 @@ import mergedeep
 import sqlalchemy.orm
 
 import mlrun.api.utils.helpers
-import mlrun.api.utils.projects.remotes.follower
+import mlrun.api.utils.projects.remotes.follower as project_follower
 import mlrun.common.schemas
 import mlrun.errors
 
 
-class Member(mlrun.api.utils.projects.remotes.follower.Member):
+class Member(project_follower.Member):
     def __init__(self) -> None:
         super().__init__()
         self._projects: typing.Dict[str, mlrun.common.schemas.Project] = {}

mlrun/api/utils/projects/remotes/nop_leader.py

@@ -15,13 +15,13 @@
 import datetime
 import typing
 
-import mlrun.api.utils.projects.remotes.leader
+import mlrun.api.utils.projects.remotes.leader as project_leader
 import mlrun.api.utils.singletons.project_member
 import mlrun.common.schemas
 import mlrun.errors
 
 
-class Member(mlrun.api.utils.projects.remotes.leader.Member):
+class Member(project_leader.Member):
     def __init__(self) -> None:
         super().__init__()
         self.db_session = None

mlrun/api/utils/runtimes/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

mlrun/api/utils/runtimes/nuclio.py

@@ -0,0 +1,43 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import mlrun.api.utils.clients.nuclio
+from mlrun.config import config
+from mlrun.errors import err_to_str
+from mlrun.utils import logger
+
+cached_nuclio_version = None
+
+
+# if nuclio version specified on mlrun config set it likewise,
+# if not specified, get it from nuclio api client
+# since this is a heavy operation (sending requests to API), and it's unlikely that the version
+# will change - cache it (this means if we upgrade nuclio, we need to restart mlrun to re-fetch the new version)
+def resolve_nuclio_version():
+    global cached_nuclio_version
+
+    if not cached_nuclio_version:
+
+        # config override everything
+        nuclio_version = config.nuclio_version
+        if not nuclio_version and config.nuclio_dashboard_url:
+            try:
+                nuclio_client = mlrun.api.utils.clients.nuclio.Client()
+                nuclio_version = nuclio_client.get_dashboard_version()
+            except Exception as exc:
+                logger.warning("Failed to resolve nuclio version", exc=err_to_str(exc))
+
+        cached_nuclio_version = nuclio_version
+
+    return cached_nuclio_version

mlrun/api/utils/scheduler.py

@@ -142,14 +142,14 @@ class Scheduler:
         self._enrich_schedule_notifications(project, name, scheduled_object)
 
         get_db().create_schedule(
-            db_session,
-            project,
-            name,
-            kind,
-            scheduled_object,
-            cron_trigger,
-            concurrency_limit,
-            labels,
+            session=db_session,
+            project=project,
+            name=name,
+            kind=kind,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            concurrency_limit=concurrency_limit,
+            labels=labels,
         )
         job = self._create_schedule_in_scheduler(
             project,
@@ -217,13 +217,13 @@ class Scheduler:
         self._enrich_schedule_notifications(project, name, scheduled_object)
 
         get_db().update_schedule(
-            db_session,
-            project,
-            name,
-            scheduled_object,
-            cron_trigger,
-            labels,
-            concurrency_limit,
+            session=db_session,
+            project=project,
+            name=name,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
         )
         db_schedule = get_db().get_schedule(db_session, project, name)
 
@@ -303,6 +303,89 @@ class Scheduler:
             )
         get_db().delete_schedules(db_session, project)
 
+    @mlrun.api.utils.helpers.ensure_running_on_chief
+    def store_schedule(
+        self,
+        db_session: Session,
+        auth_info: mlrun.common.schemas.AuthInfo,
+        project: str,
+        name: str,
+        scheduled_object: Union[Dict, Callable] = None,
+        cron_trigger: Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
+        labels: Dict = None,
+        concurrency_limit: int = None,
+        kind: mlrun.common.schemas.ScheduleKinds = None,
+    ):
+        if isinstance(cron_trigger, str):
+            cron_trigger = mlrun.common.schemas.ScheduleCronTrigger.from_crontab(
+                cron_trigger
+            )
+
+        if cron_trigger is not None:
+            self._validate_cron_trigger(cron_trigger)
+
+        logger.debug(
+            "Storing schedule",
+            project=project,
+            name=name,
+            kind=kind,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
+        )
+
+        if not kind:
+            # TODO: Need to think of a way to not use `get_schedule`
+            #  in this function or in `get_db().store_function()` in this flow
+            #  because we must have kind to ensure that auth info has access key.
+            db_schedule = get_db().get_schedule(
+                db_session, project, name, raise_on_not_found=False
+            )
+            kind = db_schedule.kind
+
+        self._ensure_auth_info_has_access_key(auth_info, kind)
+        secret_name = self._store_schedule_secrets_using_auth_secret(auth_info)
+        labels = self._append_access_key_secret_to_labels(labels, secret_name)
+        self._enrich_schedule_notifications(project, name, scheduled_object)
+
+        db_schedule = get_db().store_schedule(
+            session=db_session,
+            project=project,
+            name=name,
+            kind=kind,
+            scheduled_object=scheduled_object,
+            cron_trigger=cron_trigger,
+            labels=labels,
+            concurrency_limit=concurrency_limit,
+        )
+        if db_schedule:
+            updated_schedule = self._transform_and_enrich_db_schedule(
+                db_session, db_schedule
+            )
+
+            job = self._update_schedule_in_scheduler(
+                project,
+                name,
+                updated_schedule.kind,
+                updated_schedule.scheduled_object,
+                updated_schedule.cron_trigger,
+                updated_schedule.concurrency_limit,
+                auth_info,
+            )
+        else:
+            job = self._create_schedule_in_scheduler(
+                project,
+                name,
+                kind,
+                scheduled_object,
+                cron_trigger,
+                concurrency_limit,
+                auth_info,
+            )
+
+        self.update_schedule_next_run_time(db_session, name, project, job)
+
     def _remove_schedule_scheduler_resources(self, db_session: Session, project, name):
         self._remove_schedule_from_scheduler(project, name)
         # This is kept for backwards compatibility - if schedule was using the "old" format of storing secrets, then

mlrun/api/utils/singletons/db.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import mlrun.db
 from mlrun.api.db.base import DBInterface
 from mlrun.api.db.sqldb.db import SQLDB
-from mlrun.api.db.sqldb.session import create_session
+from mlrun.common.db.sql_session import create_session
 from mlrun.config import config
 from mlrun.utils import logger
 
@@ -34,6 +35,9 @@ def initialize_db(override_db=None):
         return
     logger.info("Creating sql db")
     db = SQLDB(config.httpdb.dsn)
+    # set the run db path to the sql db dsn
+    mlrun.db.get_or_set_dburl(config.httpdb.dsn)
+
     db_session = None
     try:
         db_session = create_session()

mlrun/api/utils/singletons/project_member.py

@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+import typing
+
 import mlrun.api.utils.projects.follower
 import mlrun.api.utils.projects.leader
 import mlrun.api.utils.projects.member
 import mlrun.config
 
 # TODO: something nicer
-project_member: mlrun.api.utils.projects.member.Member = None
+project_member: typing.Optional[mlrun.api.utils.projects.member.Member] = None
 
 
 def initialize_project_member():

mlrun/api/utils/singletons/scheduler.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from mlrun.api.db.sqldb.session import create_session
 from mlrun.api.utils.scheduler import Scheduler
+from mlrun.common.db.sql_session import create_session
 
 # TODO: something nicer
 scheduler: Scheduler = None

mlrun/artifacts/base.py

@@ -698,10 +698,10 @@ class LinkArtifact(Artifact):
         self._spec = self._verify_dict(spec, "spec", LinkArtifactSpec)
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyArtifact' will be removed in 1.5.0, use 'Artifact' instead",
+    reason="'LegacyArtifact' will be removed in 1.6.0, use 'Artifact' instead",
     category=FutureWarning,
 )
 class LegacyArtifact(ModelObj):
@@ -865,10 +865,10 @@ class LegacyArtifact(ModelObj):
         return generate_target_path(self, artifact_path, producer)
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyDirArtifact' will be removed in 1.5.0, use 'DirArtifact' instead",
+    reason="'LegacyDirArtifact' will be removed in 1.6.0, use 'DirArtifact' instead",
     category=FutureWarning,
 )
 class LegacyDirArtifact(LegacyArtifact):
@@ -901,10 +901,10 @@ class LegacyDirArtifact(LegacyArtifact):
             store_manager.object(url=target).upload(file_path)
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyLinkArtifact' will be removed in 1.5.0, use 'LinkArtifact' instead",
+    reason="'LegacyLinkArtifact' will be removed in 1.6.0, use 'LinkArtifact' instead",
     category=FutureWarning,
 )
 class LegacyLinkArtifact(LegacyArtifact):

mlrun/artifacts/dataset.py

@@ -322,10 +322,10 @@ class DatasetArtifact(Artifact):
         self.status.stats = stats
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyTableArtifact' will be removed in 1.5.0, use 'TableArtifact' instead",
+    reason="'LegacyTableArtifact' will be removed in 1.6.0, use 'TableArtifact' instead",
     category=FutureWarning,
 )
 class LegacyTableArtifact(LegacyArtifact):
@@ -375,10 +375,10 @@ class LegacyTableArtifact(LegacyArtifact):
         return csv_buffer.getvalue()
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyDatasetArtifact' will be removed in 1.5.0, use 'DatasetArtifact' instead",
+    reason="'LegacyDatasetArtifact' will be removed in 1.6.0, use 'DatasetArtifact' instead",
     category=FutureWarning,
 )
 class LegacyDatasetArtifact(LegacyArtifact):

mlrun/artifacts/manager.py

@@ -17,7 +17,6 @@ from os.path import isdir
 
 import mlrun.config
 
-from ..db import RunDBInterface
 from ..utils import (
     is_legacy_artifact,
     is_relative_path,
@@ -65,7 +64,7 @@ artifact_types = {
     "bokeh": BokehArtifact,
 }
 
-# TODO - Remove this when legacy types are deleted in 1.5.0
+# TODO - Remove this when legacy types are deleted in 1.6.0
 legacy_artifact_types = {
     "": LegacyArtifact,
     "dir": LegacyDirArtifact,
@@ -111,7 +110,7 @@ def dict_to_artifact(struct: dict) -> Artifact:
 class ArtifactManager:
     def __init__(
         self,
-        db: RunDBInterface = None,
+        db: "mlrun.db.RunDBInterface" = None,
         calc_hash=True,
     ):
         self.calc_hash = calc_hash

mlrun/artifacts/model.py

@@ -370,10 +370,10 @@ class ModelArtifact(Artifact):
         return mlrun.get_dataitem(target_model_path).get()
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyModelArtifact' will be removed in 1.5.0, use 'ModelArtifact' instead",
+    reason="'LegacyModelArtifact' will be removed in 1.6.0, use 'ModelArtifact' instead",
     category=FutureWarning,
 )
 class LegacyModelArtifact(LegacyArtifact):

mlrun/artifacts/plots.py

@@ -244,10 +244,10 @@ class PlotlyArtifact(Artifact):
         return self._figure.to_html()
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyPlotArtifact' will be removed in 1.5.0, use 'PlotArtifact' instead",
+    reason="'LegacyPlotArtifact' will be removed in 1.6.0, use 'PlotArtifact' instead",
     category=FutureWarning,
 )
 class LegacyPlotArtifact(LegacyArtifact):
@@ -291,10 +291,10 @@ class LegacyPlotArtifact(LegacyArtifact):
         return self._TEMPLATE.format(self.description or self.key, self.key, data_uri)
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyChartArtifact' will be removed in 1.5.0, use 'ChartArtifact' instead",
+    reason="'LegacyChartArtifact' will be removed in 1.6.0, use 'ChartArtifact' instead",
     category=FutureWarning,
 )
 class LegacyChartArtifact(LegacyArtifact):
@@ -365,10 +365,10 @@ class LegacyChartArtifact(LegacyArtifact):
         )
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyBokehArtifact' will be removed in 1.5.0, use 'BokehArtifact' instead",
+    reason="'LegacyBokehArtifact' will be removed in 1.6.0, use 'BokehArtifact' instead",
     category=FutureWarning,
 )
 class LegacyBokehArtifact(LegacyArtifact):
@@ -421,10 +421,10 @@ class LegacyBokehArtifact(LegacyArtifact):
         return file_html(self._figure, CDN, self.key)
 
 
-# TODO: remove in 1.5.0
+# TODO: remove in 1.6.0
 @deprecated(
     version="1.3.0",
-    reason="'LegacyPlotlyArtifact' will be removed in 1.5.0, use 'PlotlyArtifact' instead",
+    reason="'LegacyPlotlyArtifact' will be removed in 1.6.0, use 'PlotlyArtifact' instead",
     category=FutureWarning,
 )
 class LegacyPlotlyArtifact(LegacyArtifact):

mlrun/common/db/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#