mlrun 1.5.0rc1__py3-none-any.whl → 1.5.0rc2__py3-none-any.whl

mlrun/runtimes/sparkjob/abstract.py

@@ -14,21 +14,15 @@
 import os.path
 import typing
 from copy import deepcopy
-from datetime import datetime
-from typing import Dict, Optional, Tuple
 
 from kubernetes import client
 from kubernetes.client.rest import ApiException
-from sqlalchemy.orm import Session
 
+import mlrun.db
 import mlrun.errors
 import mlrun.utils.regex
-from mlrun.api.db.base import DBInterface
 from mlrun.config import config
-from mlrun.db import get_run_db
 from mlrun.errors import err_to_str
-from mlrun.runtimes.base import BaseRuntimeHandler
-from mlrun.runtimes.constants import RunStates, SparkApplicationStates
 
 from ...execution import MLClientCtx
 from ...model import RunObject
@@ -41,7 +35,7 @@ from ...utils import (
     verify_field_regex,
     verify_list_and_update_in,
 )
-from ..base import RunError, RuntimeClassMode
+from ..base import RunError
 from ..kubejob import KubejobRuntime
 from ..pod import KubeResourceSpec
 from ..utils import get_item_name, get_k8s
@@ -242,7 +236,7 @@ class AbstractSparkRuntime(KubejobRuntime):
         sj.with_driver_requests(cpu=1, mem="512m")
 
         sj.deploy()
-        get_run_db().delete_function(name=sj.metadata.name)
+        mlrun.db.get_run_db().delete_function(name=sj.metadata.name)
 
     def _is_using_gpu(self):
         driver_limits = self.spec.driver_resources.get("limits")
@@ -292,7 +286,7 @@ class AbstractSparkRuntime(KubejobRuntime):
         :return True if the function is ready (deployed)
         """
         # connect will populate the config from the server config
-        get_run_db()
+        mlrun.db.get_run_db()
         if not self.spec.build.base_image:
             self.spec.build.base_image = self._default_image
         return super().deploy(
@@ -841,124 +835,3 @@ with ctx:
     @spec.setter
     def spec(self, spec):
         raise NotImplementedError()
-
-
-class SparkRuntimeHandler(BaseRuntimeHandler):
-    kind = "spark"
-    class_modes = {
-        RuntimeClassMode.run: "spark",
-    }
-
-    def _resolve_crd_object_status_info(
-        self, db: DBInterface, db_session: Session, crd_object
-    ) -> Tuple[bool, Optional[datetime], Optional[str]]:
-        state = crd_object.get("status", {}).get("applicationState", {}).get("state")
-        in_terminal_state = state in SparkApplicationStates.terminal_states()
-        desired_run_state = SparkApplicationStates.spark_application_state_to_run_state(
-            state
-        )
-        completion_time = None
-        if in_terminal_state:
-            if crd_object.get("status", {}).get("terminationTime"):
-                completion_time = datetime.fromisoformat(
-                    crd_object.get("status", {})
-                    .get("terminationTime")
-                    .replace("Z", "+00:00")
-                )
-            else:
-                last_submission_attempt_time = crd_object.get("status", {}).get(
-                    "lastSubmissionAttemptTime"
-                )
-                if last_submission_attempt_time:
-                    last_submission_attempt_time = last_submission_attempt_time.replace(
-                        "Z", "+00:00"
-                    )
-                    completion_time = datetime.fromisoformat(
-                        last_submission_attempt_time
-                    )
-        return in_terminal_state, completion_time, desired_run_state
-
-    def _update_ui_url(
-        self,
-        db: DBInterface,
-        db_session: Session,
-        project: str,
-        uid: str,
-        crd_object,
-        run: Dict = None,
-    ):
-        app_state = (
-            crd_object.get("status", {}).get("applicationState", {}).get("state")
-        )
-        state = SparkApplicationStates.spark_application_state_to_run_state(app_state)
-        ui_url = None
-        if state == RunStates.running:
-            ui_url = (
-                crd_object.get("status", {})
-                .get("driverInfo", {})
-                .get("webUIIngressAddress")
-            )
-        db_ui_url = run.get("status", {}).get("ui_url")
-        if db_ui_url == ui_url:
-            return
-        run.setdefault("status", {})["ui_url"] = ui_url
-        db.store_run(db_session, run, uid, project)
-
-    @staticmethod
-    def _are_resources_coupled_to_run_object() -> bool:
-        return True
-
-    @staticmethod
-    def _get_object_label_selector(object_id: str) -> str:
-        return f"mlrun/uid={object_id}"
-
-    @staticmethod
-    def _get_main_runtime_resource_label_selector() -> str:
-        """
-        There are some runtimes which might have multiple k8s resources attached to a one runtime, in this case
-        we don't want to pull logs from all but rather only for the "driver"/"launcher" etc
-        :return: the label selector
-        """
-        return "spark-role=driver"
-
-    @staticmethod
-    def _get_crd_info() -> Tuple[str, str, str]:
-        return (
-            AbstractSparkRuntime.group,
-            AbstractSparkRuntime.version,
-            AbstractSparkRuntime.plural,
-        )
-
-    def _delete_extra_resources(
-        self,
-        db: DBInterface,
-        db_session: Session,
-        namespace: str,
-        deleted_resources: typing.List[Dict],
-        label_selector: str = None,
-        force: bool = False,
-        grace_period: int = None,
-    ):
-        """
-        Handling config maps deletion
-        """
-        uids = []
-        for crd_dict in deleted_resources:
-            uid = crd_dict["metadata"].get("labels", {}).get("mlrun/uid", None)
-            uids.append(uid)
-
-        config_maps = get_k8s().v1api.list_namespaced_config_map(
-            namespace, label_selector=label_selector
-        )
-        for config_map in config_maps.items:
-            try:
-                uid = config_map.metadata.labels.get("mlrun/uid", None)
-                if force or uid in uids:
-                    get_k8s().v1api.delete_namespaced_config_map(
-                        config_map.metadata.name, namespace
-                    )
-                    logger.info(f"Deleted config map: {config_map.metadata.name}")
-            except ApiException as exc:
-                # ignore error if config map is already removed
-                if exc.status != 404:
-                    raise

mlrun/serving/states.py

@@ -1516,7 +1516,7 @@ def _init_async_objects(context, steps):
     source_args = context.get_param("source_args", {})
     default_source = storey.SyncEmitSource(
         context=context,
-        explicit_ack=mlrun.mlconf.httpdb.nuclio.explicit_ack == "enabled",
+        explicit_ack=mlrun.mlconf.is_explicit_ack(),
         **source_args,
     )
     return default_source, wait_for_result

mlrun/utils/db.py

@@ -17,8 +17,6 @@ from datetime import datetime
 
 from sqlalchemy.orm import class_mapper
 
-run_time_fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
-
 
 class BaseModel:
     def to_dict(self, exclude=None):

mlrun/utils/helpers.py

@@ -35,13 +35,13 @@ import pandas
 import semver
 import yaml
 from dateutil import parser
-from deprecated import deprecated
 from pandas._libs.tslibs.timestamps import Timedelta, Timestamp
 from yaml.representer import RepresenterError
 
 import mlrun
 import mlrun.common.schemas
 import mlrun.errors
+import mlrun.utils.regex
 import mlrun.utils.version.version
 from mlrun.errors import err_to_str
 
@@ -238,6 +238,17 @@ def validate_artifact_key_name(
     )
 
 
+def validate_v3io_stream_consumer_group(
+    value: str, raise_on_failure: bool = True
+) -> bool:
+    return mlrun.utils.helpers.verify_field_regex(
+        "consumerGroup",
+        value,
+        mlrun.utils.regex.v3io_stream_consumer_group,
+        raise_on_failure=raise_on_failure,
+    )
+
+
 def get_regex_list_as_string(regex_list: List) -> str:
     """
     This function is used to combine a list of regex strings into a single regex,
@@ -753,18 +764,6 @@ def new_pipe_metadata(
     return conf
 
 
-# TODO: remove in 1.5.0
-@deprecated(
-    version="1.3.0",
-    reason="'new_pipe_meta' will be removed in 1.5.0",
-    category=FutureWarning,
-)
-def new_pipe_meta(artifact_path=None, ttl=None, *args):
-    return new_pipe_metadata(
-        artifact_path=artifact_path, cleanup_ttl=ttl, op_transformers=args
-    )
-
-
 def _convert_python_package_version_to_image_tag(version: typing.Optional[str]):
     return (
         version.replace("+", "-").replace("0.0.0-", "") if version is not None else None
@@ -1445,3 +1444,10 @@ class DeprecationHelper(object):
     def __getattr__(self, attr):
         self._warn()
         return getattr(self._new_target, attr)
+
+
+def normalize_workflow_name(name, project_name):
+    workflow_name = (
+        name.lstrip(project_name).lstrip("-") if project_name in name else name
+    )
+    return workflow_name

mlrun/utils/notifications/notification_pusher.py

@@ -20,10 +20,9 @@ from concurrent.futures import ThreadPoolExecutor
 
 from fastapi.concurrency import run_in_threadpool
 
-import mlrun.api.db.base
-import mlrun.api.db.session
 import mlrun.common.schemas
 import mlrun.config
+import mlrun.db.base
 import mlrun.lists
 import mlrun.model
 import mlrun.utils.helpers
@@ -63,10 +62,7 @@ class NotificationPusher(object):
                 if self._should_notify(run, notification):
                     self._load_notification(run, notification)
 
-    def push(
-        self,
-        db: mlrun.api.db.base.DBInterface = None,
-    ):
+    def push(self):
         """
         Asynchronously push notifications for all runs in the initialized runs list (if they should be pushed).
         When running from a sync environment, the notifications will be pushed asynchronously however the function will
@@ -82,7 +78,6 @@ class NotificationPusher(object):
                     notification_data[0],
                     notification_data[1],
                     notification_data[2],
-                    db,
                 )
 
         async def _async_push():
@@ -93,7 +88,6 @@ class NotificationPusher(object):
                         notification_data[0],
                         notification_data[1],
                         notification_data[2],
-                        db,
                     )
                 )
 
@@ -200,7 +194,6 @@ class NotificationPusher(object):
         notification: NotificationBase,
         run: mlrun.model.RunObject,
         notification_object: mlrun.model.Notification,
-        db: mlrun.api.db.base.DBInterface,
     ):
         message, severity, runs = self._prepare_notification_args(
             run, notification_object
@@ -213,7 +206,6 @@ class NotificationPusher(object):
         try:
             notification.push(message, severity, runs)
             self._update_notification_status(
-                db,
                 run.metadata.uid,
                 run.metadata.project,
                 notification_object,
@@ -222,7 +214,6 @@ class NotificationPusher(object):
             )
         except Exception as exc:
             self._update_notification_status(
-                db,
                 run.metadata.uid,
                 run.metadata.project,
                 notification_object,
@@ -235,7 +226,6 @@ class NotificationPusher(object):
         notification: NotificationBase,
         run: mlrun.model.RunObject,
         notification_object: mlrun.model.Notification,
-        db: mlrun.api.db.base.DBInterface,
     ):
         message, severity, runs = self._prepare_notification_args(
             run, notification_object
@@ -250,7 +240,6 @@ class NotificationPusher(object):
 
             await run_in_threadpool(
                 self._update_notification_status,
-                db,
                 run.metadata.uid,
                 run.metadata.project,
                 notification_object,
@@ -260,7 +249,6 @@ class NotificationPusher(object):
         except Exception as exc:
             await run_in_threadpool(
                 self._update_notification_status,
-                db,
                 run.metadata.uid,
                 run.metadata.project,
                 notification_object,
@@ -270,30 +258,22 @@ class NotificationPusher(object):
 
     @staticmethod
     def _update_notification_status(
-        db: mlrun.api.db.base.DBInterface,
         run_uid: str,
         project: str,
         notification: mlrun.model.Notification,
         status: str = None,
         sent_time: typing.Optional[datetime.datetime] = None,
     ):
-
-        # nothing to update if not running as api
-        # note, the notification mechanism may run "locally" for certain runtimes
-        if not mlrun.config.is_running_as_api():
-            return
-
-        # TODO: move to api side
-        db_session = mlrun.api.db.session.create_session()
+        db = mlrun.get_run_db()
         notification.status = status or notification.status
         notification.sent_time = sent_time or notification.sent_time
 
-        # store directly in db, no need to use crud as the secrets are already loaded
+        # There is no need to mask the params as the secrets are already loaded
         db.store_run_notifications(
-            db_session,
             [notification],
             run_uid,
             project,
+            mask_params=False,
         )
 
 

mlrun/utils/regex.py

@@ -72,8 +72,8 @@ sparkjob_name = label_value + sprakjob_length + sparkjob_service_name
 # A project name have the following restrictions:
 # It should be a valid Nuclio Project CRD name which is dns 1123 subdomain
 # It should be a valid k8s label value since Nuclio use the project name in labels of resources
-# It should be a valid namespace name (cause we plan to map it to one) which is dns 1123 label
-# of the 3 restrictions, dns 1123 label is the most strict, so we enforce only it
+# It should be a valid namespace name (because we plan to map it to one) which is dns 1123 label
+# of the 3 restrictions, dns 1123 label is strictest, so we enforce only it
 project_name = dns_1123_label
 
 # Special characters are not permitted in tag names because they can be included in the url and cause problems.
@@ -83,3 +83,8 @@ tag_name = label_value
 secret_key = k8s_secret_and_config_map_key
 
 artifact_key = [r"[^\/\\]+$"]
+
+# must not start with _
+# must be alphanumeric or _
+# max 256 length
+v3io_stream_consumer_group = [r"^(?!_)[a-zA-Z0-9_]{1,256}$"]

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "8657600387e10ed5be7da503d356e86e92c07c35",
-  "version": "1.5.0-rc1"
+  "git_commit": "dcf0e7215136557c4f4d1159a1b494defce16d40",
+  "version": "1.5.0-rc2"
 }

{mlrun-1.5.0rc1.dist-info → mlrun-1.5.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.5.0rc1
+Version: 1.5.0rc2
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -48,11 +48,12 @@ Requires-Dist: semver (~=2.13)
 Requires-Dist: dask (~=2021.11.2)
 Requires-Dist: distributed (~=2021.11.2)
 Requires-Dist: kubernetes (~=12.0)
+Requires-Dist: dependency-injector (~=4.41)
 Requires-Dist: humanfriendly (~=9.2)
 Requires-Dist: fastapi (~=0.95.2)
-Requires-Dist: fsspec (~=2023.1.0)
+Requires-Dist: fsspec (~=2023.6.0)
 Requires-Dist: v3iofs (~=0.1.15)
-Requires-Dist: storey (~=1.5.0)
+Requires-Dist: storey (~=1.5.2)
 Requires-Dist: deepdiff (~=5.0)
 Requires-Dist: pymysql (~=1.0)
 Requires-Dist: inflection (~=0.5.0)
@@ -61,16 +62,16 @@ Requires-Dist: setuptools (~=65.5)
 Requires-Dist: deprecated (~=1.2)
 Requires-Dist: jinja2 (~=3.1)
 Provides-Extra: all
-Requires-Dist: adlfs (~=2022.2.0) ; extra == 'all'
-Requires-Dist: aiobotocore (~=2.4.2) ; extra == 'all'
+Requires-Dist: adlfs (~=2023.4.0) ; extra == 'all'
+Requires-Dist: aiobotocore (~=2.5.2) ; extra == 'all'
 Requires-Dist: avro (~=1.11) ; extra == 'all'
 Requires-Dist: azure-core (~=1.24) ; extra == 'all'
 Requires-Dist: azure-identity (~=1.5) ; extra == 'all'
 Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'all'
 Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'all'
 Requires-Dist: bokeh (>=2.4.2,~=2.4) ; extra == 'all'
-Requires-Dist: boto3 (~=1.24.59) ; extra == 'all'
-Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'all'
+Requires-Dist: boto3 (~=1.26.161) ; extra == 'all'
+Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'all'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'all'
 Requires-Dist: google-cloud-storage (~=1.20) ; extra == 'all'
 Requires-Dist: google-cloud (~=0.34) ; extra == 'all'
@@ -80,7 +81,7 @@ Requires-Dist: msrest (~=0.6.21) ; extra == 'all'
 Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'all'
 Requires-Dist: pyopenssl (>=23) ; extra == 'all'
 Requires-Dist: redis (~=4.3) ; extra == 'all'
-Requires-Dist: s3fs (~=2023.1.0) ; extra == 'all'
+Requires-Dist: s3fs (~=2023.6.0) ; extra == 'all'
 Provides-Extra: api
 Requires-Dist: uvicorn (~=0.20.0) ; extra == 'api'
 Requires-Dist: dask-kubernetes (~=0.11.0) ; extra == 'api'
@@ -92,7 +93,7 @@ Provides-Extra: azure-blob-storage
 Requires-Dist: msrest (~=0.6.21) ; extra == 'azure-blob-storage'
 Requires-Dist: azure-core (~=1.24) ; extra == 'azure-blob-storage'
 Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'azure-blob-storage'
-Requires-Dist: adlfs (~=2022.2.0) ; extra == 'azure-blob-storage'
+Requires-Dist: adlfs (~=2023.4.0) ; extra == 'azure-blob-storage'
 Requires-Dist: pyopenssl (>=23) ; extra == 'azure-blob-storage'
 Provides-Extra: azure-key-vault
 Requires-Dist: azure-identity (~=1.5) ; extra == 'azure-key-vault'
@@ -101,15 +102,15 @@ Requires-Dist: pyopenssl (>=23) ; extra == 'azure-key-vault'
 Provides-Extra: bokeh
 Requires-Dist: bokeh (>=2.4.2,~=2.4) ; extra == 'bokeh'
 Provides-Extra: complete
-Requires-Dist: adlfs (~=2022.2.0) ; extra == 'complete'
-Requires-Dist: aiobotocore (~=2.4.2) ; extra == 'complete'
+Requires-Dist: adlfs (~=2023.4.0) ; extra == 'complete'
+Requires-Dist: aiobotocore (~=2.5.2) ; extra == 'complete'
 Requires-Dist: avro (~=1.11) ; extra == 'complete'
 Requires-Dist: azure-core (~=1.24) ; extra == 'complete'
 Requires-Dist: azure-identity (~=1.5) ; extra == 'complete'
 Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'complete'
 Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'complete'
-Requires-Dist: boto3 (~=1.24.59) ; extra == 'complete'
-Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'complete'
+Requires-Dist: boto3 (~=1.26.161) ; extra == 'complete'
+Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'complete'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'complete'
 Requires-Dist: graphviz (~=0.20.0) ; extra == 'complete'
 Requires-Dist: kafka-python (~=2.0) ; extra == 'complete'
@@ -117,19 +118,19 @@ Requires-Dist: msrest (~=0.6.21) ; extra == 'complete'
 Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'complete'
 Requires-Dist: pyopenssl (>=23) ; extra == 'complete'
 Requires-Dist: redis (~=4.3) ; extra == 'complete'
-Requires-Dist: s3fs (~=2023.1.0) ; extra == 'complete'
+Requires-Dist: s3fs (~=2023.6.0) ; extra == 'complete'
 Provides-Extra: complete-api
-Requires-Dist: adlfs (~=2022.2.0) ; extra == 'complete-api'
-Requires-Dist: aiobotocore (~=2.4.2) ; extra == 'complete-api'
+Requires-Dist: adlfs (~=2023.4.0) ; extra == 'complete-api'
+Requires-Dist: aiobotocore (~=2.5.2) ; extra == 'complete-api'
 Requires-Dist: apscheduler (~=3.6) ; extra == 'complete-api'
 Requires-Dist: avro (~=1.11) ; extra == 'complete-api'
 Requires-Dist: azure-core (~=1.24) ; extra == 'complete-api'
 Requires-Dist: azure-identity (~=1.5) ; extra == 'complete-api'
 Requires-Dist: azure-keyvault-secrets (~=4.2) ; extra == 'complete-api'
 Requires-Dist: azure-storage-blob (~=12.13) ; extra == 'complete-api'
-Requires-Dist: boto3 (~=1.24.59) ; extra == 'complete-api'
+Requires-Dist: boto3 (~=1.26.161) ; extra == 'complete-api'
 Requires-Dist: dask-kubernetes (~=0.11.0) ; extra == 'complete-api'
-Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'complete-api'
+Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'complete-api'
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'complete-api'
 Requires-Dist: graphviz (~=0.20.0) ; extra == 'complete-api'
 Requires-Dist: igz-mgmt (~=0.0.10) ; extra == 'complete-api'
@@ -139,7 +140,7 @@ Requires-Dist: objgraph (~=3.5) ; extra == 'complete-api'
 Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'complete-api'
 Requires-Dist: pyopenssl (>=23) ; extra == 'complete-api'
 Requires-Dist: redis (~=4.3) ; extra == 'complete-api'
-Requires-Dist: s3fs (~=2023.1.0) ; extra == 'complete-api'
+Requires-Dist: s3fs (~=2023.6.0) ; extra == 'complete-api'
 Requires-Dist: sqlite3-to-mysql (~=1.4) ; extra == 'complete-api'
 Requires-Dist: uvicorn (~=0.20.0) ; extra == 'complete-api'
 Provides-Extra: google-cloud
@@ -149,7 +150,7 @@ Requires-Dist: google-cloud (~=0.34) ; extra == 'google-cloud'
 Provides-Extra: google-cloud-bigquery
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas] (~=3.2) ; extra == 'google-cloud-bigquery'
 Provides-Extra: google-cloud-storage
-Requires-Dist: gcsfs (~=2023.1.0) ; extra == 'google-cloud-storage'
+Requires-Dist: gcsfs (~=2023.6.0) ; extra == 'google-cloud-storage'
 Provides-Extra: graphviz
 Requires-Dist: graphviz (~=0.20.0) ; extra == 'graphviz'
 Provides-Extra: kafka
@@ -160,9 +161,9 @@ Requires-Dist: plotly (<5.12.0,~=5.4) ; extra == 'plotly'
 Provides-Extra: redis
 Requires-Dist: redis (~=4.3) ; extra == 'redis'
 Provides-Extra: s3
-Requires-Dist: boto3 (~=1.24.59) ; extra == 's3'
-Requires-Dist: aiobotocore (~=2.4.2) ; extra == 's3'
-Requires-Dist: s3fs (~=2023.1.0) ; extra == 's3'
+Requires-Dist: boto3 (~=1.26.161) ; extra == 's3'
+Requires-Dist: aiobotocore (~=2.5.2) ; extra == 's3'
+Requires-Dist: s3fs (~=2023.6.0) ; extra == 's3'
 
 <a id="top"></a>
 [![Build Status](https://github.com/mlrun/mlrun/workflows/CI/badge.svg)](https://github.com/mlrun/mlrun/actions)