mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/api/utils/db/mysql.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,29 +32,16 @@ class MySQLUtil(object):
         "functions",
     ]
 
-    def __init__(self):
-        mysql_dsn_data = self.get_mysql_dsn_data()
-        if not mysql_dsn_data:
-            raise RuntimeError(f"Invalid mysql dsn: {self.get_dsn()}")
-
-    @staticmethod
-    def wait_for_db_liveness(logger, retry_interval=3, timeout=2 * 60):
-        logger.debug("Waiting for database liveness")
-        mysql_dsn_data = MySQLUtil.get_mysql_dsn_data()
-        if not mysql_dsn_data:
-            dsn = MySQLUtil.get_dsn()
-            if "sqlite" in dsn:
-                logger.debug("SQLite DB is used, liveness check not needed")
-            else:
-                logger.warn(
-                    f"Invalid mysql dsn: {MySQLUtil.get_dsn()}, assuming live and skipping liveness verification"
-                )
-            return
+    def __init__(self, logger: mlrun.utils.Logger):
+        self._logger = logger
 
+    def wait_for_db_liveness(self, retry_interval=3, timeout=2 * 60):
+        self._logger.debug("Waiting for database liveness")
+        mysql_dsn_data = self.get_mysql_dsn_data()
         tmp_connection = mlrun.utils.retry_until_successful(
             retry_interval,
             timeout,
-            logger,
+            self._logger,
             True,
             pymysql.connect,
             host=mysql_dsn_data["host"],
@@ -62,7 +49,7 @@ class MySQLUtil(object):
             port=int(mysql_dsn_data["port"]),
             database=mysql_dsn_data["database"],
         )
-        logger.debug("Database ready for connection")
+        self._logger.debug("Database ready for connection")
         tmp_connection.close()
 
     def check_db_has_tables(self):
@@ -78,6 +65,18 @@ class MySQLUtil(object):
         finally:
             connection.close()
 
+    def set_modes(self, modes):
+        if not modes or modes in ["nil", "none"]:
+            self._logger.debug("No sql modes were given, bailing", modes=modes)
+            return
+        connection = self._create_connection()
+        try:
+            self._logger.debug("Setting sql modes", modes=modes)
+            with connection.cursor() as cursor:
+                cursor.execute("SET GLOBAL sql_mode=%s;", (modes,))
+        finally:
+            connection.close()
+
     def check_db_has_data(self):
         connection = self._create_connection()
         try:
@@ -101,10 +100,6 @@ class MySQLUtil(object):
             database=mysql_dsn_data["database"],
         )
 
-    @staticmethod
-    def get_dsn() -> str:
-        return os.environ.get(MySQLUtil.dsn_env_var, "")
-
     @staticmethod
     def get_mysql_dsn_data() -> typing.Optional[dict]:
         match = re.match(MySQLUtil.dsn_regex, MySQLUtil.get_dsn())
@@ -112,3 +107,7 @@ class MySQLUtil(object):
             return None
 
         return match.groupdict()
+
+    @staticmethod
+    def get_dsn() -> str:
+        return os.environ.get(MySQLUtil.dsn_env_var, "")

mlrun/api/utils/db/sql_collation.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/api/utils/db/sqlite_migration.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -64,7 +64,7 @@ class SQLiteMigrationUtil(object):
         self._migrator = self._create_migrator()
         self._mysql_util = None
         if self._mysql_dsn_data:
-            self._mysql_util = MySQLUtil()
+            self._mysql_util = MySQLUtil(logger)
 
     def is_database_migration_needed(self) -> bool:
         # if some data is missing, don't transfer the data

mlrun/api/utils/events/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

mlrun/api/utils/events/base.py

@@ -0,0 +1,57 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import abc
+import typing
+
+import mlrun.common.schemas
+
+
+class BaseEventClient:
+    @abc.abstractmethod
+    def emit(self, event):
+        pass
+
+    def generate_auth_secret_event(
+        self,
+        username: str,
+        secret_name: str,
+        action: mlrun.common.schemas.AuthSecretEventActions,
+    ):
+        """
+        Generate an auth secret event
+        :param username:  username
+        :param secret_name:  secret name
+        :param action: preformed action
+        :return: event object to emit
+        """
+        pass
+
+    @abc.abstractmethod
+    def generate_project_secret_event(
+        self,
+        project: str,
+        secret_name: str,
+        secret_keys: typing.List[str] = None,
+        action: mlrun.common.schemas.SecretEventActions = mlrun.common.schemas.SecretEventActions.created,
+    ):
+        """
+        Generate a project secret event
+        :param project: project name
+        :param secret_name: secret name
+        :param secret_keys: secret keys, optional, only relevant for created/updated events
+        :param action: preformed action
+        :return: event object to emit
+        """
+        pass

mlrun/api/utils/events/events_factory.py

@@ -0,0 +1,41 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import mlrun.api.utils.events.base
+import mlrun.api.utils.events.iguazio
+import mlrun.api.utils.events.nop
+import mlrun.common.schemas
+import mlrun.utils.singleton
+
+
+class EventsFactory(object):
+    @staticmethod
+    def get_events_client(
+        kind: mlrun.common.schemas.EventClientKinds = None, **kwargs
+    ) -> mlrun.api.utils.events.base.BaseEventClient:
+        if mlrun.mlconf.events.mode == mlrun.common.schemas.EventsModes.disabled:
+            return mlrun.api.utils.events.nop.NopClient()
+
+        if not kind:
+            if mlrun.mlconf.get_parsed_igz_version():
+                kind = mlrun.common.schemas.EventClientKinds.iguazio
+
+        if kind == mlrun.common.schemas.EventClientKinds.iguazio:
+            if not mlrun.mlconf.get_parsed_igz_version():
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Iguazio events client can only be used in Iguazio environment"
+                )
+            return mlrun.api.utils.events.iguazio.Client(**kwargs)
+
+        return mlrun.api.utils.events.nop.NopClient()

mlrun/api/utils/events/iguazio.py

@@ -0,0 +1,217 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+
+import igz_mgmt.schemas.events
+
+import mlrun.api.utils.clients.iguazio
+import mlrun.api.utils.events.base as base_events
+import mlrun.common.schemas
+from mlrun.utils import logger
+
+PROJECT_AUTH_SECRET_CREATED = "Security.Project.AuthSecret.Created"
+PROJECT_AUTH_SECRET_UPDATED = "Security.Project.AuthSecret.Updated"
+PROJECT_SECRET_CREATED = "Security.Project.Secret.Created"
+PROJECT_SECRET_UPDATED = "Security.Project.Secret.Updated"
+PROJECT_SECRET_DELETED = "Security.Project.Secret.Deleted"
+
+
+class Client(base_events.BaseEventClient):
+    def __init__(self, access_key: str = None, verbose: bool = None):
+        self.access_key = (
+            access_key
+            or mlrun.mlconf.events.access_key
+            or mlrun.mlconf.get_v3io_access_key()
+        )
+        self.verbose = verbose if verbose is not None else mlrun.mlconf.events.verbose
+        self.source = "mlrun-api"
+
+    def emit(self, event: igz_mgmt.Event):
+        try:
+            logger.debug("Emitting event", event=event)
+            mlrun.api.utils.clients.iguazio.Client().emit_manual_event(
+                self.access_key, event
+            )
+        except Exception as exc:
+            if self.verbose:
+                logger.warning(
+                    "Failed to emit event",
+                    event=event,
+                    exc_info=exc,
+                )
+
+    def generate_auth_secret_event(
+        self,
+        username: str,
+        secret_name: str,
+        action: mlrun.common.schemas.AuthSecretEventActions,
+    ) -> igz_mgmt.Event:
+        """
+        Generate an auth secret event
+        :param username:        username
+        :param secret_name:     secret name
+        :param action:          preformed action
+        :return: event object to emit
+        """
+        if action == mlrun.common.schemas.SecretEventActions.created:
+            return self._generate_auth_secret_created_event(username, secret_name)
+        elif action == mlrun.common.schemas.SecretEventActions.updated:
+            return self._generate_auth_secret_updated_event(username, secret_name)
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(f"Unsupported action {action}")
+
+    def generate_project_secret_event(
+        self,
+        project: str,
+        secret_name: str,
+        secret_keys: typing.List[str] = None,
+        action: mlrun.common.schemas.SecretEventActions = mlrun.common.schemas.SecretEventActions.created,
+    ) -> igz_mgmt.Event:
+        """
+        Generate a project secret event
+        :param project:     project name
+        :param secret_name: secret name
+        :param secret_keys: secret keys, optional, only relevant for created/updated events
+        :param action:      preformed action
+        :return: event object to emit
+        """
+        if action == mlrun.common.schemas.SecretEventActions.created:
+            return self._generate_project_secret_created_event(
+                project, secret_name, secret_keys
+            )
+        elif action == mlrun.common.schemas.SecretEventActions.updated:
+            return self._generate_project_secret_updated_event(
+                project, secret_name, secret_keys
+            )
+        elif action == mlrun.common.schemas.SecretEventActions.deleted:
+            return self._generate_project_secret_deleted_event(project, secret_name)
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(f"Unsupported action {action}")
+
+    def _generate_auth_secret_created_event(
+        self, username: str, secret_name: str
+    ) -> igz_mgmt.Event:
+        return igz_mgmt.Event(
+            source=self.source,
+            kind=PROJECT_AUTH_SECRET_CREATED,
+            description=f"User {username} created secret {secret_name}",
+            parameters_text=[
+                igz_mgmt.schemas.events.ParametersText(name="username", value=username),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_name", value=secret_name
+                ),
+            ],
+            severity=igz_mgmt.constants.EventSeverity.info,
+            classification=igz_mgmt.constants.EventClassification.security,
+            system_event=False,
+            visibility=igz_mgmt.constants.EventVisibility.external,
+        )
+
+    def _generate_auth_secret_updated_event(
+        self, username: str, secret_name: str
+    ) -> igz_mgmt.Event:
+        return igz_mgmt.Event(
+            source=self.source,
+            kind=PROJECT_AUTH_SECRET_UPDATED,
+            description=f"User {username} updated secret {secret_name}",
+            parameters_text=[
+                igz_mgmt.schemas.events.ParametersText(name="username", value=username),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_name", value=secret_name
+                ),
+            ],
+            severity=igz_mgmt.constants.EventSeverity.info,
+            classification=igz_mgmt.constants.EventClassification.security,
+            system_event=False,
+            visibility=igz_mgmt.constants.EventVisibility.external,
+        )
+
+    def _generate_project_secret_created_event(
+        self, project: str, secret_name: str, secret_keys: typing.List[str]
+    ) -> igz_mgmt.Event:
+        normalized_secret_keys = self._list_to_string(secret_keys)
+        return igz_mgmt.Event(
+            source=self.source,
+            kind=PROJECT_SECRET_CREATED,
+            parameters_text=[
+                igz_mgmt.schemas.events.ParametersText(
+                    name="project_name", value=project
+                ),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_name", value=secret_name
+                ),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_keys", value=normalized_secret_keys
+                ),
+            ],
+            description=f"Project {project} secret created",
+            severity=igz_mgmt.constants.EventSeverity.info,
+            classification=igz_mgmt.constants.EventClassification.security,
+            system_event=False,
+            visibility=igz_mgmt.constants.EventVisibility.external,
+        )
+
+    def _generate_project_secret_updated_event(
+        self,
+        project: str,
+        secret_name: str,
+        secret_keys: typing.List[str],
+    ) -> igz_mgmt.Event:
+        normalized_secret_keys = self._list_to_string(secret_keys)
+        return igz_mgmt.Event(
+            source=self.source,
+            kind=PROJECT_SECRET_UPDATED,
+            description=f"Project {project} secret updated",
+            parameters_text=[
+                igz_mgmt.schemas.events.ParametersText(
+                    name="project_name", value=project
+                ),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_name", value=secret_name
+                ),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_keys", value=normalized_secret_keys
+                ),
+            ],
+            severity=igz_mgmt.constants.EventSeverity.info,
+            classification=igz_mgmt.constants.EventClassification.security,
+            system_event=False,
+            visibility=igz_mgmt.constants.EventVisibility.external,
+        )
+
+    def _generate_project_secret_deleted_event(
+        self, project: str, secret_name: str
+    ) -> igz_mgmt.Event:
+        return igz_mgmt.Event(
+            source=self.source,
+            kind=PROJECT_SECRET_DELETED,
+            description=f"Project {project} secret deleted",
+            parameters_text=[
+                igz_mgmt.schemas.events.ParametersText(
+                    name="project_name", value=project
+                ),
+                igz_mgmt.schemas.events.ParametersText(
+                    name="secret_name", value=secret_name
+                ),
+            ],
+            severity=igz_mgmt.constants.EventSeverity.info,
+            classification=igz_mgmt.constants.EventClassification.security,
+            system_event=False,
+            visibility=igz_mgmt.constants.EventVisibility.external,
+        )
+
+    @staticmethod
+    def _list_to_string(list_to_convert: typing.List[str]) -> str:
+        return ", ".join(list_to_convert)

mlrun/api/utils/events/nop.py

@@ -0,0 +1,55 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+
+import mlrun.api.utils.events.base as base_events
+import mlrun.common.schemas
+
+
+class NopClient(base_events.BaseEventClient):
+    def emit(self, event):
+        return
+
+    def generate_auth_secret_event(
+        self,
+        username: str,
+        secret_name: str,
+        action: mlrun.common.schemas.AuthSecretEventActions,
+    ):
+        """
+        Generate an auth secret event
+        :param username:  username
+        :param secret_name:  secret name
+        :param action: preformed action
+        :return: event object to emit
+        """
+        return
+
+    def generate_project_secret_event(
+        self,
+        project: str,
+        secret_name: str,
+        secret_keys: typing.List[str] = None,
+        action: mlrun.common.schemas.SecretEventActions = mlrun.common.schemas.SecretEventActions.created,
+    ):
+        """
+        Generate a project secret event
+        :param project: project name
+        :param secret_name: secret name
+        :param secret_keys: secret keys, optional, only relevant for created/updated events
+        :param action: preformed action
+        :return: event object to emit
+        """
+        pass

mlrun/api/utils/helpers.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,22 +13,12 @@
 # limitations under the License.
 #
 import asyncio
-import enum
 
 import mlrun
-import mlrun.api.schemas
+import mlrun.common.schemas
 from mlrun.utils import logger
 
 
-# TODO: From python 3.11 StrEnum is built-in and this will not be needed
-class StrEnum(str, enum.Enum):
-    def __str__(self):
-        return self.value
-
-    def __repr__(self):
-        return self.value
-
-
 def ensure_running_on_chief(function):
     """
     The motivation of this function is to catch development bugs in which we are accidentally using functions / flows
@@ -41,7 +31,7 @@ def ensure_running_on_chief(function):
     def _ensure_running_on_chief():
         if (
             mlrun.mlconf.httpdb.clusterization.role
-            != mlrun.api.schemas.ClusterizationRole.chief
+            != mlrun.common.schemas.ClusterizationRole.chief
         ):
             if (
                 mlrun.mlconf.httpdb.clusterization.ensure_function_running_on_chief_mode
@@ -67,4 +57,17 @@ def ensure_running_on_chief(function):
 
     if asyncio.iscoroutinefunction(function):
         return async_wrapper
+
+    # ensure method name is preserved
+    wrapper.__name__ = function.__name__
+
     return wrapper
+
+
+def minimize_project_schema(
+    project: mlrun.common.schemas.Project,
+) -> mlrun.common.schemas.Project:
+    project.spec.functions = None
+    project.spec.workflows = None
+    project.spec.artifacts = None
+    return project

mlrun/api/utils/memory_reports.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/api/utils/periodic.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,9 +34,12 @@ async def _periodic_function_wrapper(interval: int, function, *args, **kwargs):
                 await function(*args, **kwargs)
             else:
                 await run_in_threadpool(function, *args, **kwargs)
-        except Exception:
+        except Exception as exc:
             logger.warning(
-                f"Failed during periodic function execution: {function.__name__}, exc: {traceback.format_exc()}"
+                "Failed during periodic function execution",
+                func_name=function.__name__,
+                exc=mlrun.errors.err_to_str(exc),
+                tb=traceback.format_exc(),
             )
         await asyncio.sleep(interval)
 

mlrun/api/utils/projects/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.