mlrun 1.7.1__py3-none-any.whl → 1.7.1rc1__py3-none-any.whl

mlrun/common/formatters/artifact.py

@@ -21,7 +21,6 @@ from .base import ObjectFormat
 
 
 class ArtifactFormat(ObjectFormat, mlrun.common.types.StrEnum):
-    full = "full"
     minimal = "minimal"
 
     @staticmethod

mlrun/common/formatters/feature_set.py

@@ -28,17 +28,6 @@ class FeatureSetFormat(ObjectFormat, mlrun.common.types.StrEnum):
         return {
             FeatureSetFormat.full: None,
             FeatureSetFormat.minimal: FeatureSetFormat.filter_obj_method(
-                [
-                    "metadata.name",
-                    "metadata.project",
-                    "metadata.tag",
-                    "metadata.uid",
-                    "metadata.labels",
-                    "spec.entities",
-                    "spec.description",
-                    "spec.targets",
-                    "spec.engine",  # It's not needed by the UI, but we override it anyway to storey if empty
-                    "status.state",
-                ]
+                ["kind", "metadata", "spec", "status.state"]
             ),
         }[_format]

mlrun/db/httpdb.py

@@ -1075,9 +1075,7 @@ class HTTPRunDB(RunDBInterface):
         category: Union[str, mlrun.common.schemas.ArtifactCategories] = None,
         tree: str = None,
         producer_uri: str = None,
-        format_: Optional[
-            mlrun.common.formatters.ArtifactFormat
-        ] = mlrun.common.formatters.ArtifactFormat.full,
+        format_: mlrun.common.formatters.ArtifactFormat = mlrun.common.formatters.ArtifactFormat.full,
         limit: int = None,
     ) -> ArtifactList:
         """List artifacts filtered by various parameters.

mlrun/model_monitoring/applications/evidently_base.py

@@ -23,7 +23,7 @@ import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
 from mlrun.errors import MLRunIncompatibleVersionError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.39")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.32")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -82,10 +82,9 @@ class TDEngineSchema:
         super_table: str,
         columns: dict[str, _TDEngineColumn],
         tags: dict[str, str],
-        project: str,
         database: Optional[str] = None,
     ):
-        self.super_table = f"{super_table}_{project.replace('-', '_')}"
+        self.super_table = super_table
         self.columns = columns
         self.tags = tags
         self.database = database or _MODEL_MONITORING_DATABASE
@@ -149,9 +148,6 @@ class TDEngineSchema:
     ) -> str:
         return f"DROP TABLE if EXISTS {self.database}.{subtable};"
 
-    def drop_supertable_query(self) -> str:
-        return f"DROP STABLE if EXISTS {self.database}.{self.super_table};"
-
     def _get_subtables_query(
         self,
         values: dict[str, Union[str, int, float, datetime.datetime]],
@@ -231,7 +227,7 @@ class TDEngineSchema:
 
 @dataclass
 class AppResultTable(TDEngineSchema):
-    def __init__(self, project: str, database: Optional[str] = None):
+    def __init__(self, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.APP_RESULTS
         columns = {
             mm_schemas.WriterEvent.END_INFER_TIME: _TDEngineColumn.TIMESTAMP,
@@ -240,23 +236,18 @@ class AppResultTable(TDEngineSchema):
             mm_schemas.ResultData.RESULT_STATUS: _TDEngineColumn.INT,
         }
         tags = {
+            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.APPLICATION_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.ResultData.RESULT_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.ResultData.RESULT_KIND: _TDEngineColumn.INT,
         }
-        super().__init__(
-            super_table=super_table,
-            columns=columns,
-            tags=tags,
-            database=database,
-            project=project,
-        )
+        super().__init__(super_table, columns, tags, database)
 
 
 @dataclass
 class Metrics(TDEngineSchema):
-    def __init__(self, project: str, database: Optional[str] = None):
+    def __init__(self, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.METRICS
         columns = {
             mm_schemas.WriterEvent.END_INFER_TIME: _TDEngineColumn.TIMESTAMP,
@@ -264,22 +255,17 @@ class Metrics(TDEngineSchema):
             mm_schemas.MetricData.METRIC_VALUE: _TDEngineColumn.FLOAT,
         }
         tags = {
+            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.APPLICATION_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.MetricData.METRIC_NAME: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(
-            super_table=super_table,
-            columns=columns,
-            tags=tags,
-            database=database,
-            project=project,
-        )
+        super().__init__(super_table, columns, tags, database)
 
 
 @dataclass
 class Predictions(TDEngineSchema):
-    def __init__(self, project: str, database: Optional[str] = None):
+    def __init__(self, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.PREDICTIONS
         columns = {
             mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
@@ -287,12 +273,7 @@ class Predictions(TDEngineSchema):
             mm_schemas.EventKeyMetrics.CUSTOM_METRICS: _TDEngineColumn.BINARY_10000,
         }
         tags = {
+            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(
-            super_table=super_table,
-            columns=columns,
-            tags=tags,
-            database=database,
-            project=project,
-        )
+        super().__init__(super_table, columns, tags, database)

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -82,13 +82,13 @@ class TDEngineConnector(TSDBConnector):
         """Initialize the super tables for the TSDB."""
         self.tables = {
             mm_schemas.TDEngineSuperTables.APP_RESULTS: tdengine_schemas.AppResultTable(
-                project=self.project, database=self.database
+                self.database
             ),
             mm_schemas.TDEngineSuperTables.METRICS: tdengine_schemas.Metrics(
-                project=self.project, database=self.database
+                self.database
             ),
             mm_schemas.TDEngineSuperTables.PREDICTIONS: tdengine_schemas.Predictions(
-                project=self.project, database=self.database
+                self.database
             ),
         }
 
@@ -112,9 +112,11 @@ class TDEngineConnector(TSDBConnector):
         """
 
         table_name = (
+            f"{self.project}_"
             f"{event[mm_schemas.WriterEvent.ENDPOINT_ID]}_"
-            f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}"
+            f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}_"
         )
+        event[mm_schemas.EventFieldType.PROJECT] = self.project
 
         if kind == mm_schemas.WriterEventKind.RESULT:
             # Write a new result
@@ -185,9 +187,7 @@ class TDEngineConnector(TSDBConnector):
                 name=name,
                 after=after,
                 url=self._tdengine_connection_string,
-                supertable=self.tables[
-                    mm_schemas.TDEngineSuperTables.PREDICTIONS
-                ].super_table,
+                supertable=mm_schemas.TDEngineSuperTables.PREDICTIONS,
                 table_col=mm_schemas.EventFieldType.TABLE_COLUMN,
                 time_col=mm_schemas.EventFieldType.TIME,
                 database=self.database,
@@ -220,23 +220,22 @@ class TDEngineConnector(TSDBConnector):
             "Deleting all project resources using the TDEngine connector",
             project=self.project,
         )
-        drop_statements = []
         for table in self.tables:
-            drop_statements.append(self.tables[table].drop_supertable_query())
-
-        try:
-            self.connection.run(
-                statements=drop_statements,
+            get_subtable_names_query = self.tables[table]._get_subtables_query(
+                values={mm_schemas.EventFieldType.PROJECT: self.project}
+            )
+            subtables = self.connection.run(
+                query=get_subtable_names_query,
                 timeout=self._timeout,
                 retries=self._retries,
-            )
-        except Exception as e:
-            logger.warning(
-                "Failed to drop TDEngine tables. You may need to drop them manually. "
-                "These can be found under the following supertables: app_results, "
-                "metrics, and predictions.",
-                project=self.project,
-                error=mlrun.errors.err_to_str(e),
+            ).data
+            drop_statements = []
+            for subtable in subtables:
+                drop_statements.append(
+                    self.tables[table]._drop_subtable_query(subtable=subtable[0])
+                )
+            self.connection.run(
+                statements=drop_statements, timeout=self._timeout, retries=self._retries
             )
         logger.debug(
             "Deleted all project resources using the TDEngine connector",
@@ -289,6 +288,13 @@ class TDEngineConnector(TSDBConnector):
         :raise:  MLRunInvalidArgumentError if query the provided table failed.
         """
 
+        project_condition = f"project = '{self.project}'"
+        filter_query = (
+            f"({filter_query}) AND ({project_condition})"
+            if filter_query
+            else project_condition
+        )
+
         full_query = tdengine_schemas.TDEngineSchema._get_records_query(
             table=table,
             start=start,
@@ -340,12 +346,12 @@ class TDEngineConnector(TSDBConnector):
         timestamp_column = mm_schemas.WriterEvent.END_INFER_TIME
         columns = [timestamp_column, mm_schemas.WriterEvent.APPLICATION_NAME]
         if type == "metrics":
-            table = self.tables[mm_schemas.TDEngineSuperTables.METRICS].super_table
+            table = mm_schemas.TDEngineSuperTables.METRICS
             name = mm_schemas.MetricData.METRIC_NAME
             columns += [name, mm_schemas.MetricData.METRIC_VALUE]
             df_handler = self.df_to_metrics_values
         elif type == "results":
-            table = self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table
+            table = mm_schemas.TDEngineSuperTables.APP_RESULTS
             name = mm_schemas.ResultData.RESULT_NAME
             columns += [
                 name,
@@ -411,7 +417,7 @@ class TDEngineConnector(TSDBConnector):
                 "both or neither of `aggregation_window` and `agg_funcs` must be provided"
             )
         df = self._get_records(
-            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
+            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.LATENCY],

mlrun/platforms/iguazio.py

@@ -104,32 +104,32 @@ class OutputStream:
         self._mock = mock
         self._mock_queue = []
 
-    def create_stream(self):
-        # this import creates an import loop via the utils module, so putting it in execution path
-        from mlrun.utils.helpers import logger
-
-        logger.debug(
-            "Creating output stream",
-            endpoint=self._endpoint,
-            container=self._container,
-            stream_path=self._stream_path,
-            shards=self._shards,
-            retention_in_hours=self._retention_in_hours,
-        )
-        response = self._v3io_client.stream.create(
-            container=self._container,
-            stream_path=self._stream_path,
-            shard_count=self._shards or 1,
-            retention_period_hours=self._retention_in_hours or 24,
-            raise_for_status=v3io.dataplane.RaiseForStatus.never,
-        )
-        if not (response.status_code == 400 and "ResourceInUse" in str(response.body)):
-            response.raise_for_status([409, 204])
-
     def _lazy_init(self):
         if self._create and not self._mock:
+            # this import creates an import loop via the utils module, so putting it in execution path
+            from mlrun.utils.helpers import logger
+
             self._create = False
-            self.create_stream()
+
+            logger.debug(
+                "Creating output stream",
+                endpoint=self._endpoint,
+                container=self._container,
+                stream_path=self._stream_path,
+                shards=self._shards,
+                retention_in_hours=self._retention_in_hours,
+            )
+            response = self._v3io_client.stream.create(
+                container=self._container,
+                stream_path=self._stream_path,
+                shard_count=self._shards or 1,
+                retention_period_hours=self._retention_in_hours or 24,
+                raise_for_status=v3io.dataplane.RaiseForStatus.never,
+            )
+            if not (
+                response.status_code == 400 and "ResourceInUse" in str(response.body)
+            ):
+                response.raise_for_status([409, 204])
 
     def push(self, data, partition_key=None):
         self._lazy_init()

mlrun/projects/__init__.py

@@ -27,12 +27,7 @@ __all__ = [
 ]
 
 from .operations import build_function, deploy_function, run_function  # noqa
-from .pipelines import (
-    import_remote_project,
-    load_and_run_workflow,
-    load_and_run,
-    pipeline_context,
-)  # noqa
+from .pipelines import load_and_run, pipeline_context  # noqa
 from .project import (
     MlrunProject,
     ProjectMetadata,

mlrun/projects/pipelines.py

@@ -984,25 +984,14 @@ def github_webhook(request):
     return {"msg": "pushed"}
 
 
-def load_and_run(context, *args, **kwargs):
-    """
-    This function serves as an alias to `load_and_run_workflow`,
-    allowing to continue using `load_and_run` without modifying existing workflows or exported runs.
-    This approach ensures backward compatibility,
-    while directing all new calls to the updated `load_and_run_workflow` function.
-    """
-    kwargs.pop("load_only", None)
-    kwargs.pop("save", None)
-    load_and_run_workflow(context, *args, **kwargs)
-
-
-def load_and_run_workflow(
+def load_and_run(
     context: mlrun.execution.MLClientCtx,
     url: str = None,
     project_name: str = "",
     init_git: bool = None,
     subpath: str = None,
     clone: bool = False,
+    save: bool = True,
     workflow_name: str = None,
     workflow_path: str = None,
     workflow_arguments: dict[str, typing.Any] = None,
@@ -1015,12 +1004,14 @@ def load_and_run_workflow(
     local: bool = None,
     schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
     cleanup_ttl: int = None,
+    load_only: bool = False,
     wait_for_completion: bool = False,
     project_context: str = None,
 ):
     """
     Auxiliary function that the RemoteRunner run once or run every schedule.
     This function loads a project from a given remote source and then runs the workflow.
+
     :param context:             mlrun context.
     :param url:                 remote url that represents the project's source.
                                 See 'mlrun.load_project()' for details
@@ -1028,6 +1019,7 @@ def load_and_run_workflow(
     :param init_git:            if True, will git init the context dir
     :param subpath:             project subpath (within the archive)
     :param clone:               if True, always clone (delete any existing content)
+    :param save:                whether to save the created project and artifact in the DB
     :param workflow_name:       name of the workflow
     :param workflow_path:       url to a workflow file, if not a project workflow
     :param workflow_arguments:  kubeflow pipelines arguments (parameters)
@@ -1043,31 +1035,48 @@ def load_and_run_workflow(
     :param schedule:            ScheduleCronTrigger class instance or a standard crontab expression string
     :param cleanup_ttl:         pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
                                 workflow and all its resources are deleted)
+    :param load_only:           for just loading the project, inner use.
     :param wait_for_completion: wait for workflow completion before returning
     :param project_context:     project context path (used for loading the project)
     """
-    project_context = project_context or f"./{project_name}"
-
-    # Load the project to fetch files which the runner needs, such as remote source files
-    pull_remote_project_files(
-        context=context,
-        project_context=project_context,
-        url=url,
-        project_name=project_name,
-        init_git=init_git,
-        subpath=subpath,
-        clone=clone,
-        schedule=schedule,
-        workflow_name=workflow_name,
-    )
+    try:
+        project = mlrun.load_project(
+            context=project_context or f"./{project_name}",
+            url=url,
+            name=project_name,
+            init_git=init_git,
+            subpath=subpath,
+            clone=clone,
+            save=save,
+            sync_functions=True,
+        )
+    except Exception as error:
+        if schedule:
+            notification_pusher = mlrun.utils.notifications.CustomNotificationPusher(
+                ["slack"]
+            )
+            url = get_ui_url(project_name, context.uid)
+            link = f"<{url}|*view workflow job details*>"
+            message = (
+                f":x: Failed to run scheduled workflow {workflow_name} in Project {project_name} !\n"
+                f"error: ```{error}```\n{link}"
+            )
+            # Sending Slack Notification without losing the original error:
+            try:
+                notification_pusher.push(
+                    message=message,
+                    severity=mlrun.common.schemas.NotificationSeverity.ERROR,
+                )
 
-    # Retrieve the project object:
-    # - If the project exists in the MLRun database, it will be loaded from there.
-    # - If it doesn't exist in the database, it will be created from the previously loaded local directory.
-    project = mlrun.get_or_create_project(
-        context=project_context or f"./{project_name}",
-        name=project_name,
-    )
+            except Exception as exc:
+                logger.error("Failed to send slack notification", exc=err_to_str(exc))
+
+        raise error
+
+    context.logger.info(f"Loaded project {project.name} successfully")
+
+    if load_only:
+        return
 
     # extract "start" notification if exists
     start_notifications = [
@@ -1100,156 +1109,18 @@ def load_and_run_workflow(
         raise RuntimeError(f"Workflow {workflow_log_message} failed") from run.exc
 
     if wait_for_completion:
-        handle_workflow_completion(
-            run=run,
-            project=project,
-            context=context,
-            workflow_log_message=workflow_log_message,
-        )
-
-
-def pull_remote_project_files(
-    context: mlrun.execution.MLClientCtx,
-    project_context: str,
-    url: str,
-    project_name: str,
-    init_git: typing.Optional[bool],
-    subpath: typing.Optional[str],
-    clone: bool,
-    schedule: typing.Optional[
-        typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger]
-    ],
-    workflow_name: typing.Optional[str],
-) -> None:
-    """
-    Load the project to clone remote files if they exist.
-    If an exception occurs during project loading, send a notification if the workflow is scheduled.
-
-    :param context:        MLRun execution context.
-    :param project_context: Path to the project context.
-    :param url:            URL of the project repository.
-    :param project_name:   Name of the project.
-    :param init_git:       Initialize a git repository.
-    :param subpath:        Project subpath within the repository.
-    :param clone:          Whether to clone the repository.
-    :param schedule:       Schedule for running the workflow.
-    :param workflow_name:  Name of the workflow to run.
-    """
-    try:
-        # Load the project to clone remote files if they exist.
-        # Using save=False to avoid overriding changes from the database if it already exists.
-        mlrun.load_project(
-            context=project_context,
-            url=url,
-            name=project_name,
-            init_git=init_git,
-            subpath=subpath,
-            clone=clone,
-            save=False,
-        )
-    except Exception as error:
-        notify_scheduled_workflow_failure(
-            schedule=schedule,
-            project_name=project_name,
-            workflow_name=workflow_name,
-            error=error,
-            context_uid=context.uid,
-        )
-        raise error
-
-
-def notify_scheduled_workflow_failure(
-    schedule,
-    project_name: str,
-    workflow_name: str,
-    error: Exception,
-    context_uid: str,
-) -> None:
-    if schedule:
-        notification_pusher = mlrun.utils.notifications.CustomNotificationPusher(
-            ["slack"]
-        )
-        url = get_ui_url(project_name, context_uid)
-        link = f"<{url}|*view workflow job details*>"
-        message = (
-            f":x: Failed to run scheduled workflow {workflow_name} "
-            f"in Project {project_name}!\n"
-            f"Error: ```{err_to_str(error)}```\n{link}"
-        )
-        # Sending Slack Notification without losing the original error:
         try:
-            notification_pusher.push(
-                message=message,
-                severity=mlrun.common.schemas.NotificationSeverity.ERROR,
-            )
-
+            run.wait_for_completion()
         except Exception as exc:
-            logger.error("Failed to send slack notification", exc=err_to_str(exc))
-
-
-def handle_workflow_completion(
-    run: _PipelineRunStatus,
-    project,
-    context: mlrun.execution.MLClientCtx,
-    workflow_log_message: str,
-) -> None:
-    """
-    Handle workflow completion by waiting for it to finish and logging the final state.
-
-    :param run:                 Run object containing workflow execution details.
-    :param project:             MLRun project object.
-    :param context:             MLRun execution context.
-    :param workflow_log_message: Message used for logging.
-    """
-    try:
-        run.wait_for_completion()
-    except Exception as exc:
-        mlrun.utils.logger.error(
-            "Failed waiting for workflow completion",
-            workflow=workflow_log_message,
-            exc=err_to_str(exc),
-        )
-
-    pipeline_state, _, _ = project.get_run_status(run)
-    context.log_result(key="workflow_state", value=pipeline_state, commit=True)
-    if pipeline_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
-        raise RuntimeError(
-            f"Workflow {workflow_log_message} failed, state={pipeline_state}"
-        )
-
-
-def import_remote_project(
-    context: mlrun.execution.MLClientCtx,
-    url: str = None,
-    project_name: str = "",
-    init_git: bool = None,
-    subpath: str = None,
-    clone: bool = False,
-    save: bool = True,
-    project_context: str = None,
-):
-    """
-    This function loads a project from a given remote source.
-
-    :param context:             mlrun context.
-    :param url:                 remote url that represents the project's source.
-                                See 'mlrun.load_project()' for details
-    :param project_name:        project name
-    :param init_git:            if True, will git init the context dir
-    :param subpath:             project subpath (within the archive)
-    :param clone:               if True, always clone (delete any existing content)
-    :param save:                whether to save the created project and artifact in the DB
-    :param project_context:     project context path (used for loading the project)
-    """
-    project = mlrun.load_project(
-        context=project_context or f"./{project_name}",
-        url=url,
-        name=project_name,
-        init_git=init_git,
-        subpath=subpath,
-        clone=clone,
-        save=save,
-        sync_functions=True,
-    )
+            logger.error(
+                "Failed waiting for workflow completion",
+                workflow=workflow_log_message,
+                exc=err_to_str(exc),
+            )
 
-    context.logger.info(f"Loaded project {project.name} successfully")
+        pipeline_state, _, _ = project.get_run_status(run)
+        context.log_result(key="workflow_state", value=pipeline_state, commit=True)
+        if pipeline_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
+            raise RuntimeError(
+                f"Workflow {workflow_log_message} failed, state={pipeline_state}"
+            )