mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc2__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -81,16 +81,16 @@ class TDEngineConnector(TSDBConnector):
         """Initialize the super tables for the TSDB."""
         self.tables = {
             mm_schemas.TDEngineSuperTables.APP_RESULTS: tdengine_schemas.AppResultTable(
-                self.database
+                project=self.project, database=self.database
             ),
             mm_schemas.TDEngineSuperTables.METRICS: tdengine_schemas.Metrics(
-                self.database
+                project=self.project, database=self.database
             ),
             mm_schemas.TDEngineSuperTables.PREDICTIONS: tdengine_schemas.Predictions(
-                self.database
+                project=self.project, database=self.database
             ),
             mm_schemas.TDEngineSuperTables.ERRORS: tdengine_schemas.Errors(
-                self.database
+                project=self.project, database=self.database
             ),
         }
 
@@ -114,11 +114,9 @@ class TDEngineConnector(TSDBConnector):
         """
 
         table_name = (
-            f"{self.project}_"
             f"{event[mm_schemas.WriterEvent.ENDPOINT_ID]}_"
-            f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}_"
+            f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}"
         )
-        event[mm_schemas.EventFieldType.PROJECT] = self.project
 
         if kind == mm_schemas.WriterEventKind.RESULT:
             # Write a new result
@@ -188,7 +186,9 @@ class TDEngineConnector(TSDBConnector):
                 name=name,
                 after=after,
                 url=self._tdengine_connection_string,
-                supertable=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+                supertable=self.tables[
+                    mm_schemas.TDEngineSuperTables.PREDICTIONS
+                ].super_table,
                 table_col=mm_schemas.EventFieldType.TABLE_COLUMN,
                 time_col=mm_schemas.EventFieldType.TIME,
                 database=self.database,
@@ -197,7 +197,6 @@ class TDEngineConnector(TSDBConnector):
                     mm_schemas.EventKeyMetrics.CUSTOM_METRICS,
                 ],
                 tag_cols=[
-                    mm_schemas.EventFieldType.PROJECT,
                     mm_schemas.EventFieldType.ENDPOINT_ID,
                 ],
                 max_events=1000,
@@ -227,7 +226,7 @@ class TDEngineConnector(TSDBConnector):
             name="tsdb_error",
             after="error_extractor",
             url=self._tdengine_connection_string,
-            supertable=mm_schemas.TDEngineSuperTables.ERRORS,
+            supertable=self.tables[mm_schemas.TDEngineSuperTables.ERRORS].super_table,
             table_col=mm_schemas.EventFieldType.TABLE_COLUMN,
             time_col=mm_schemas.EventFieldType.TIME,
             database=self.database,
@@ -235,7 +234,6 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.EventFieldType.MODEL_ERROR,
             ],
             tag_cols=[
-                mm_schemas.EventFieldType.PROJECT,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
                 mm_schemas.EventFieldType.ERROR_TYPE,
             ],
@@ -251,22 +249,23 @@ class TDEngineConnector(TSDBConnector):
             "Deleting all project resources using the TDEngine connector",
             project=self.project,
         )
+        drop_statements = []
         for table in self.tables:
-            get_subtable_names_query = self.tables[table]._get_subtables_query(
-                values={mm_schemas.EventFieldType.PROJECT: self.project}
-            )
-            subtables = self.connection.run(
-                query=get_subtable_names_query,
+            drop_statements.append(self.tables[table].drop_supertable_query())
+
+        try:
+            self.connection.run(
+                statements=drop_statements,
                 timeout=self._timeout,
                 retries=self._retries,
-            ).data
-            drop_statements = []
-            for subtable in subtables:
-                drop_statements.append(
-                    self.tables[table]._drop_subtable_query(subtable=subtable[0])
-                )
-            self.connection.run(
-                statements=drop_statements, timeout=self._timeout, retries=self._retries
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to drop TDEngine tables. You may need to drop them manually. "
+                "These can be found under the following supertables: app_results, "
+                "metrics, and predictions.",
+                project=self.project,
+                error=mlrun.errors.err_to_str(e),
             )
         logger.debug(
             "Deleted all project resources using the TDEngine connector",
@@ -331,13 +330,6 @@ class TDEngineConnector(TSDBConnector):
         :raise:  MLRunInvalidArgumentError if query the provided table failed.
         """
 
-        project_condition = f"project = '{self.project}'"
-        filter_query = (
-            f"({filter_query}) AND ({project_condition})"
-            if filter_query
-            else project_condition
-        )
-
         full_query = tdengine_schemas.TDEngineSchema._get_records_query(
             table=table,
             start=start,
@@ -400,12 +392,12 @@ class TDEngineConnector(TSDBConnector):
                     project=self.project,
                     endpoint_id=endpoint_id,
                 )
-            table = mm_schemas.TDEngineSuperTables.METRICS
+            table = self.tables[mm_schemas.TDEngineSuperTables.METRICS].super_table
             name = mm_schemas.MetricData.METRIC_NAME
             columns += [name, mm_schemas.MetricData.METRIC_VALUE]
             df_handler = self.df_to_metrics_values
         elif type == "results":
-            table = mm_schemas.TDEngineSuperTables.APP_RESULTS
+            table = self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table
             name = mm_schemas.ResultData.RESULT_NAME
             columns += [
                 name,
@@ -477,7 +469,7 @@ class TDEngineConnector(TSDBConnector):
                 "both or neither of `aggregation_window` and `agg_funcs` must be provided"
             )
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.LATENCY],
@@ -527,7 +519,7 @@ class TDEngineConnector(TSDBConnector):
         )
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
             columns=[
@@ -571,7 +563,7 @@ class TDEngineConnector(TSDBConnector):
         start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.APP_RESULTS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table,
             start=start,
             end=end,
             columns=[
@@ -602,7 +594,7 @@ class TDEngineConnector(TSDBConnector):
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.METRICS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.METRICS].super_table,
             start=start,
             end=end,
             columns=[
@@ -638,7 +630,7 @@ class TDEngineConnector(TSDBConnector):
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.APP_RESULTS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table,
             start=start,
             end=end,
             columns=[
@@ -679,7 +671,7 @@ class TDEngineConnector(TSDBConnector):
         )
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.ERRORS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.ERRORS].super_table,
             start=start,
             end=end,
             columns=[
@@ -711,7 +703,7 @@ class TDEngineConnector(TSDBConnector):
         )
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
             columns=[

mlrun/model_monitoring/helpers.py

@@ -20,6 +20,7 @@ import numpy as np
 import pandas as pd
 
 if typing.TYPE_CHECKING:
+    from mlrun.datastore import DataItem
     from mlrun.db.base import RunDBInterface
     from mlrun.projects import MlrunProject
 
@@ -28,7 +29,6 @@ import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
-import mlrun.datastore
 import mlrun.model_monitoring
 import mlrun.utils.helpers
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
@@ -134,9 +134,7 @@ def _get_monitoring_drift_measures_file_path(project: str, endpoint_id: str) ->
     )
 
 
-def get_monitoring_current_stats_data(
-    project: str, endpoint_id: str
-) -> mlrun.datastore.DataItem:
+def get_monitoring_current_stats_data(project: str, endpoint_id: str) -> "DataItem":
     """
     getter for data item of current stats for project and endpoint
     :param project: project name str
@@ -150,9 +148,7 @@ def get_monitoring_current_stats_data(
     )
 
 
-def get_monitoring_drift_measures_data(
-    project: str, endpoint_id: str
-) -> mlrun.datastore.DataItem:
+def get_monitoring_drift_measures_data(project: str, endpoint_id: str) -> "DataItem":
     """
     getter for data item of drift measures for project and endpoint
     :param project: project name str
@@ -437,9 +433,7 @@ def _get_monitoring_schedules_file_path(*, project: str, endpoint_id: str) -> st
     )
 
 
-def get_monitoring_schedules_data(
-    *, project: str, endpoint_id: str
-) -> mlrun.datastore.DataItem:
+def get_monitoring_schedules_data(*, project: str, endpoint_id: str) -> "DataItem":
     """
     Get the model monitoring schedules' data item of the project's model endpoint.
     """

mlrun/model_monitoring/stream_processing.py

@@ -30,6 +30,7 @@ import mlrun.model_monitoring.db
 import mlrun.serving.states
 import mlrun.utils
 from mlrun.common.schemas.model_monitoring.constants import (
+    EndpointType,
     EventFieldType,
     EventKeyMetrics,
     EventLiveStats,
@@ -783,6 +784,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
             if not feature_names and self._infer_columns_from_data:
                 feature_names = self._infer_feature_names_from_data(event)
 
+            endpoint_type = int(endpoint_record.get(EventFieldType.ENDPOINT_TYPE))
             if not feature_names:
                 logger.warn(
                     "Feature names are not initialized, they will be automatically generated",
@@ -801,11 +803,12 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     },
                 )
 
-                update_monitoring_feature_set(
-                    endpoint_record=endpoint_record,
-                    feature_names=feature_names,
-                    feature_values=feature_values,
-                )
+                if endpoint_type != EndpointType.ROUTER.value:
+                    update_monitoring_feature_set(
+                        endpoint_record=endpoint_record,
+                        feature_names=feature_names,
+                        feature_values=feature_values,
+                    )
 
             # Similar process with label columns
             if not label_columns and self._infer_columns_from_data:
@@ -825,11 +828,12 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     endpoint_id=endpoint_id,
                     attributes={EventFieldType.LABEL_NAMES: json.dumps(label_columns)},
                 )
-                update_monitoring_feature_set(
-                    endpoint_record=endpoint_record,
-                    feature_names=label_columns,
-                    feature_values=label_values,
-                )
+                if endpoint_type != EndpointType.ROUTER.value:
+                    update_monitoring_feature_set(
+                        endpoint_record=endpoint_record,
+                        feature_names=label_columns,
+                        feature_values=label_values,
+                    )
 
             self.label_columns[endpoint_id] = label_columns
             self.feature_names[endpoint_id] = feature_names
@@ -842,7 +846,6 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
             )
 
             # Update the endpoint type within the endpoint types dictionary
-            endpoint_type = int(endpoint_record.get(EventFieldType.ENDPOINT_TYPE))
             self.endpoint_type[endpoint_id] = endpoint_type
 
         # Add feature_name:value pairs along with a mapping dictionary of all of these pairs

mlrun/platforms/__init__.py

@@ -17,19 +17,6 @@ from pprint import pprint
 from time import sleep
 from typing import Optional
 
-from mlrun_pipelines.common.mounts import VolumeMount
-from mlrun_pipelines.mounts import (
-    auto_mount,
-    mount_configmap,
-    mount_hostpath,
-    mount_pvc,
-    mount_s3,
-    mount_secret,
-    mount_v3io,
-    set_env_variables,
-    v3io_cred,
-)
-
 from .iguazio import (
     V3ioStreamClient,
     add_or_refresh_credentials,

mlrun/projects/__init__.py

@@ -27,7 +27,12 @@ __all__ = [
 ]
 
 from .operations import build_function, deploy_function, run_function  # noqa
-from .pipelines import load_and_run, pipeline_context  # noqa
+from .pipelines import (
+    import_remote_project,
+    load_and_run_workflow,
+    load_and_run,
+    pipeline_context,
+)  # noqa
 from .project import (
     MlrunProject,
     ProjectMetadata,

mlrun/projects/pipelines.py

@@ -983,14 +983,25 @@ def github_webhook(request):
     return {"msg": "pushed"}
 
 
-def load_and_run(
+def load_and_run(context, *args, **kwargs):
+    """
+    This function serves as an alias to `load_and_run_workflow`,
+    allowing to continue using `load_and_run` without modifying existing workflows or exported runs.
+    This approach ensures backward compatibility,
+    while directing all new calls to the updated `load_and_run_workflow` function.
+    """
+    kwargs.pop("load_only", None)
+    kwargs.pop("save", None)
+    load_and_run_workflow(context, *args, **kwargs)
+
+
+def load_and_run_workflow(
     context: mlrun.execution.MLClientCtx,
     url: typing.Optional[str] = None,
     project_name: str = "",
     init_git: typing.Optional[bool] = None,
     subpath: typing.Optional[str] = None,
     clone: bool = False,
-    save: bool = True,
     workflow_name: typing.Optional[str] = None,
     workflow_path: typing.Optional[str] = None,
     workflow_arguments: typing.Optional[dict[str, typing.Any]] = None,
@@ -1003,14 +1014,12 @@ def load_and_run(
     local: typing.Optional[bool] = None,
     schedule: typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger] = None,
     cleanup_ttl: typing.Optional[int] = None,
-    load_only: bool = False,
     wait_for_completion: bool = False,
     project_context: typing.Optional[str] = None,
 ):
     """
     Auxiliary function that the RemoteRunner run once or run every schedule.
     This function loads a project from a given remote source and then runs the workflow.
-
     :param context:             mlrun context.
     :param url:                 remote url that represents the project's source.
                                 See 'mlrun.load_project()' for details
@@ -1018,7 +1027,6 @@ def load_and_run(
     :param init_git:            if True, will git init the context dir
     :param subpath:             project subpath (within the archive)
     :param clone:               if True, always clone (delete any existing content)
-    :param save:                whether to save the created project and artifact in the DB
     :param workflow_name:       name of the workflow
     :param workflow_path:       url to a workflow file, if not a project workflow
     :param workflow_arguments:  kubeflow pipelines arguments (parameters)
@@ -1034,48 +1042,31 @@ def load_and_run(
     :param schedule:            ScheduleCronTrigger class instance or a standard crontab expression string
     :param cleanup_ttl:         pipeline cleanup ttl in secs (time to wait after workflow completion, at which point the
                                 workflow and all its resources are deleted)
-    :param load_only:           for just loading the project, inner use.
     :param wait_for_completion: wait for workflow completion before returning
     :param project_context:     project context path (used for loading the project)
     """
-    try:
-        project = mlrun.load_project(
-            context=project_context or f"./{project_name}",
-            url=url,
-            name=project_name,
-            init_git=init_git,
-            subpath=subpath,
-            clone=clone,
-            save=save,
-            sync_functions=True,
-        )
-    except Exception as error:
-        if schedule:
-            notification_pusher = mlrun.utils.notifications.CustomNotificationPusher(
-                ["slack"]
-            )
-            url = get_ui_url(project_name, context.uid)
-            link = f"<{url}|*view workflow job details*>"
-            message = (
-                f":x: Failed to run scheduled workflow {workflow_name} in Project {project_name} !\n"
-                f"error: ```{error}```\n{link}"
-            )
-            # Sending Slack Notification without losing the original error:
-            try:
-                notification_pusher.push(
-                    message=message,
-                    severity=mlrun.common.schemas.NotificationSeverity.ERROR,
-                )
-
-            except Exception as exc:
-                logger.error("Failed to send slack notification", exc=err_to_str(exc))
-
-        raise error
-
-    context.logger.info(f"Loaded project {project.name} successfully")
+    project_context = project_context or f"./{project_name}"
+
+    # Load the project to fetch files which the runner needs, such as remote source files
+    pull_remote_project_files(
+        context=context,
+        project_context=project_context,
+        url=url,
+        project_name=project_name,
+        init_git=init_git,
+        subpath=subpath,
+        clone=clone,
+        schedule=schedule,
+        workflow_name=workflow_name,
+    )
 
-    if load_only:
-        return
+    # Retrieve the project object:
+    # - If the project exists in the MLRun database, it will be loaded from there.
+    # - If it doesn't exist in the database, it will be created from the previously loaded local directory.
+    project = mlrun.get_or_create_project(
+        context=project_context or f"./{project_name}",
+        name=project_name,
+    )
 
     # extract "start" notification if exists
     start_notifications = [
@@ -1108,18 +1099,156 @@ def load_and_run(
         raise RuntimeError(f"Workflow {workflow_log_message} failed") from run.exc
 
     if wait_for_completion:
+        handle_workflow_completion(
+            run=run,
+            project=project,
+            context=context,
+            workflow_log_message=workflow_log_message,
+        )
+
+
+def pull_remote_project_files(
+    context: mlrun.execution.MLClientCtx,
+    project_context: str,
+    url: str,
+    project_name: str,
+    init_git: typing.Optional[bool],
+    subpath: typing.Optional[str],
+    clone: bool,
+    schedule: typing.Optional[
+        typing.Union[str, mlrun.common.schemas.ScheduleCronTrigger]
+    ],
+    workflow_name: typing.Optional[str],
+) -> None:
+    """
+    Load the project to clone remote files if they exist.
+    If an exception occurs during project loading, send a notification if the workflow is scheduled.
+
+    :param context:        MLRun execution context.
+    :param project_context: Path to the project context.
+    :param url:            URL of the project repository.
+    :param project_name:   Name of the project.
+    :param init_git:       Initialize a git repository.
+    :param subpath:        Project subpath within the repository.
+    :param clone:          Whether to clone the repository.
+    :param schedule:       Schedule for running the workflow.
+    :param workflow_name:  Name of the workflow to run.
+    """
+    try:
+        # Load the project to clone remote files if they exist.
+        # Using save=False to avoid overriding changes from the database if it already exists.
+        mlrun.load_project(
+            context=project_context,
+            url=url,
+            name=project_name,
+            init_git=init_git,
+            subpath=subpath,
+            clone=clone,
+            save=False,
+        )
+    except Exception as error:
+        notify_scheduled_workflow_failure(
+            schedule=schedule,
+            project_name=project_name,
+            workflow_name=workflow_name,
+            error=error,
+            context_uid=context.uid,
+        )
+        raise error
+
+
+def notify_scheduled_workflow_failure(
+    schedule,
+    project_name: str,
+    workflow_name: str,
+    error: Exception,
+    context_uid: str,
+) -> None:
+    if schedule:
+        notification_pusher = mlrun.utils.notifications.CustomNotificationPusher(
+            ["slack"]
+        )
+        url = get_ui_url(project_name, context_uid)
+        link = f"<{url}|*view workflow job details*>"
+        message = (
+            f":x: Failed to run scheduled workflow {workflow_name} "
+            f"in Project {project_name}!\n"
+            f"Error: ```{err_to_str(error)}```\n{link}"
+        )
+        # Sending Slack Notification without losing the original error:
         try:
-            run.wait_for_completion()
-        except Exception as exc:
-            logger.error(
-                "Failed waiting for workflow completion",
-                workflow=workflow_log_message,
-                exc=err_to_str(exc),
+            notification_pusher.push(
+                message=message,
+                severity=mlrun.common.schemas.NotificationSeverity.ERROR,
             )
 
-        pipeline_state, _, _ = project.get_run_status(run)
-        context.log_result(key="workflow_state", value=pipeline_state, commit=True)
-        if pipeline_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
-            raise RuntimeError(
-                f"Workflow {workflow_log_message} failed, state={pipeline_state}"
-            )
+        except Exception as exc:
+            logger.error("Failed to send slack notification", exc=err_to_str(exc))
+
+
+def handle_workflow_completion(
+    run: _PipelineRunStatus,
+    project,
+    context: mlrun.execution.MLClientCtx,
+    workflow_log_message: str,
+) -> None:
+    """
+    Handle workflow completion by waiting for it to finish and logging the final state.
+
+    :param run:                 Run object containing workflow execution details.
+    :param project:             MLRun project object.
+    :param context:             MLRun execution context.
+    :param workflow_log_message: Message used for logging.
+    """
+    try:
+        run.wait_for_completion()
+    except Exception as exc:
+        mlrun.utils.logger.error(
+            "Failed waiting for workflow completion",
+            workflow=workflow_log_message,
+            exc=err_to_str(exc),
+        )
+
+    pipeline_state, _, _ = project.get_run_status(run)
+    context.log_result(key="workflow_state", value=pipeline_state, commit=True)
+    if pipeline_state != mlrun_pipelines.common.models.RunStatuses.succeeded:
+        raise RuntimeError(
+            f"Workflow {workflow_log_message} failed, state={pipeline_state}"
+        )
+
+
+def import_remote_project(
+    context: mlrun.execution.MLClientCtx,
+    url: typing.Optional[str] = None,
+    project_name: str = "",
+    init_git: typing.Optional[bool] = None,
+    subpath: typing.Optional[str] = None,
+    clone: bool = False,
+    save: bool = True,
+    project_context: typing.Optional[str] = None,
+):
+    """
+    This function loads a project from a given remote source.
+
+    :param context:             mlrun context.
+    :param url:                 remote url that represents the project's source.
+                                See 'mlrun.load_project()' for details
+    :param project_name:        project name
+    :param init_git:            if True, will git init the context dir
+    :param subpath:             project subpath (within the archive)
+    :param clone:               if True, always clone (delete any existing content)
+    :param save:                whether to save the created project and artifact in the DB
+    :param project_context:     project context path (used for loading the project)
+    """
+    project = mlrun.load_project(
+        context=project_context or f"./{project_name}",
+        url=url,
+        name=project_name,
+        init_git=init_git,
+        subpath=subpath,
+        clone=clone,
+        save=save,
+        sync_functions=True,
+    )
+
+    context.logger.info(f"Loaded project {project.name} successfully")