mlrun 1.8.0rc1__py3-none-any.whl → 1.8.0rc3__py3-none-any.whl

mlrun/model.py

@@ -24,7 +24,7 @@ from datetime import datetime
 from os import environ
 from typing import Any, Optional, Union
 
-import pydantic.error_wrappers
+import pydantic.v1.error_wrappers
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
@@ -739,7 +739,7 @@ class Notification(ModelObj):
     def validate_notification(self):
         try:
             mlrun.common.schemas.notification.Notification(**self.to_dict())
-        except pydantic.error_wrappers.ValidationError as exc:
+        except pydantic.v1.error_wrappers.ValidationError as exc:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "Invalid notification object"
             ) from exc

mlrun/model_monitoring/applications/results.py

@@ -17,8 +17,8 @@ import json
 import re
 from abc import ABC, abstractmethod
 
-from pydantic import validator
-from pydantic.dataclasses import dataclass
+from pydantic.v1 import validator
+from pydantic.v1.dataclasses import dataclass
 
 import mlrun.common.helpers
 import mlrun.common.model_monitoring.helpers

mlrun/model_monitoring/db/tsdb/base.py

@@ -17,7 +17,7 @@ from abc import ABC, abstractmethod
 from datetime import datetime
 
 import pandas as pd
-import pydantic
+import pydantic.v1
 
 import mlrun.common.schemas.model_monitoring as mm_schemas
 import mlrun.model_monitoring.db.tsdb.helpers
@@ -427,7 +427,7 @@ class TSDBConnector(ABC):
                         ),  # pyright: ignore[reportArgumentType]
                     )
                 )
-            except pydantic.ValidationError:
+            except pydantic.v1.ValidationError:
                 logger.exception(
                     "Failed to convert data-frame into `ModelEndpointMonitoringResultValues`",
                     full_name=full_name,

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -82,9 +82,10 @@ class TDEngineSchema:
         super_table: str,
         columns: dict[str, _TDEngineColumn],
         tags: dict[str, str],
+        project: str,
         database: Optional[str] = None,
     ):
-        self.super_table = super_table
+        self.super_table = f"{super_table}_{project.replace('-', '_')}"
         self.columns = columns
         self.tags = tags
         self.database = database or _MODEL_MONITORING_DATABASE
@@ -148,6 +149,9 @@ class TDEngineSchema:
     ) -> str:
         return f"DROP TABLE if EXISTS {self.database}.{subtable};"
 
+    def drop_supertable_query(self) -> str:
+        return f"DROP STABLE if EXISTS {self.database}.{self.super_table};"
+
     def _get_subtables_query(
         self,
         values: dict[str, Union[str, int, float, datetime.datetime]],
@@ -260,7 +264,7 @@ class TDEngineSchema:
 
 @dataclass
 class AppResultTable(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.APP_RESULTS
         columns = {
             mm_schemas.WriterEvent.END_INFER_TIME: _TDEngineColumn.TIMESTAMP,
@@ -270,18 +274,23 @@ class AppResultTable(TDEngineSchema):
             mm_schemas.ResultData.RESULT_EXTRA_DATA: _TDEngineColumn.BINARY_1000,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.APPLICATION_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.ResultData.RESULT_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.ResultData.RESULT_KIND: _TDEngineColumn.INT,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )
 
 
 @dataclass
 class Metrics(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.METRICS
         columns = {
             mm_schemas.WriterEvent.END_INFER_TIME: _TDEngineColumn.TIMESTAMP,
@@ -289,17 +298,22 @@ class Metrics(TDEngineSchema):
             mm_schemas.MetricData.METRIC_VALUE: _TDEngineColumn.FLOAT,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.APPLICATION_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.MetricData.METRIC_NAME: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )
 
 
 @dataclass
 class Predictions(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.PREDICTIONS
         columns = {
             mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
@@ -307,23 +321,33 @@ class Predictions(TDEngineSchema):
             mm_schemas.EventKeyMetrics.CUSTOM_METRICS: _TDEngineColumn.BINARY_1000,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )
 
 
 @dataclass
 class Errors(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.ERRORS
         columns = {
             mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
             mm_schemas.EventFieldType.MODEL_ERROR: _TDEngineColumn.BINARY_1000,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.EventFieldType.ERROR_TYPE: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -81,16 +81,16 @@ class TDEngineConnector(TSDBConnector):
         """Initialize the super tables for the TSDB."""
         self.tables = {
             mm_schemas.TDEngineSuperTables.APP_RESULTS: tdengine_schemas.AppResultTable(
-                self.database
+                project=self.project, database=self.database
             ),
             mm_schemas.TDEngineSuperTables.METRICS: tdengine_schemas.Metrics(
-                self.database
+                project=self.project, database=self.database
             ),
             mm_schemas.TDEngineSuperTables.PREDICTIONS: tdengine_schemas.Predictions(
-                self.database
+                project=self.project, database=self.database
             ),
             mm_schemas.TDEngineSuperTables.ERRORS: tdengine_schemas.Errors(
-                self.database
+                project=self.project, database=self.database
             ),
         }
 
@@ -114,11 +114,9 @@ class TDEngineConnector(TSDBConnector):
         """
 
         table_name = (
-            f"{self.project}_"
             f"{event[mm_schemas.WriterEvent.ENDPOINT_ID]}_"
-            f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}_"
+            f"{event[mm_schemas.WriterEvent.APPLICATION_NAME]}"
         )
-        event[mm_schemas.EventFieldType.PROJECT] = self.project
 
         if kind == mm_schemas.WriterEventKind.RESULT:
             # Write a new result
@@ -188,7 +186,9 @@ class TDEngineConnector(TSDBConnector):
                 name=name,
                 after=after,
                 url=self._tdengine_connection_string,
-                supertable=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+                supertable=self.tables[
+                    mm_schemas.TDEngineSuperTables.PREDICTIONS
+                ].super_table,
                 table_col=mm_schemas.EventFieldType.TABLE_COLUMN,
                 time_col=mm_schemas.EventFieldType.TIME,
                 database=self.database,
@@ -197,7 +197,6 @@ class TDEngineConnector(TSDBConnector):
                     mm_schemas.EventKeyMetrics.CUSTOM_METRICS,
                 ],
                 tag_cols=[
-                    mm_schemas.EventFieldType.PROJECT,
                     mm_schemas.EventFieldType.ENDPOINT_ID,
                 ],
                 max_events=1000,
@@ -227,7 +226,7 @@ class TDEngineConnector(TSDBConnector):
             name="tsdb_error",
             after="error_extractor",
             url=self._tdengine_connection_string,
-            supertable=mm_schemas.TDEngineSuperTables.ERRORS,
+            supertable=self.tables[mm_schemas.TDEngineSuperTables.ERRORS].super_table,
             table_col=mm_schemas.EventFieldType.TABLE_COLUMN,
             time_col=mm_schemas.EventFieldType.TIME,
             database=self.database,
@@ -235,7 +234,6 @@ class TDEngineConnector(TSDBConnector):
                 mm_schemas.EventFieldType.MODEL_ERROR,
             ],
             tag_cols=[
-                mm_schemas.EventFieldType.PROJECT,
                 mm_schemas.EventFieldType.ENDPOINT_ID,
                 mm_schemas.EventFieldType.ERROR_TYPE,
             ],
@@ -251,22 +249,23 @@ class TDEngineConnector(TSDBConnector):
             "Deleting all project resources using the TDEngine connector",
             project=self.project,
         )
+        drop_statements = []
         for table in self.tables:
-            get_subtable_names_query = self.tables[table]._get_subtables_query(
-                values={mm_schemas.EventFieldType.PROJECT: self.project}
-            )
-            subtables = self.connection.run(
-                query=get_subtable_names_query,
+            drop_statements.append(self.tables[table].drop_supertable_query())
+
+        try:
+            self.connection.run(
+                statements=drop_statements,
                 timeout=self._timeout,
                 retries=self._retries,
-            ).data
-            drop_statements = []
-            for subtable in subtables:
-                drop_statements.append(
-                    self.tables[table]._drop_subtable_query(subtable=subtable[0])
-                )
-            self.connection.run(
-                statements=drop_statements, timeout=self._timeout, retries=self._retries
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to drop TDEngine tables. You may need to drop them manually. "
+                "These can be found under the following supertables: app_results, "
+                "metrics, and predictions.",
+                project=self.project,
+                error=mlrun.errors.err_to_str(e),
             )
         logger.debug(
             "Deleted all project resources using the TDEngine connector",
@@ -331,13 +330,6 @@ class TDEngineConnector(TSDBConnector):
         :raise:  MLRunInvalidArgumentError if query the provided table failed.
         """
 
-        project_condition = f"project = '{self.project}'"
-        filter_query = (
-            f"({filter_query}) AND ({project_condition})"
-            if filter_query
-            else project_condition
-        )
-
         full_query = tdengine_schemas.TDEngineSchema._get_records_query(
             table=table,
             start=start,
@@ -400,12 +392,12 @@ class TDEngineConnector(TSDBConnector):
                     project=self.project,
                     endpoint_id=endpoint_id,
                 )
-            table = mm_schemas.TDEngineSuperTables.METRICS
+            table = self.tables[mm_schemas.TDEngineSuperTables.METRICS].super_table
             name = mm_schemas.MetricData.METRIC_NAME
             columns += [name, mm_schemas.MetricData.METRIC_VALUE]
             df_handler = self.df_to_metrics_values
         elif type == "results":
-            table = mm_schemas.TDEngineSuperTables.APP_RESULTS
+            table = self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table
             name = mm_schemas.ResultData.RESULT_NAME
             columns += [
                 name,
@@ -477,7 +469,7 @@ class TDEngineConnector(TSDBConnector):
                 "both or neither of `aggregation_window` and `agg_funcs` must be provided"
             )
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
             columns=[mm_schemas.EventFieldType.LATENCY],
@@ -527,7 +519,7 @@ class TDEngineConnector(TSDBConnector):
         )
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
             columns=[
@@ -571,7 +563,7 @@ class TDEngineConnector(TSDBConnector):
         start = start or (mlrun.utils.datetime_now() - timedelta(hours=24))
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.APP_RESULTS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table,
             start=start,
             end=end,
             columns=[
@@ -602,7 +594,7 @@ class TDEngineConnector(TSDBConnector):
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.METRICS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.METRICS].super_table,
             start=start,
             end=end,
             columns=[
@@ -638,7 +630,7 @@ class TDEngineConnector(TSDBConnector):
     ) -> pd.DataFrame:
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.APP_RESULTS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.APP_RESULTS].super_table,
             start=start,
             end=end,
             columns=[
@@ -679,7 +671,7 @@ class TDEngineConnector(TSDBConnector):
         )
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.ERRORS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.ERRORS].super_table,
             start=start,
             end=end,
             columns=[
@@ -711,7 +703,7 @@ class TDEngineConnector(TSDBConnector):
         )
         start, end = self._get_start_end(start, end)
         df = self._get_records(
-            table=mm_schemas.TDEngineSuperTables.PREDICTIONS,
+            table=self.tables[mm_schemas.TDEngineSuperTables.PREDICTIONS].super_table,
             start=start,
             end=end,
             columns=[

mlrun/model_monitoring/helpers.py

@@ -20,6 +20,7 @@ import numpy as np
 import pandas as pd
 
 if typing.TYPE_CHECKING:
+    from mlrun.datastore import DataItem
     from mlrun.db.base import RunDBInterface
     from mlrun.projects import MlrunProject
 
@@ -28,7 +29,6 @@ import mlrun.artifacts
 import mlrun.common.model_monitoring.helpers
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
-import mlrun.datastore
 import mlrun.model_monitoring
 import mlrun.utils.helpers
 from mlrun.common.schemas.model_monitoring.model_endpoints import (
@@ -134,9 +134,7 @@ def _get_monitoring_drift_measures_file_path(project: str, endpoint_id: str) ->
     )
 
 
-def get_monitoring_current_stats_data(
-    project: str, endpoint_id: str
-) -> mlrun.datastore.DataItem:
+def get_monitoring_current_stats_data(project: str, endpoint_id: str) -> "DataItem":
     """
     getter for data item of current stats for project and endpoint
     :param project: project name str
@@ -150,9 +148,7 @@ def get_monitoring_current_stats_data(
     )
 
 
-def get_monitoring_drift_measures_data(
-    project: str, endpoint_id: str
-) -> mlrun.datastore.DataItem:
+def get_monitoring_drift_measures_data(project: str, endpoint_id: str) -> "DataItem":
     """
     getter for data item of drift measures for project and endpoint
     :param project: project name str
@@ -437,9 +433,7 @@ def _get_monitoring_schedules_file_path(*, project: str, endpoint_id: str) -> st
     )
 
 
-def get_monitoring_schedules_data(
-    *, project: str, endpoint_id: str
-) -> mlrun.datastore.DataItem:
+def get_monitoring_schedules_data(*, project: str, endpoint_id: str) -> "DataItem":
     """
     Get the model monitoring schedules' data item of the project's model endpoint.
     """

mlrun/model_monitoring/stream_processing.py

@@ -30,6 +30,7 @@ import mlrun.model_monitoring.db
 import mlrun.serving.states
 import mlrun.utils
 from mlrun.common.schemas.model_monitoring.constants import (
+    EndpointType,
     EventFieldType,
     EventKeyMetrics,
     EventLiveStats,
@@ -783,6 +784,7 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
             if not feature_names and self._infer_columns_from_data:
                 feature_names = self._infer_feature_names_from_data(event)
 
+            endpoint_type = int(endpoint_record.get(EventFieldType.ENDPOINT_TYPE))
             if not feature_names:
                 logger.warn(
                     "Feature names are not initialized, they will be automatically generated",
@@ -801,11 +803,12 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     },
                 )
 
-                update_monitoring_feature_set(
-                    endpoint_record=endpoint_record,
-                    feature_names=feature_names,
-                    feature_values=feature_values,
-                )
+                if endpoint_type != EndpointType.ROUTER.value:
+                    update_monitoring_feature_set(
+                        endpoint_record=endpoint_record,
+                        feature_names=feature_names,
+                        feature_values=feature_values,
+                    )
 
             # Similar process with label columns
             if not label_columns and self._infer_columns_from_data:
@@ -825,11 +828,12 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
                     endpoint_id=endpoint_id,
                     attributes={EventFieldType.LABEL_NAMES: json.dumps(label_columns)},
                 )
-                update_monitoring_feature_set(
-                    endpoint_record=endpoint_record,
-                    feature_names=label_columns,
-                    feature_values=label_values,
-                )
+                if endpoint_type != EndpointType.ROUTER.value:
+                    update_monitoring_feature_set(
+                        endpoint_record=endpoint_record,
+                        feature_names=label_columns,
+                        feature_values=label_values,
+                    )
 
             self.label_columns[endpoint_id] = label_columns
             self.feature_names[endpoint_id] = feature_names
@@ -842,7 +846,6 @@ class MapFeatureNames(mlrun.feature_store.steps.MapClass):
             )
 
             # Update the endpoint type within the endpoint types dictionary
-            endpoint_type = int(endpoint_record.get(EventFieldType.ENDPOINT_TYPE))
             self.endpoint_type[endpoint_id] = endpoint_type
 
         # Add feature_name:value pairs along with a mapping dictionary of all of these pairs

mlrun/platforms/__init__.py

@@ -13,23 +13,11 @@
 # limitations under the License.
 
 import json
+import warnings
 from pprint import pprint
 from time import sleep
 from typing import Optional
 
-from mlrun_pipelines.common.mounts import VolumeMount
-from mlrun_pipelines.mounts import (
-    auto_mount,
-    mount_configmap,
-    mount_hostpath,
-    mount_pvc,
-    mount_s3,
-    mount_secret,
-    mount_v3io,
-    set_env_variables,
-    v3io_cred,
-)
-
 from .iguazio import (
     V3ioStreamClient,
     add_or_refresh_credentials,
@@ -37,6 +25,49 @@ from .iguazio import (
 )
 
 
+class _DeprecationHelper:
+    """A helper class to deprecate old schemas"""
+
+    def __init__(self, new_target: str, version="1.8.0"):
+        self._new_target = new_target
+        self._version = version
+
+    def __call__(self, *args, **kwargs):
+        self._warn()
+        return self._lazy_load()(*args, **kwargs)
+
+    def __getattr__(self, attr):
+        self._warn()
+        return getattr(self._lazy_load(), attr)
+
+    def _lazy_load(self, *args, **kwargs):
+        import mlrun.runtimes.mounts as mlrun_mounts
+
+        return getattr(mlrun_mounts, self._new_target)
+
+    def _warn(self):
+        warnings.warn(
+            f"mlrun.platforms.{self._new_target} is deprecated since version {self._version}, "
+            f"and will be removed in 1.10. Use mlrun.runtimes.mounts.{self._new_target} instead.",
+            FutureWarning,
+        )
+
+
+# TODO: Remove in 1.10
+# For backwards compatibility
+VolumeMount = _DeprecationHelper("VolumeMount")
+auto_mount = _DeprecationHelper("auto_mount")
+mount_configmap = _DeprecationHelper("mount_configmap")
+mount_hostpath = _DeprecationHelper("mount_hostpath")
+mount_pvc = _DeprecationHelper("mount_pvc")
+mount_s3 = _DeprecationHelper("mount_s3")
+mount_secret = _DeprecationHelper("mount_secret")
+mount_v3io = _DeprecationHelper("mount_v3io")
+set_env_variables = _DeprecationHelper("set_env_variables")
+v3io_cred = _DeprecationHelper("v3io_cred")
+# eof 'For backwards compatibility'
+
+
 def watch_stream(
     url,
     shard_ids: Optional[list] = None,

mlrun/projects/__init__.py

@@ -27,7 +27,12 @@ __all__ = [
 ]
 
 from .operations import build_function, deploy_function, run_function  # noqa
-from .pipelines import load_and_run, pipeline_context  # noqa
+from .pipelines import (
+    import_remote_project,
+    load_and_run_workflow,
+    load_and_run,
+    pipeline_context,
+)  # noqa
 from .project import (
     MlrunProject,
     ProjectMetadata,