mlrun 1.7.1rc4__py3-none-any.whl → 1.8.0rc8__py3-none-any.whl

mlrun/model_monitoring/db/tsdb/tdengine/schemas.py

@@ -26,7 +26,7 @@ _MODEL_MONITORING_DATABASE = "mlrun_model_monitoring"
 
 
 class _TDEngineColumnType:
-    def __init__(self, data_type: str, length: int = None):
+    def __init__(self, data_type: str, length: Optional[int] = None):
         self.data_type = data_type
         self.length = length
 
@@ -46,7 +46,7 @@ class _TDEngineColumn(mlrun.common.types.StrEnum):
     INT = _TDEngineColumnType("INT")
     BINARY_40 = _TDEngineColumnType("BINARY", 40)
     BINARY_64 = _TDEngineColumnType("BINARY", 64)
-    BINARY_10000 = _TDEngineColumnType("BINARY", 10000)
+    BINARY_1000 = _TDEngineColumnType("BINARY", 1000)
 
 
 def values_to_column(values, column_type):
@@ -61,7 +61,7 @@ def values_to_column(values, column_type):
         return taosws.binary_to_column(values)
     if column_type == _TDEngineColumn.BINARY_64:
         return taosws.binary_to_column(values)
-    if column_type == _TDEngineColumn.BINARY_10000:
+    if column_type == _TDEngineColumn.BINARY_1000:
         return taosws.binary_to_column(values)
 
     raise mlrun.errors.MLRunInvalidArgumentError(
@@ -82,9 +82,10 @@ class TDEngineSchema:
         super_table: str,
         columns: dict[str, _TDEngineColumn],
         tags: dict[str, str],
+        project: str,
         database: Optional[str] = None,
     ):
-        self.super_table = super_table
+        self.super_table = f"{super_table}_{project.replace('-', '_')}"
         self.columns = columns
         self.tags = tags
         self.database = database or _MODEL_MONITORING_DATABASE
@@ -148,6 +149,9 @@ class TDEngineSchema:
     ) -> str:
         return f"DROP TABLE if EXISTS {self.database}.{subtable};"
 
+    def drop_supertable_query(self) -> str:
+        return f"DROP STABLE if EXISTS {self.database}.{self.super_table};"
+
     def _get_subtables_query(
         self,
         values: dict[str, Union[str, int, float, datetime.datetime]],
@@ -166,7 +170,7 @@ class TDEngineSchema:
         table: str,
         start: datetime.datetime,
         end: datetime.datetime,
-        columns_to_filter: list[str] = None,
+        columns_to_filter: Optional[list[str]] = None,
         filter_query: Optional[str] = None,
         interval: Optional[str] = None,
         limit: int = 0,
@@ -174,6 +178,10 @@ class TDEngineSchema:
         sliding_window_step: Optional[str] = None,
         timestamp_column: str = "time",
         database: str = _MODEL_MONITORING_DATABASE,
+        group_by: Optional[Union[list[str], str]] = None,
+        preform_agg_funcs_columns: Optional[list[str]] = None,
+        order_by: Optional[str] = None,
+        desc: Optional[bool] = None,
     ) -> str:
         if agg_funcs and not columns_to_filter:
             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -190,15 +198,37 @@ class TDEngineSchema:
             raise mlrun.errors.MLRunInvalidArgumentError(
                 "`interval` must be provided when using sliding window"
             )
+        if group_by and not agg_funcs:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "aggregate functions must be provided when using group by"
+            )
+        if desc and not order_by:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "`order_by` must be provided when using descending"
+            )
 
         with StringIO() as query:
             query.write("SELECT ")
             if interval:
                 query.write("_wstart, _wend, ")
             if agg_funcs:
+                preform_agg_funcs_columns = (
+                    columns_to_filter
+                    if preform_agg_funcs_columns is None
+                    else preform_agg_funcs_columns
+                )
                 query.write(
                     ", ".join(
-                        [f"{a}({col})" for a in agg_funcs for col in columns_to_filter]
+                        [
+                            f"{a}({col})"
+                            if col.upper()
+                            in map(
+                                str.upper, preform_agg_funcs_columns
+                            )  # Case-insensitive check
+                            else f"{col}"
+                            for a in agg_funcs
+                            for col in columns_to_filter
+                        ]
                     )
                 )
             elif columns_to_filter:
@@ -215,6 +245,13 @@ class TDEngineSchema:
                     query.write(f"{timestamp_column} >= '{start}' AND ")
                 if end:
                     query.write(f"{timestamp_column} <= '{end}'")
+            if group_by:
+                if isinstance(group_by, list):
+                    group_by = ", ".join(group_by)
+                query.write(f" GROUP BY {group_by}")
+            if order_by:
+                desc = " DESC" if desc else ""
+                query.write(f" ORDER BY {order_by}{desc}")
             if interval:
                 query.write(f" INTERVAL({interval})")
             if sliding_window_step:
@@ -227,27 +264,33 @@ class TDEngineSchema:
 
 @dataclass
 class AppResultTable(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.APP_RESULTS
         columns = {
             mm_schemas.WriterEvent.END_INFER_TIME: _TDEngineColumn.TIMESTAMP,
             mm_schemas.WriterEvent.START_INFER_TIME: _TDEngineColumn.TIMESTAMP,
             mm_schemas.ResultData.RESULT_VALUE: _TDEngineColumn.FLOAT,
             mm_schemas.ResultData.RESULT_STATUS: _TDEngineColumn.INT,
+            mm_schemas.ResultData.RESULT_EXTRA_DATA: _TDEngineColumn.BINARY_1000,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.APPLICATION_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.ResultData.RESULT_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.ResultData.RESULT_KIND: _TDEngineColumn.INT,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )
 
 
 @dataclass
 class Metrics(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.METRICS
         columns = {
             mm_schemas.WriterEvent.END_INFER_TIME: _TDEngineColumn.TIMESTAMP,
@@ -255,25 +298,56 @@ class Metrics(TDEngineSchema):
             mm_schemas.MetricData.METRIC_VALUE: _TDEngineColumn.FLOAT,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.APPLICATION_NAME: _TDEngineColumn.BINARY_64,
             mm_schemas.MetricData.METRIC_NAME: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )
 
 
 @dataclass
 class Predictions(TDEngineSchema):
-    def __init__(self, database: Optional[str] = None):
+    def __init__(self, project: str, database: Optional[str] = None):
         super_table = mm_schemas.TDEngineSuperTables.PREDICTIONS
         columns = {
             mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
             mm_schemas.EventFieldType.LATENCY: _TDEngineColumn.FLOAT,
-            mm_schemas.EventKeyMetrics.CUSTOM_METRICS: _TDEngineColumn.BINARY_10000,
+            mm_schemas.EventKeyMetrics.CUSTOM_METRICS: _TDEngineColumn.BINARY_1000,
         }
         tags = {
-            mm_schemas.EventFieldType.PROJECT: _TDEngineColumn.BINARY_64,
             mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
         }
-        super().__init__(super_table, columns, tags, database)
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )
+
+
+@dataclass
+class Errors(TDEngineSchema):
+    def __init__(self, project: str, database: Optional[str] = None):
+        super_table = mm_schemas.TDEngineSuperTables.ERRORS
+        columns = {
+            mm_schemas.EventFieldType.TIME: _TDEngineColumn.TIMESTAMP,
+            mm_schemas.EventFieldType.MODEL_ERROR: _TDEngineColumn.BINARY_1000,
+        }
+        tags = {
+            mm_schemas.WriterEvent.ENDPOINT_ID: _TDEngineColumn.BINARY_64,
+            mm_schemas.EventFieldType.ERROR_TYPE: _TDEngineColumn.BINARY_64,
+        }
+        super().__init__(
+            super_table=super_table,
+            columns=columns,
+            tags=tags,
+            database=database,
+            project=project,
+        )

mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 
 import json
+from datetime import datetime
 
 import mlrun.feature_store.steps
 from mlrun.common.schemas.model_monitoring import (
     EventFieldType,
     EventKeyMetrics,
 )
+from mlrun.utils import logger
 
 
 class ProcessBeforeTDEngine(mlrun.feature_store.steps.MapClass):
@@ -40,3 +42,34 @@ class ProcessBeforeTDEngine(mlrun.feature_store.steps.MapClass):
         event[EventFieldType.TABLE_COLUMN] = "_" + event.get(EventFieldType.ENDPOINT_ID)
 
         return event
+
+
+class ErrorExtractor(mlrun.feature_store.steps.MapClass):
+    def __init__(self, **kwargs):
+        """
+        Prepare the event for insertion into the TDEngine error table
+        """
+        super().__init__(**kwargs)
+
+    def do(self, event):
+        error = str(event.get("error"))
+        if len(error) > 1000:
+            error = error[-1000:]
+            logger.warning(
+                f"Error message exceeds 1000 chars: The error message writen to TSDB will be it last "
+                f"1000 chars, Error:  {error}",
+                event=event,
+            )
+        timestamp = datetime.fromisoformat(event.get("when"))
+        endpoint_id = event[EventFieldType.ENDPOINT_ID]
+        event = {
+            EventFieldType.MODEL_ERROR: error,
+            EventFieldType.ERROR_TYPE: EventFieldType.INFER_ERROR,
+            EventFieldType.ENDPOINT_ID: endpoint_id,
+            EventFieldType.TIME: timestamp,
+            EventFieldType.PROJECT: event[EventFieldType.FUNCTION_URI].split("/")[0],
+            EventFieldType.TABLE_COLUMN: "_err_"
+            + event.get(EventFieldType.ENDPOINT_ID),
+        }
+        logger.info("Write error to errors TSDB table", event=event)
+        return event