mlrun 1.7.0rc34__py3-none-any.whl → 1.7.0rc35__py3-none-any.whl

mlrun/artifacts/base.py

@@ -632,6 +632,7 @@ class DirArtifactSpec(ArtifactSpec):
         "src_path",
         "target_path",
         "db_key",
+        "producer",
     ]
 
 

mlrun/common/schemas/__init__.py

@@ -151,7 +151,6 @@ from .model_monitoring import (
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
-    PrometheusEndpoints,
     TSDBTarget,
     V3IOTSDBTables,
 )

mlrun/common/schemas/model_monitoring/__init__.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from .constants import (
+    V3IO_MODEL_MONITORING_DB,
     ControllerPolicy,
     DriftStatus,
     EndpointType,
@@ -31,8 +32,6 @@ from .constants import (
     MonitoringFunctionNames,
     PredictionsQueryConstants,
     ProjectSecretKeys,
-    PrometheusEndpoints,
-    PrometheusMetric,
     ResultData,
     ResultKindApp,
     SchedulingKeys,

mlrun/common/schemas/model_monitoring/constants.py

@@ -170,7 +170,6 @@ class StreamKind(MonitoringStrEnum):
 class TSDBTarget(MonitoringStrEnum):
     V3IO_TSDB = "v3io-tsdb"
     TDEngine = "tdengine"
-    PROMETHEUS = "prometheus"
 
 
 class ProjectSecretKeys:
@@ -231,21 +230,6 @@ class EndpointType(IntEnum):
     LEAF_EP = 3  # end point that is a child of a router
 
 
-class PrometheusMetric:
-    PREDICTIONS_TOTAL = "predictions_total"
-    MODEL_LATENCY_SECONDS = "model_latency_seconds"
-    INCOME_FEATURES = "income_features"
-    ERRORS_TOTAL = "errors_total"
-    DRIFT_METRICS = "drift_metrics"
-    DRIFT_STATUS = "drift_status"
-
-
-class PrometheusEndpoints(MonitoringStrEnum):
-    MODEL_MONITORING_METRICS = "/model-monitoring-metrics"
-    MONITORING_BATCH_METRICS = "/monitoring-batch-metrics"
-    MONITORING_DRIFT_STATUS = "/monitoring-drift-status"
-
-
 class MonitoringFunctionNames(MonitoringStrEnum):
     STREAM = "model-monitoring-stream"
     APPLICATION_CONTROLLER = "model-monitoring-controller"
@@ -381,3 +365,6 @@ class SpecialApps:
 
 
 _RESERVED_FUNCTION_NAMES = MonitoringFunctionNames.list() + [SpecialApps.MLRUN_INFRA]
+
+
+V3IO_MODEL_MONITORING_DB = "v3io"

mlrun/common/schemas/notification.py

@@ -55,7 +55,7 @@ class Notification(pydantic.BaseModel):
     message: str
     severity: NotificationSeverity
     when: list[str]
-    condition: str
+    condition: str = None
     params: dict[str, typing.Any] = None
     status: NotificationStatus = None
     sent_time: typing.Union[str, datetime.datetime] = None

mlrun/common/types.py

@@ -30,6 +30,7 @@ class HTTPMethod(StrEnum):
     GET = "GET"
     POST = "POST"
     DELETE = "DELETE"
+    PATCH = "PATCH"
 
 
 class Operation(StrEnum):

mlrun/config.py

@@ -485,10 +485,10 @@ default_config = {
             # pip install <requirement_specifier>, e.g. mlrun==0.5.4, mlrun~=0.5,
             # git+https://github.com/mlrun/mlrun@development. by default uses the version
             "mlrun_version_specifier": "",
-            "kaniko_image": "gcr.io/kaniko-project/executor:v1.21.1",  # kaniko builder image
+            "kaniko_image": "gcr.io/kaniko-project/executor:v1.23.2",  # kaniko builder image
             "kaniko_init_container_image": "alpine:3.18",
             # image for kaniko init container when docker registry is ECR
-            "kaniko_aws_cli_image": "amazon/aws-cli:2.7.10",
+            "kaniko_aws_cli_image": "amazon/aws-cli:2.17.16",
             # kaniko sometimes fails to get filesystem from image, this is a workaround to retry the process
             # a known issue in Kaniko - https://github.com/GoogleContainerTools/kaniko/issues/1717
             "kaniko_image_fs_extraction_retries": "3",
@@ -1237,12 +1237,11 @@ class Config:
 
         return storage_options
 
-    def is_explicit_ack(self, version=None) -> bool:
-        if not version:
-            version = self.nuclio_version
+    def is_explicit_ack_enabled(self) -> bool:
         return self.httpdb.nuclio.explicit_ack == "enabled" and (
-            not version
-            or semver.VersionInfo.parse(version) >= semver.VersionInfo.parse("1.12.10")
+            not self.nuclio_version
+            or semver.VersionInfo.parse(self.nuclio_version)
+            >= semver.VersionInfo.parse("1.12.10")
         )
 
 

mlrun/datastore/sources.py

@@ -85,7 +85,8 @@ class BaseSourceDriver(DataSource):
             )
 
         explicit_ack = (
-            is_explicit_ack_supported(context) and mlrun.mlconf.is_explicit_ack()
+            is_explicit_ack_supported(context)
+            and mlrun.mlconf.is_explicit_ack_enabled()
         )
         return storey.SyncEmitSource(
             context=context,
@@ -944,7 +945,8 @@ class OnlineSource(BaseSourceDriver):
 
         source_args = self.attributes.get("source_args", {})
         explicit_ack = (
-            is_explicit_ack_supported(context) and mlrun.mlconf.is_explicit_ack()
+            is_explicit_ack_supported(context)
+            and mlrun.mlconf.is_explicit_ack_enabled()
         )
         # TODO: Change to AsyncEmitSource once we can drop support for nuclio<1.12.10
         src_class = storey.SyncEmitSource(
@@ -1029,7 +1031,8 @@ class StreamSource(OnlineSource):
         engine = "async"
         if hasattr(function.spec, "graph") and function.spec.graph.engine:
             engine = function.spec.graph.engine
-        if mlrun.mlconf.is_explicit_ack() and engine == "async":
+
+        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
             kwargs["explicit_ack_mode"] = "explicitOnly"
             kwargs["worker_allocation_mode"] = "static"
 
@@ -1116,7 +1119,8 @@ class KafkaSource(OnlineSource):
         engine = "async"
         if hasattr(function.spec, "graph") and function.spec.graph.engine:
             engine = function.spec.graph.engine
-        if mlrun.mlconf.is_explicit_ack() and engine == "async":
+
+        if mlrun.mlconf.is_explicit_ack_enabled() and engine == "async":
             explicit_ack_mode = "explicitOnly"
             extra_attributes["workerAllocationMode"] = extra_attributes.get(
                 "worker_allocation_mode", "static"

mlrun/db/base.py

@@ -242,9 +242,8 @@ class RunDBInterface(ABC):
             )
             artifact_identifiers.append(
                 mlrun.common.schemas.ArtifactIdentifier(
-                    key=mlrun.utils.get_in_artifact(artifact_obj, "key"),
-                    # we are passing tree as uid when storing an artifact, so if uid is not defined,
-                    # pass the tree as uid
+                    # we pass the db_key and not the key so the API will be able to find the artifact in the db
+                    key=mlrun.utils.get_in_artifact(artifact_obj, "db_key"),
                     uid=mlrun.utils.get_in_artifact(artifact_obj, "uid"),
                     producer_id=mlrun.utils.get_in_artifact(artifact_obj, "tree"),
                     kind=mlrun.utils.get_in_artifact(artifact_obj, "kind"),

mlrun/db/httpdb.py

@@ -1015,7 +1015,7 @@ class HTTPRunDB(RunDBInterface):
             "format": format_,
             "tag": tag,
             "tree": tree,
-            "object_uid": uid,
+            "object-uid": uid,
         }
         if iter is not None:
             params["iter"] = str(iter)
@@ -1051,7 +1051,7 @@ class HTTPRunDB(RunDBInterface):
             "key": key,
             "tag": tag,
             "tree": tree,
-            "object_uid": uid,
+            "object-uid": uid,
             "iter": iter,
             "deletion_strategy": deletion_strategy,
         }
@@ -3380,7 +3380,7 @@ class HTTPRunDB(RunDBInterface):
                                          By default, the image is mlrun/mlrun.
         """
         self.api_call(
-            method=mlrun.common.types.HTTPMethod.POST,
+            method=mlrun.common.types.HTTPMethod.PATCH,
             path=f"projects/{project}/model-monitoring/model-monitoring-controller",
             params={
                 "base_period": base_period,

mlrun/model.py

@@ -754,7 +754,7 @@ class Notification(ModelObj):
                 "Both 'secret_params' and 'params' are empty, at least one must be defined."
             )
 
-        notification_class.validate_params(secret_params or params)
+        notification_class.validate_params(secret_params | params)
 
     @staticmethod
     def validate_notification_uniqueness(notifications: list["Notification"]):

mlrun/model_monitoring/applications/evidently_base.py

@@ -23,7 +23,7 @@ import mlrun.model_monitoring.applications.base as mm_base
 import mlrun.model_monitoring.applications.context as mm_context
 from mlrun.errors import MLRunIncompatibleVersionError
 
-SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.11")
+SUPPORTED_EVIDENTLY_VERSION = semver.Version.parse("0.4.32")
 
 
 def _check_evidently_version(*, cur: semver.Version, ref: semver.Version) -> None:
@@ -57,12 +57,11 @@ except ModuleNotFoundError:
 
 
 if _HAS_EVIDENTLY:
-    from evidently.renderers.notebook_utils import determine_template
     from evidently.report.report import Report
     from evidently.suite.base_suite import Suite
     from evidently.ui.type_aliases import STR_UUID
     from evidently.ui.workspace import Workspace
-    from evidently.utils.dashboard import TemplateParams
+    from evidently.utils.dashboard import TemplateParams, file_html_template
 
 
 class EvidentlyModelMonitoringApplicationBase(mm_base.ModelMonitoringApplicationBase):
@@ -123,7 +122,7 @@ class EvidentlyModelMonitoringApplicationBase(mm_base.ModelMonitoringApplication
             additional_graphs={},
         )
 
-        dashboard_html = self._render(determine_template("inline"), template_params)
+        dashboard_html = self._render(file_html_template, template_params)
         self.context.log_artifact(
             artifact_name, body=dashboard_html.encode("utf-8"), format="html"
         )
@@ -201,7 +200,7 @@ class EvidentlyModelMonitoringApplicationBaseV2(
             additional_graphs={},
         )
 
-        dashboard_html = self._render(determine_template("inline"), template_params)
+        dashboard_html = self._render(file_html_template, template_params)
         monitoring_context.log_artifact(
             artifact_name, body=dashboard_html.encode("utf-8"), format="html"
         )

mlrun/model_monitoring/db/stores/sqldb/sql_store.py

@@ -177,6 +177,11 @@ class SQLStoreBase(StoreBase):
         param table:     SQLAlchemy declarative table.
         :param criteria: A list of binary expressions that filter the query.
         """
+        if not self._engine.has_table(table.__tablename__):
+            logger.debug(
+                f"Table {table.__tablename__} does not exist in the database. Skipping deletion."
+            )
+            return
         with create_session(dsn=self._sql_connection_string) as session:
             # Generate and commit the delete query
             session.query(

mlrun/model_monitoring/db/stores/v3io_kv/kv_store.py

@@ -408,14 +408,14 @@ class KVStoreBase(StoreBase):
 
         """
         try:
-            data = self.client.kv.get(
+            response = self.client.kv.get(
                 container=self._get_monitoring_schedules_container(
                     project_name=self.project
                 ),
                 table_path=endpoint_id,
                 key=application_name,
             )
-            return data.output.item[mm_schemas.SchedulingKeys.LAST_ANALYZED]
+            return response.output.item[mm_schemas.SchedulingKeys.LAST_ANALYZED]
         except v3io.dataplane.response.HttpResponseError as err:
             logger.debug("Error while getting last analyzed time", err=err)
             raise mlrun.errors.MLRunNotFoundError(

mlrun/model_monitoring/db/tsdb/base.py

@@ -27,7 +27,7 @@ from mlrun.utils import logger
 class TSDBConnector(ABC):
     type: typing.ClassVar[str]
 
-    def __init__(self, project: str):
+    def __init__(self, project: str) -> None:
         """
         Initialize a new TSDB connector. The connector is used to interact with the TSDB and store monitoring data.
         At the moment we have 3 different types of monitoring data:
@@ -42,10 +42,10 @@ class TSDBConnector(ABC):
         writer.
 
         :param project: the name of the project.
-
         """
         self.project = project
 
+    @abstractmethod
     def apply_monitoring_stream_steps(self, graph):
         """
         Apply TSDB steps on the provided monitoring graph. Throughout these steps, the graph stores live data of
@@ -58,6 +58,7 @@ class TSDBConnector(ABC):
         """
         pass
 
+    @abstractmethod
     def write_application_event(
         self,
         event: dict,
@@ -69,13 +70,14 @@ class TSDBConnector(ABC):
         :raise mlrun.errors.MLRunRuntimeError: If an error occurred while writing the event.
         """
 
+    @abstractmethod
     def delete_tsdb_resources(self):
         """
         Delete all project resources in the TSDB connector, such as model endpoints data and drift results.
         """
-
         pass
 
+    @abstractmethod
     def get_model_endpoint_real_time_metrics(
         self,
         endpoint_id: str,
@@ -102,6 +104,7 @@ class TSDBConnector(ABC):
         """
         pass
 
+    @abstractmethod
     def create_tables(self) -> None:
         """
         Create the TSDB tables using the TSDB connector. At the moment we support 3 types of tables:

mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 import json
 
@@ -21,8 +20,6 @@ from mlrun.common.schemas.model_monitoring import (
     EventKeyMetrics,
 )
 
-_TABLE_COLUMN = "table_column"
-
 
 class ProcessBeforeTDEngine(mlrun.feature_store.steps.MapClass):
     def __init__(self, **kwargs):

mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py

@@ -11,7 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+
+from typing import Any
 
 import mlrun.feature_store.steps
 from mlrun.common.schemas.model_monitoring import (
@@ -21,6 +22,24 @@ from mlrun.common.schemas.model_monitoring import (
 )
 
 
+def _normalize_dict_for_v3io_frames(event: dict[str, Any]) -> dict[str, Any]:
+    """
+    Normalize user defined keys - input data to a model and its predictions,
+    to a form V3IO frames tolerates.
+
+    The dictionary keys should conform to '^[a-zA-Z_:]([a-zA-Z0-9_:])*$'.
+    """
+    prefix = "_"
+
+    def norm_key(key: str) -> str:
+        key = key.replace("-", "_")  # hyphens `-` are not allowed
+        if key and key[0].isdigit():  # starting with a digit is not allowed
+            return prefix + key
+        return key
+
+    return {norm_key(k): v for k, v in event.items()}
+
+
 class ProcessBeforeTSDB(mlrun.feature_store.steps.MapClass):
     def __init__(self, **kwargs):
         """
@@ -68,8 +87,8 @@ class ProcessBeforeTSDB(mlrun.feature_store.steps.MapClass):
         # endpoint_features includes the event values of each feature and prediction
         endpoint_features = {
             EventFieldType.RECORD_TYPE: EventKeyMetrics.ENDPOINT_FEATURES,
-            **event[EventFieldType.NAMED_PREDICTIONS],
-            **event[EventFieldType.NAMED_FEATURES],
+            **_normalize_dict_for_v3io_frames(event[EventFieldType.NAMED_PREDICTIONS]),
+            **_normalize_dict_for_v3io_frames(event[EventFieldType.NAMED_FEATURES]),
             **base_event,
         }
         # Create a dictionary that includes both base_metrics and endpoint_features

mlrun/model_monitoring/stream_processing.py

@@ -27,7 +27,6 @@ import mlrun.datastore.targets
 import mlrun.feature_store as fstore
 import mlrun.feature_store.steps
 import mlrun.model_monitoring.db
-import mlrun.model_monitoring.prometheus
 import mlrun.serving.states
 import mlrun.utils
 from mlrun.common.schemas.model_monitoring.constants import (
@@ -37,7 +36,6 @@ from mlrun.common.schemas.model_monitoring.constants import (
     FileTargetKind,
     ModelEndpointTarget,
     ProjectSecretKeys,
-    PrometheusEndpoints,
 )
 from mlrun.utils import logger
 
@@ -172,39 +170,12 @@ class EventStreamProcessor:
             fn.set_topology(mlrun.serving.states.StepKinds.flow),
         )
 
-        # Event routing based on the provided path
-        def apply_event_routing():
-            typing.cast(
-                mlrun.serving.TaskStep,
-                graph.add_step(
-                    "EventRouting",
-                    full_event=True,
-                    project=self.project,
-                ),
-            ).respond()
-
-        apply_event_routing()
-
-        # Filter out events with '-' in the path basename from going forward
-        # through the next steps of the stream graph
-        def apply_storey_filter_stream_events():
-            # Filter events with Prometheus endpoints path
-            graph.add_step(
-                "storey.Filter",
-                "filter_stream_event",
-                _fn=f"(event.path not in {PrometheusEndpoints.list()})",
-                full_event=True,
-            )
-
-        apply_storey_filter_stream_events()
-
         # Process endpoint event: splitting into sub-events and validate event data
         def apply_process_endpoint_event():
             graph.add_step(
                 "ProcessEndpointEvent",
                 full_event=True,
                 project=self.project,
-                after="filter_stream_event",
             )
 
         apply_process_endpoint_event()
@@ -324,33 +295,10 @@ class EventStreamProcessor:
 
         apply_storey_sample_window()
 
-        # TSDB branch (skip to Prometheus if in CE env)
-        if not mlrun.mlconf.is_ce_mode():
-            tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-                project=self.project, secret_provider=secret_provider
-            )
-            tsdb_connector.apply_monitoring_stream_steps(graph=graph)
-
-        else:
-            # Prometheus
-            # Increase the prediction counter by 1 and update the latency value
-            graph.add_step(
-                "IncCounter",
-                name="IncCounter",
-                after="MapFeatureNames",
-                project=self.project,
-            )
-
-            # Record a sample of features and labels
-            def apply_record_features_to_prometheus():
-                graph.add_step(
-                    "RecordFeatures",
-                    name="RecordFeaturesToPrometheus",
-                    after="sample",
-                    project=self.project,
-                )
-
-            apply_record_features_to_prometheus()
+        tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
+            project=self.project, secret_provider=secret_provider
+        )
+        tsdb_connector.apply_monitoring_stream_steps(graph=graph)
 
         # Parquet branch
         # Filter and validate different keys before writing the data to Parquet target
@@ -542,11 +490,7 @@ class ProcessEndpointEvent(mlrun.feature_store.steps.MapClass):
         error = event.get("error")
         if error:
             self.error_count[endpoint_id] += 1
-            mlrun.model_monitoring.prometheus.write_errors(
-                project=self.project,
-                endpoint_id=event["endpoint_id"],
-                model_name=event["model"],
-            )
+            # TODO: write to tsdb / kv once in a while
             raise mlrun.errors.MLRunInvalidArgumentError(str(error))
 
         # Validate event fields
@@ -973,98 +917,6 @@ class InferSchema(mlrun.feature_store.steps.MapClass):
         return event
 
 
-class EventRouting(mlrun.feature_store.steps.MapClass):
-    """
-    Router the event according to the configured path under event.path. Please note that this step returns the result
-    to the caller. At the moment there are several paths:
-
-    - /model-monitoring-metrics (GET): return Prometheus registry results as a text. Will be used by Prometheus client
-    to scrape the results from the monitoring stream memory.
-
-    - /monitoring-batch-metrics (POST): update the Prometheus registry with the provided statistical metrics such as the
-     statistical metrics from the monitoring batch job. Note that the event body is a list of dictionaries of different
-     metrics.
-
-    - /monitoring-drift-status (POST): update the Prometheus registry with the provided model drift status.
-
-    """
-
-    def __init__(
-        self,
-        project: str,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.project: str = project
-
-    def do(self, event):
-        if event.path == PrometheusEndpoints.MODEL_MONITORING_METRICS:
-            # Return a parsed Prometheus registry file
-            event.body = mlrun.model_monitoring.prometheus.get_registry()
-        elif event.path == PrometheusEndpoints.MONITORING_BATCH_METRICS:
-            # Update statistical metrics
-            for event_metric in event.body:
-                mlrun.model_monitoring.prometheus.write_drift_metrics(
-                    project=self.project,
-                    endpoint_id=event_metric[EventFieldType.ENDPOINT_ID],
-                    metric=event_metric[EventFieldType.METRIC],
-                    value=event_metric[EventFieldType.VALUE],
-                )
-        elif event.path == PrometheusEndpoints.MONITORING_DRIFT_STATUS:
-            # Update drift status
-            mlrun.model_monitoring.prometheus.write_drift_status(
-                project=self.project,
-                endpoint_id=event.body[EventFieldType.ENDPOINT_ID],
-                drift_status=event.body[EventFieldType.DRIFT_STATUS],
-            )
-
-        return event
-
-
-class IncCounter(mlrun.feature_store.steps.MapClass):
-    """Increase prediction counter by 1 and update the total latency value"""
-
-    def __init__(self, project: str, **kwargs):
-        super().__init__(**kwargs)
-        self.project: str = project
-
-    def do(self, event):
-        # Compute prediction per second
-
-        mlrun.model_monitoring.prometheus.write_predictions_and_latency_metrics(
-            project=self.project,
-            endpoint_id=event[EventFieldType.ENDPOINT_ID],
-            latency=event[EventFieldType.LATENCY],
-            model_name=event[EventFieldType.MODEL],
-            endpoint_type=event[EventFieldType.ENDPOINT_TYPE],
-        )
-
-        return event
-
-
-class RecordFeatures(mlrun.feature_store.steps.MapClass):
-    """Record a sample of features and labels in Prometheus registry"""
-
-    def __init__(self, project: str, **kwargs):
-        super().__init__(**kwargs)
-        self.project: str = project
-
-    def do(self, event):
-        # Generate a dictionary of features and predictions
-        features = {
-            **event[EventFieldType.NAMED_PREDICTIONS],
-            **event[EventFieldType.NAMED_FEATURES],
-        }
-
-        mlrun.model_monitoring.prometheus.write_income_features(
-            project=self.project,
-            endpoint_id=event[EventFieldType.ENDPOINT_ID],
-            features=features,
-        )
-
-        return event
-
-
 def update_endpoint_record(
     project: str,
     endpoint_id: str,