mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/model_monitoring/model_monitoring_batch.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,14 +28,16 @@ import v3io.dataplane
 import v3io_frames
 
 import mlrun
-import mlrun.api.schemas
+import mlrun.common.model_monitoring
+import mlrun.common.schemas
 import mlrun.data_types.infer
 import mlrun.feature_store as fstore
+import mlrun.model_monitoring
+import mlrun.model_monitoring.stores
 import mlrun.run
 import mlrun.utils.helpers
 import mlrun.utils.model_monitoring
 import mlrun.utils.v3io_clients
-from mlrun.model_monitoring.constants import EventFieldType
 from mlrun.utils import logger
 
 
@@ -461,6 +463,7 @@ def calculate_inputs_statistics(
 
     :returns: The calculated statistics of the inputs data.
     """
+
     # Use `DFDataInfer` to calculate the statistics over the inputs:
     inputs_statistics = mlrun.data_types.infer.DFDataInfer.get_stats(
         df=inputs,
@@ -493,8 +496,6 @@ class BatchProcessor:
         self,
         context: mlrun.run.MLClientCtx,
         project: str,
-        model_monitoring_access_key: str,
-        v3io_access_key: str,
     ):
 
         """
@@ -502,60 +503,16 @@ class BatchProcessor:
 
         :param context:                     An MLRun context.
         :param project:                     Project name.
-        :param model_monitoring_access_key: Access key to apply the model monitoring process.
-        :param v3io_access_key:             Token key for v3io.
         """
         self.context = context
         self.project = project
 
-        self.v3io_access_key = v3io_access_key
-        self.model_monitoring_access_key = (
-            model_monitoring_access_key or v3io_access_key
-        )
-
         # Initialize virtual drift object
         self.virtual_drift = VirtualDrift(inf_capping=10)
 
-        # Define the required paths for the project objects.
-        # Note that the kv table, tsdb, and the input stream paths are located at the default location
-        # while the parquet path is located at the user-space location
-        template = mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default
-        kv_path = template.format(project=self.project, kind="endpoints")
-        (
-            _,
-            self.kv_container,
-            self.kv_path,
-        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(kv_path)
-        tsdb_path = template.format(project=project, kind="events")
-        (
-            _,
-            self.tsdb_container,
-            self.tsdb_path,
-        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(tsdb_path)
-        stream_path = template.format(project=self.project, kind="log_stream")
-        (
-            _,
-            self.stream_container,
-            self.stream_path,
-        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(stream_path)
-        self.parquet_path = (
-            mlrun.mlconf.model_endpoint_monitoring.store_prefixes.user_space.format(
-                project=project, kind="parquet"
-            )
-        )
-
         logger.info(
             "Initializing BatchProcessor",
             project=project,
-            model_monitoring_access_key_initalized=bool(model_monitoring_access_key),
-            v3io_access_key_initialized=bool(v3io_access_key),
-            parquet_path=self.parquet_path,
-            kv_container=self.kv_container,
-            kv_path=self.kv_path,
-            tsdb_container=self.tsdb_container,
-            tsdb_path=self.tsdb_path,
-            stream_container=self.stream_container,
-            stream_path=self.stream_path,
         )
 
         # Get drift thresholds from the model monitoring configuration
@@ -567,46 +524,87 @@ class BatchProcessor:
         )
 
         # Get a runtime database
-        self.db = mlrun.get_run_db()
 
-        # Get the frames clients based on the v3io configuration
-        # it will be used later for writing the results into the tsdb
-        self.v3io = mlrun.utils.v3io_clients.get_v3io_client(
-            access_key=self.v3io_access_key
-        )
-        self.frames = mlrun.utils.v3io_clients.get_frames_client(
-            address=mlrun.mlconf.v3io_framesd,
-            container=self.tsdb_container,
-            token=self.v3io_access_key,
+        self.db = mlrun.model_monitoring.stores.get_model_endpoint_store(
+            project=project
         )
 
+        if not mlrun.mlconf.is_ce_mode():
+            # TODO: Once there is a time series DB alternative in a non-CE deployment, we need to update this if
+            #  statement to be applied only for V3IO TSDB
+            self._initialize_v3io_configurations()
+
         # If an error occurs, it will be raised using the following argument
         self.exception = None
 
         # Get the batch interval range
-        self.batch_dict = context.parameters[EventFieldType.BATCH_INTERVALS_DICT]
+        self.batch_dict = context.parameters[
+            mlrun.common.model_monitoring.EventFieldType.BATCH_INTERVALS_DICT
+        ]
 
-        # TODO: This will be removed in 1.2.0 once the job params can be parsed with different types
+        # TODO: This will be removed in 1.5.0 once the job params can be parsed with different types
         # Convert batch dict string into a dictionary
         if isinstance(self.batch_dict, str):
             self._parse_batch_dict_str()
 
+    def _initialize_v3io_configurations(self):
+        self.v3io_access_key = os.environ.get("V3IO_ACCESS_KEY")
+        self.model_monitoring_access_key = (
+            os.environ.get("MODEL_MONITORING_ACCESS_KEY") or self.v3io_access_key
+        )
+
+        # Define the required paths for the project objects
+        tsdb_path = mlrun.mlconf.get_model_monitoring_file_target_path(
+            project=self.project,
+            kind=mlrun.common.model_monitoring.FileTargetKind.EVENTS,
+        )
+        (
+            _,
+            self.tsdb_container,
+            self.tsdb_path,
+        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(tsdb_path)
+        # stream_path = template.format(project=self.project, kind="log_stream")
+        stream_path = mlrun.mlconf.get_model_monitoring_file_target_path(
+            project=self.project,
+            kind=mlrun.common.model_monitoring.FileTargetKind.LOG_STREAM,
+        )
+        (
+            _,
+            self.stream_container,
+            self.stream_path,
+        ) = mlrun.utils.model_monitoring.parse_model_endpoint_store_prefix(stream_path)
+
+        # Get the frames clients based on the v3io configuration
+        # it will be used later for writing the results into the tsdb
+        self.v3io = mlrun.utils.v3io_clients.get_v3io_client(
+            access_key=self.v3io_access_key
+        )
+        self.frames = mlrun.utils.v3io_clients.get_frames_client(
+            address=mlrun.mlconf.v3io_framesd,
+            container=self.tsdb_container,
+            token=self.v3io_access_key,
+        )
+
     def post_init(self):
         """
         Preprocess of the batch processing.
         """
 
-        # create v3io stream based on the input stream
-        response = self.v3io.create_stream(
-            container=self.stream_container,
-            path=self.stream_path,
-            shard_count=1,
-            raise_for_status=v3io.dataplane.RaiseForStatus.never,
-            access_key=self.v3io_access_key,
-        )
+        if not mlrun.mlconf.is_ce_mode():
+            # Create v3io stream based on the input stream
+            response = self.v3io.create_stream(
+                container=self.stream_container,
+                path=self.stream_path,
+                shard_count=1,
+                raise_for_status=v3io.dataplane.RaiseForStatus.never,
+                access_key=self.v3io_access_key,
+            )
 
-        if not (response.status_code == 400 and "ResourceInUse" in str(response.body)):
-            response.raise_for_status([409, 204, 403])
+            if not (
+                response.status_code == 400 and "ResourceInUse" in str(response.body)
+            ):
+                response.raise_for_status([409, 204, 403])
+        pass
 
     def run(self):
         """
@@ -614,231 +612,218 @@ class BatchProcessor:
         """
         # Get model endpoints (each deployed project has at least 1 serving model):
         try:
-            endpoints = self.db.list_model_endpoints(self.project)
+            endpoints = self.db.list_model_endpoints()
         except Exception as e:
             logger.error("Failed to list endpoints", exc=e)
             return
 
-        active_endpoints = set()
-        for endpoint in endpoints.endpoints:
+        for endpoint in endpoints:
             if (
-                endpoint.spec.active
-                and endpoint.spec.monitoring_mode
-                == mlrun.api.schemas.ModelMonitoringMode.enabled.value
+                endpoint[mlrun.common.model_monitoring.EventFieldType.ACTIVE]
+                and endpoint[
+                    mlrun.common.model_monitoring.EventFieldType.MONITORING_MODE
+                ]
+                == mlrun.common.model_monitoring.ModelMonitoringMode.enabled.value
             ):
-                active_endpoints.add(endpoint.metadata.uid)
-
-        # perform drift analysis for each model endpoint
-        for endpoint_id in active_endpoints:
-            try:
-
-                # Get model endpoint object:
-                endpoint = self.db.get_model_endpoint(
-                    project=self.project, endpoint_id=endpoint_id
-                )
-
                 # Skip router endpoint:
                 if (
-                    endpoint.status.endpoint_type
-                    == mlrun.utils.model_monitoring.EndpointType.ROUTER
+                    int(
+                        endpoint[
+                            mlrun.common.model_monitoring.EventFieldType.ENDPOINT_TYPE
+                        ]
+                    )
+                    == mlrun.common.model_monitoring.EndpointType.ROUTER
                 ):
-                    # endpoint.status.feature_stats is None
-                    logger.info(f"{endpoint_id} is router skipping")
+                    # Router endpoint has no feature stats
+                    logger.info(
+                        f"{endpoint[mlrun.common.model_monitoring.EventFieldType.UID]} is router skipping"
+                    )
                     continue
+                self.update_drift_metrics(endpoint=endpoint)
 
-                # convert feature set into dataframe and get the latest dataset
-                (
-                    _,
-                    serving_function_name,
-                    _,
-                    _,
-                ) = mlrun.utils.helpers.parse_versioned_object_uri(
-                    endpoint.spec.function_uri
-                )
-
-                model_name = endpoint.spec.model.replace(":", "-")
+    def update_drift_metrics(self, endpoint: dict):
+        try:
+            # Convert feature set into dataframe and get the latest dataset
+            (
+                _,
+                serving_function_name,
+                _,
+                _,
+            ) = mlrun.utils.helpers.parse_versioned_object_uri(
+                endpoint[mlrun.common.model_monitoring.EventFieldType.FUNCTION_URI]
+            )
 
-                m_fs = fstore.get_feature_set(
-                    f"store://feature-sets/{self.project}/monitoring-{serving_function_name}-{model_name}"
-                )
+            model_name = endpoint[
+                mlrun.common.model_monitoring.EventFieldType.MODEL
+            ].replace(":", "-")
 
-                # Getting batch interval start time and end time
-                start_time, end_time = self.get_interval_range()
+            m_fs = fstore.get_feature_set(
+                f"store://feature-sets/{self.project}/monitoring-{serving_function_name}-{model_name}"
+            )
 
-                try:
-                    df = m_fs.to_dataframe(
-                        start_time=start_time,
-                        end_time=end_time,
-                        time_column="timestamp",
-                    )
+            # Getting batch interval start time and end time
+            start_time, end_time = self._get_interval_range()
 
-                    if len(df) == 0:
-                        logger.warn(
-                            "Not enough model events since the beginning of the batch interval",
-                            parquet_target=m_fs.status.targets[0].path,
-                            endpoint=endpoint_id,
-                            min_rqeuired_events=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_max_events,
-                            start_time=str(
-                                datetime.datetime.now() - datetime.timedelta(hours=1)
-                            ),
-                            end_time=str(datetime.datetime.now()),
-                        )
-                        continue
+            try:
+                df = m_fs.to_dataframe(
+                    start_time=start_time,
+                    end_time=end_time,
+                    time_column=mlrun.common.model_monitoring.EventFieldType.TIMESTAMP,
+                )
 
-                # TODO: The below warn will be removed once the state of the Feature Store target is updated
-                #       as expected. In that case, the existence of the file will be checked before trying to get
-                #       the offline data from the feature set.
-                # Continue if not enough events provided since the deployment of the model endpoint
-                except FileNotFoundError:
+                if len(df) == 0:
                     logger.warn(
-                        "Parquet not found, probably due to not enough model events",
+                        "Not enough model events since the beginning of the batch interval",
                         parquet_target=m_fs.status.targets[0].path,
-                        endpoint=endpoint_id,
+                        endpoint=endpoint[
+                            mlrun.common.model_monitoring.EventFieldType.UID
+                        ],
                         min_rqeuired_events=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_max_events,
+                        start_time=str(
+                            datetime.datetime.now() - datetime.timedelta(hours=1)
+                        ),
+                        end_time=str(datetime.datetime.now()),
                     )
-                    continue
+                    return
+
+            # TODO: The below warn will be removed once the state of the Feature Store target is updated
+            #       as expected. In that case, the existence of the file will be checked before trying to get
+            #       the offline data from the feature set.
+            # Continue if not enough events provided since the deployment of the model endpoint
+            except FileNotFoundError:
+                logger.warn(
+                    "Parquet not found, probably due to not enough model events",
+                    parquet_target=m_fs.status.targets[0].path,
+                    endpoint=endpoint[mlrun.common.model_monitoring.EventFieldType.UID],
+                    min_rqeuired_events=mlrun.mlconf.model_endpoint_monitoring.parquet_batching_max_events,
+                )
+                return
 
-                # Get feature names from monitoring feature set
-                feature_names = [
-                    feature_name["name"]
-                    for feature_name in m_fs.spec.features.to_dict()
-                ]
+            # Get feature names from monitoring feature set
+            feature_names = [
+                feature_name["name"] for feature_name in m_fs.spec.features.to_dict()
+            ]
 
-                # Create DataFrame based on the input features
-                stats_columns = [
-                    "timestamp",
-                    *feature_names,
-                ]
+            # Create DataFrame based on the input features
+            stats_columns = [
+                mlrun.common.model_monitoring.EventFieldType.TIMESTAMP,
+                *feature_names,
+            ]
 
-                # Add label names if provided
-                if endpoint.spec.label_names:
-                    stats_columns.extend(endpoint.spec.label_names)
+            # Add label names if provided
+            if endpoint[mlrun.common.model_monitoring.EventFieldType.LABEL_NAMES]:
+                labels = endpoint[
+                    mlrun.common.model_monitoring.EventFieldType.LABEL_NAMES
+                ]
+                if isinstance(labels, str):
+                    labels = json.loads(labels)
+                stats_columns.extend(labels)
+            named_features_df = df[stats_columns].copy()
+
+            # Infer feature set stats and schema
+            fstore.api._infer_from_static_df(
+                named_features_df,
+                m_fs,
+                options=mlrun.data_types.infer.InferOptions.all_stats(),
+            )
 
-                named_features_df = df[stats_columns].copy()
+            # Save feature set to apply changes
+            m_fs.save()
 
-                # Infer feature set stats and schema
-                fstore.api._infer_from_static_df(
-                    named_features_df,
-                    m_fs,
-                    options=mlrun.data_types.infer.InferOptions.all_stats(),
-                )
+            # Get the timestamp of the latest request:
+            timestamp = df[mlrun.common.model_monitoring.EventFieldType.TIMESTAMP].iloc[
+                -1
+            ]
 
-                # Save feature set to apply changes
-                m_fs.save()
+            # Get the feature stats from the model endpoint for reference data
+            feature_stats = json.loads(
+                endpoint[mlrun.common.model_monitoring.EventFieldType.FEATURE_STATS]
+            )
 
-                # Get the timestamp of the latest request:
-                timestamp = df["timestamp"].iloc[-1]
+            # Get the current stats:
+            current_stats = calculate_inputs_statistics(
+                sample_set_statistics=feature_stats,
+                inputs=named_features_df,
+            )
 
-                # Get the current stats:
-                current_stats = calculate_inputs_statistics(
-                    sample_set_statistics=endpoint.status.feature_stats,
-                    inputs=named_features_df,
+            # Compute the drift based on the histogram of the current stats and the histogram of the original
+            # feature stats that can be found in the model endpoint object:
+            drift_result = self.virtual_drift.compute_drift_from_histograms(
+                feature_stats=feature_stats,
+                current_stats=current_stats,
+            )
+            logger.info("Drift result", drift_result=drift_result)
+
+            # Get drift thresholds from the model configuration:
+            monitor_configuration = (
+                json.loads(
+                    endpoint[
+                        mlrun.common.model_monitoring.EventFieldType.MONITOR_CONFIGURATION
+                    ]
                 )
+                or {}
+            )
+            possible_drift = monitor_configuration.get(
+                "possible_drift", self.default_possible_drift_threshold
+            )
+            drift_detected = monitor_configuration.get(
+                "drift_detected", self.default_drift_detected_threshold
+            )
 
-                # Compute the drift based on the histogram of the current stats and the histogram of the original
-                # feature stats that can be found in the model endpoint object:
-                drift_result = self.virtual_drift.compute_drift_from_histograms(
-                    feature_stats=endpoint.status.feature_stats,
-                    current_stats=current_stats,
-                )
-                logger.info("Drift result", drift_result=drift_result)
+            # Check for possible drift based on the results of the statistical metrics defined above:
+            drift_status, drift_measure = self.virtual_drift.check_for_drift(
+                metrics_results_dictionary=drift_result,
+                possible_drift_threshold=possible_drift,
+                drift_detected_threshold=drift_detected,
+            )
+            logger.info(
+                "Drift status",
+                endpoint_id=endpoint[mlrun.common.model_monitoring.EventFieldType.UID],
+                drift_status=drift_status.value,
+                drift_measure=drift_measure,
+            )
 
-                # Get drift thresholds from the model configuration:
-                monitor_configuration = endpoint.spec.monitor_configuration or {}
-                possible_drift = monitor_configuration.get(
-                    "possible_drift", self.default_possible_drift_threshold
-                )
-                drift_detected = monitor_configuration.get(
-                    "drift_detected", self.default_drift_detected_threshold
-                )
+            attributes = {
+                "current_stats": json.dumps(current_stats),
+                "drift_measures": json.dumps(drift_result),
+                "drift_status": drift_status.value,
+            }
 
-                # Check for possible drift based on the results of the statistical metrics defined above:
-                drift_status, drift_measure = self.virtual_drift.check_for_drift(
-                    metrics_results_dictionary=drift_result,
-                    possible_drift_threshold=possible_drift,
-                    drift_detected_threshold=drift_detected,
-                )
-                logger.info(
-                    "Drift status",
-                    endpoint_id=endpoint_id,
-                    drift_status=drift_status.value,
+            self.db.update_model_endpoint(
+                endpoint_id=endpoint[mlrun.common.model_monitoring.EventFieldType.UID],
+                attributes=attributes,
+            )
+
+            if not mlrun.mlconf.is_ce_mode():
+                # Update drift results in TSDB
+                self._update_drift_in_input_stream(
+                    endpoint_id=endpoint[
+                        mlrun.common.model_monitoring.EventFieldType.UID
+                    ],
+                    drift_status=drift_status,
                     drift_measure=drift_measure,
+                    drift_result=drift_result,
+                    timestamp=timestamp,
                 )
-
-                # If drift was detected, add the results to the input stream
-                if (
-                    drift_status == DriftStatus.POSSIBLE_DRIFT
-                    or drift_status == DriftStatus.DRIFT_DETECTED
-                ):
-                    self.v3io.stream.put_records(
-                        container=self.stream_container,
-                        stream_path=self.stream_path,
-                        records=[
-                            {
-                                "data": json.dumps(
-                                    {
-                                        "endpoint_id": endpoint_id,
-                                        "drift_status": drift_status.value,
-                                        "drift_measure": drift_measure,
-                                        "drift_per_feature": {**drift_result},
-                                    }
-                                )
-                            }
-                        ],
-                    )
-
-                attributes = {
-                    "current_stats": json.dumps(current_stats),
-                    "drift_measures": json.dumps(drift_result),
-                    "drift_status": drift_status.value,
-                }
-
-                self.db.patch_model_endpoint(
-                    project=self.project,
-                    endpoint_id=endpoint_id,
-                    attributes=attributes,
+                logger.info(
+                    "Done updating drift measures",
+                    endpoint_id=endpoint[
+                        mlrun.common.model_monitoring.EventFieldType.UID
+                    ],
                 )
 
-                # Update the results in tsdb:
-                tsdb_drift_measures = {
-                    "endpoint_id": endpoint_id,
-                    "timestamp": pd.to_datetime(
-                        timestamp,
-                        format=EventFieldType.TIME_FORMAT,
-                    ),
-                    "record_type": "drift_measures",
-                    "tvd_mean": drift_result["tvd_mean"],
-                    "kld_mean": drift_result["kld_mean"],
-                    "hellinger_mean": drift_result["hellinger_mean"],
-                }
-
-                try:
-                    self.frames.write(
-                        backend="tsdb",
-                        table=self.tsdb_path,
-                        dfs=pd.DataFrame.from_dict([tsdb_drift_measures]),
-                        index_cols=["timestamp", "endpoint_id", "record_type"],
-                    )
-                except v3io_frames.errors.Error as err:
-                    logger.warn(
-                        "Could not write drift measures to TSDB",
-                        err=err,
-                        tsdb_path=self.tsdb_path,
-                        endpoint=endpoint_id,
-                    )
-
-                logger.info("Done updating drift measures", endpoint_id=endpoint_id)
-
-            except Exception as e:
-                logger.error(f"Exception for endpoint {endpoint_id}")
-                self.exception = e
+        except Exception as e:
+            logger.error(
+                f"Exception for endpoint {endpoint[mlrun.common.model_monitoring.EventFieldType.UID]}"
+            )
+            self.exception = e
 
-    def get_interval_range(self) -> Tuple[datetime.datetime, datetime.datetime]:
+    def _get_interval_range(self) -> Tuple[datetime.datetime, datetime.datetime]:
         """Getting batch interval time range"""
         minutes, hours, days = (
-            self.batch_dict[EventFieldType.MINUTES],
-            self.batch_dict[EventFieldType.HOURS],
-            self.batch_dict[EventFieldType.DAYS],
+            self.batch_dict[mlrun.common.model_monitoring.EventFieldType.MINUTES],
+            self.batch_dict[mlrun.common.model_monitoring.EventFieldType.HOURS],
+            self.batch_dict[mlrun.common.model_monitoring.EventFieldType.DAYS],
         )
         start_time = datetime.datetime.now() - datetime.timedelta(
             minutes=minutes, hours=hours, days=days
@@ -858,13 +843,79 @@ class BatchProcessor:
             pair_list = pair.split(":")
             self.batch_dict[pair_list[0]] = float(pair_list[1])
 
+    def _update_drift_in_input_stream(
+        self,
+        endpoint_id: str,
+        drift_status: DriftStatus,
+        drift_measure: float,
+        drift_result: Dict[str, Dict[str, Any]],
+        timestamp: pd._libs.tslibs.timestamps.Timestamp,
+    ):
+        """Update drift results in input stream.
+
+        :param endpoint_id:   The unique id of the model endpoint.
+        :param drift_status:  Drift status result. Possible values can be found under DriftStatus enum class.
+        :param drift_measure: The drift result (float) based on the mean of the Total Variance Distance and the
+                              Hellinger distance.
+        :param drift_result:  A dictionary that includes the drift results for each feature.
+        :param timestamp:     Pandas Timestamp value.
+
+        """
+
+        if (
+            drift_status == DriftStatus.POSSIBLE_DRIFT
+            or drift_status == DriftStatus.DRIFT_DETECTED
+        ):
+            self.v3io.stream.put_records(
+                container=self.stream_container,
+                stream_path=self.stream_path,
+                records=[
+                    {
+                        "data": json.dumps(
+                            {
+                                "endpoint_id": endpoint_id,
+                                "drift_status": drift_status.value,
+                                "drift_measure": drift_measure,
+                                "drift_per_feature": {**drift_result},
+                            }
+                        )
+                    }
+                ],
+            )
+
+        # Update the results in tsdb:
+        tsdb_drift_measures = {
+            "endpoint_id": endpoint_id,
+            "timestamp": pd.to_datetime(
+                timestamp,
+                format=mlrun.common.model_monitoring.EventFieldType.TIME_FORMAT,
+            ),
+            "record_type": "drift_measures",
+            "tvd_mean": drift_result["tvd_mean"],
+            "kld_mean": drift_result["kld_mean"],
+            "hellinger_mean": drift_result["hellinger_mean"],
+        }
+
+        try:
+            self.frames.write(
+                backend="tsdb",
+                table=self.tsdb_path,
+                dfs=pd.DataFrame.from_dict([tsdb_drift_measures]),
+                index_cols=["timestamp", "endpoint_id", "record_type"],
+            )
+        except v3io_frames.errors.Error as err:
+            logger.warn(
+                "Could not write drift measures to TSDB",
+                err=err,
+                tsdb_path=self.tsdb_path,
+                endpoint=endpoint_id,
+            )
+
 
 def handler(context: mlrun.run.MLClientCtx):
     batch_processor = BatchProcessor(
         context=context,
         project=context.project,
-        model_monitoring_access_key=os.environ.get("MODEL_MONITORING_ACCESS_KEY"),
-        v3io_access_key=os.environ.get("V3IO_ACCESS_KEY"),
     )
     batch_processor.post_init()
     batch_processor.run()