mlrun 1.8.0rc29__py3-none-any.whl → 1.8.0rc31__py3-none-any.whl

mlrun/model_monitoring/applications/base.py

@@ -95,8 +95,8 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         sample_data: Optional[pd.DataFrame] = None,
         reference_data: Optional[pd.DataFrame] = None,
         endpoints: Optional[list[tuple[str, str]]] = None,
-        start: Optional[datetime] = None,
-        end: Optional[datetime] = None,
+        start: Optional[str] = None,
+        end: Optional[str] = None,
         base_period: Optional[int] = None,
     ):
         """
@@ -124,7 +124,6 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             return self.do_tracking(monitoring_context)
 
         if endpoints is not None:
-            start, end = self._validate_times(start, end, base_period)
             for window_start, window_end in self._window_generator(
                 start, end, base_period
             ):
@@ -137,43 +136,40 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
                             mm_constants.ApplicationEvent.END_INFER_TIME: window_end,
                         }
                     )
-                    context.log_result(
-                        f"{endpoint_name}_{window_start.isoformat()}_{window_end.isoformat()}",
-                        result,
+                    result_key = (
+                        f"{endpoint_name}_{window_start.isoformat()}_{window_end.isoformat()}"
+                        if window_start and window_end
+                        else endpoint_name
                     )
+                    context.log_result(result_key, result)
         else:
             return call_do_tracking()
 
-    @staticmethod
-    def _validate_times(
-        start: Optional[datetime],
-        end: Optional[datetime],
-        base_period: Optional[int],
-    ) -> tuple[datetime, datetime]:
-        if (start is None) or (end is None):
-            raise mlrun.errors.MLRunValueError(
-                "When `endpoint_names` is provided, you must also pass the start and end times"
-            )
-        if (base_period is not None) and not (
-            isinstance(base_period, int) and base_period > 0
-        ):
-            raise mlrun.errors.MLRunValueError(
-                "`base_period` must be a nonnegative integer - the number of minutes in a monitoring window"
-            )
-        return start, end
-
     @staticmethod
     def _window_generator(
-        start: datetime, end: datetime, base_period: Optional[int]
-    ) -> Iterator[tuple[datetime, datetime]]:
+        start: Optional[str], end: Optional[str], base_period: Optional[int]
+    ) -> Iterator[tuple[Optional[datetime], Optional[datetime]]]:
+        if start is None or end is None:
+            # A single window based on the `sample_data` input - see `_handler`.
+            yield None, None
+            return
+
+        start_dt = datetime.fromisoformat(start)
+        end_dt = datetime.fromisoformat(end)
+
         if base_period is None:
-            yield start, end
+            yield start_dt, end_dt
             return
 
+        if not isinstance(base_period, int) or base_period <= 0:
+            raise mlrun.errors.MLRunValueError(
+                "`base_period` must be a nonnegative integer - the number of minutes in a monitoring window"
+            )
+
         window_length = timedelta(minutes=base_period)
-        current_start_time = start
-        while current_start_time < end:
-            current_end_time = min(current_start_time + window_length, end)
+        current_start_time = start_dt
+        while current_start_time < end_dt:
+            current_end_time = min(current_start_time + window_length, end_dt)
             yield current_start_time, current_end_time
             current_start_time = current_end_time
 
@@ -369,13 +365,25 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
         :param requirements:      List of Python requirements to be installed in the image.
         :param requirements_file: Path to a Python requirements file to be installed in the image.
         :param endpoints:         A list of tuples of the model endpoint (name, uid) to get the data from.
-                                  If provided, you have to provide also the start and end times of the data to analyze.
-        :param start:             The start time of the sample data.
-        :param end:               The end time of the sample data.
+                                  If provided, and ``sample_data`` is not, you have to provide also the ``start`` and
+                                  ``end`` times of the data to analyze from the model endpoints.
+        :param start:             The start time of the endpoint's data, not included.
+                                  If you want the model endpoint's data at ``start`` included, you need to subtract a
+                                  small ``datetime.timedelta`` from it.
+        :param end:               The end time of the endpoint's data, included.
+                                  Please note: when ``start`` and ``end`` are set, they create a left-open time interval
+                                  ("window") :math:`(\\text{start}, \\text{end}]` that excludes the endpoint's data at
+                                  ``start`` and includes the data at ``end``:
+                                  :math:`\\text{start} < t \\leq \\text{end}`, :math:`t` is the time taken in the
+                                  window's data.
         :param base_period:       The window length in minutes. If ``None``, the whole window from ``start`` to ``end``
                                   is taken. If an integer is specified, the application is run from ``start`` to ``end``
                                   in ``base_period`` length windows, except for the last window that ends at ``end`` and
-                                  therefore may be shorter.
+                                  therefore may be shorter:
+                                  :math:`(\\text{start}, \\text{start} + \\text{base_period}],
+                                  (\\text{start} + \\text{base_period}, \\text{start} + 2\\cdot\\text{base_period}],
+                                  ..., (\\text{start} + m\\cdot\\text{base_period}, \\text{end}]`,
+                                  where :math:`m` is some positive integer.
 
         :returns: The output of the
                   :py:meth:`~mlrun.model_monitoring.applications.ModelMonitoringApplicationBase.do_tracking`
@@ -395,16 +403,23 @@ class ModelMonitoringApplicationBase(MonitoringApplicationToDict, ABC):
             project=project,
         )
 
-        params: dict[str, Union[list[tuple[str, str]], datetime, int, None]] = {}
+        params: dict[str, Union[list[tuple[str, str]], str, int, None]] = {}
         if endpoints:
-            start, end = cls._validate_times(start, end, base_period)
             params["endpoints"] = endpoints
-            params["start"] = start
-            params["end"] = end
-            params["base_period"] = base_period
+            if sample_data is None:
+                if start is None or end is None:
+                    raise mlrun.errors.MLRunValueError(
+                        "`start` and `end` times must be provided when `endpoints` "
+                        "is provided without `sample_data`"
+                    )
+                params["start"] = (
+                    start.isoformat() if isinstance(start, datetime) else start
+                )
+                params["end"] = end.isoformat() if isinstance(end, datetime) else end
+                params["base_period"] = base_period
         elif start or end or base_period:
             raise mlrun.errors.MLRunValueError(
-                "Custom start and end times or base_period are supported only with endpoints data"
+                "Custom `start` and `end` times or base_period are supported only with endpoints data"
             )
 
         inputs: dict[str, str] = {}

mlrun/model_monitoring/applications/context.py

@@ -111,9 +111,6 @@ class MonitoringApplicationContext:
         self.endpoint_name = cast(
             str, event.get(mm_constants.ApplicationEvent.ENDPOINT_NAME)
         )
-        self.output_stream_uri = cast(
-            str, event.get(mm_constants.ApplicationEvent.OUTPUT_STREAM_URI)
-        )
 
         self._feature_stats: Optional[FeatureStats] = feature_stats
         self._sample_df_stats: Optional[FeatureStats] = None

mlrun/model_monitoring/applications/results.py

@@ -22,7 +22,7 @@ from pydantic.v1.dataclasses import dataclass
 
 import mlrun.common.helpers
 import mlrun.common.model_monitoring.helpers
-import mlrun.common.schemas.model_monitoring.constants as mm_constant
+import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.utils.v3io_clients
 from mlrun.utils import logger
 
@@ -33,10 +33,10 @@ class _ModelMonitoringApplicationDataRes(ABC):
     name: str
 
     def __post_init__(self):
-        pat = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*")
+        pat = re.compile(mm_constants.RESULT_NAME_PATTERN)
         if not re.fullmatch(pat, self.name):
             raise mlrun.errors.MLRunValueError(
-                "Attribute name must comply with the regex `[a-zA-Z_][a-zA-Z0-9_]*`"
+                f"Attribute name must comply with the regex `{mm_constants.RESULT_NAME_PATTERN}`"
             )
 
     @abstractmethod
@@ -63,8 +63,8 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
 
     name: str
     value: float
-    kind: mm_constant.ResultKindApp
-    status: mm_constant.ResultStatusApp
+    kind: mm_constants.ResultKindApp
+    status: mm_constants.ResultStatusApp
     extra_data: dict = dataclasses.field(default_factory=dict)
 
     def to_dict(self):
@@ -74,11 +74,11 @@ class ModelMonitoringApplicationResult(_ModelMonitoringApplicationDataRes):
         :returns:    (dict) Dictionary representation of the result.
         """
         return {
-            mm_constant.ResultData.RESULT_NAME: self.name,
-            mm_constant.ResultData.RESULT_VALUE: self.value,
-            mm_constant.ResultData.RESULT_KIND: self.kind.value,
-            mm_constant.ResultData.RESULT_STATUS: self.status.value,
-            mm_constant.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
+            mm_constants.ResultData.RESULT_NAME: self.name,
+            mm_constants.ResultData.RESULT_VALUE: self.value,
+            mm_constants.ResultData.RESULT_KIND: self.kind.value,
+            mm_constants.ResultData.RESULT_STATUS: self.status.value,
+            mm_constants.ResultData.RESULT_EXTRA_DATA: json.dumps(self.extra_data),
         }
 
     @validator("extra_data")
@@ -118,8 +118,8 @@ class ModelMonitoringApplicationMetric(_ModelMonitoringApplicationDataRes):
         :returns:    (dict) Dictionary representation of the result.
         """
         return {
-            mm_constant.MetricData.METRIC_NAME: self.name,
-            mm_constant.MetricData.METRIC_VALUE: self.value,
+            mm_constants.MetricData.METRIC_NAME: self.name,
+            mm_constants.MetricData.METRIC_VALUE: self.value,
         }
 
 
@@ -134,7 +134,7 @@ class _ModelMonitoringApplicationStats(_ModelMonitoringApplicationDataRes):
 
     """
 
-    name: mm_constant.StatsKind
+    name: mm_constants.StatsKind
     timestamp: str
     stats: dict = dataclasses.field(default_factory=dict)
 
@@ -145,7 +145,7 @@ class _ModelMonitoringApplicationStats(_ModelMonitoringApplicationDataRes):
         :returns:    (dict) Dictionary representation of the result.
         """
         return {
-            mm_constant.StatsData.STATS_NAME: self.name,
-            mm_constant.StatsData.STATS: self.stats,
-            mm_constant.StatsData.TIMESTAMP: self.timestamp,
+            mm_constants.StatsData.STATS_NAME: self.name,
+            mm_constants.StatsData.STATS: self.stats,
+            mm_constants.StatsData.TIMESTAMP: self.timestamp,
         }

mlrun/model_monitoring/controller.py

@@ -27,15 +27,15 @@ import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.feature_store as fstore
 import mlrun.model_monitoring
+import mlrun.model_monitoring.helpers
 from mlrun.common.schemas import EndpointType
 from mlrun.common.schemas.model_monitoring.constants import (
     ControllerEvent,
     ControllerEventKind,
 )
-from mlrun.datastore import get_stream_pusher
 from mlrun.errors import err_to_str
 from mlrun.model_monitoring.db._schedules import ModelMonitoringSchedulesFile
-from mlrun.model_monitoring.helpers import batch_dict2timedelta, get_stream_path
+from mlrun.model_monitoring.helpers import batch_dict2timedelta
 from mlrun.utils import datetime_now, logger
 
 _SECONDS_IN_DAY = int(datetime.timedelta(days=1).total_seconds())
@@ -249,10 +249,11 @@ class MonitoringApplicationController:
         self._window_length = _get_window_length()
 
         self.model_monitoring_access_key = self._get_model_monitoring_access_key()
-        self.v3io_access_key = mlrun.get_secret_or_env("V3IO_ACCESS_KEY")
-        self.storage_options = None
-        if mlrun.mlconf.artifact_path.startswith("s3://"):
-            self.storage_options = mlrun.mlconf.get_s3_storage_options()
+        self.v3io_access_key = mlrun.mlconf.get_v3io_access_key()
+        store, _, _ = mlrun.store_manager.get_or_create_store(
+            mlrun.mlconf.artifact_path
+        )
+        self.storage_options = store.get_storage_options()
 
     @staticmethod
     def _get_model_monitoring_access_key() -> Optional[str]:
@@ -482,24 +483,23 @@ class MonitoringApplicationController:
             ),
             mm_constants.ApplicationEvent.ENDPOINT_ID: endpoint_id,
             mm_constants.ApplicationEvent.ENDPOINT_NAME: endpoint_name,
-            mm_constants.ApplicationEvent.OUTPUT_STREAM_URI: get_stream_path(
-                project=project,
-                function_name=mm_constants.MonitoringFunctionNames.WRITER,
-            ),
         }
         for app_name in applications_names:
             data.update({mm_constants.ApplicationEvent.APPLICATION_NAME: app_name})
-            stream_uri = get_stream_path(project=project, function_name=app_name)
+
+            app_stream = mlrun.model_monitoring.helpers.get_output_stream(
+                project=project,
+                function_name=app_name,
+                v3io_access_key=model_monitoring_access_key,
+            )
 
             logger.info(
                 "Pushing data to application stream",
                 endpoint_id=endpoint_id,
                 app_name=app_name,
-                stream_uri=stream_uri,
-            )
-            get_stream_pusher(stream_uri, access_key=model_monitoring_access_key).push(
-                [data]
+                app_stream_type=str(type(app_stream)),
             )
+            app_stream.push([data])
 
     def push_regular_event_to_controller_stream(self) -> None:
         """
@@ -628,10 +628,6 @@ class MonitoringApplicationController:
         :param feature_set_uri: the feature set uri string
         :param stream_access_key: access key to apply the model monitoring process.
         """
-        stream_uri = get_stream_path(
-            project=project,
-            function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
-        )
         event = {
             ControllerEvent.KIND.value: kind,
             ControllerEvent.PROJECT.value: project,
@@ -643,15 +639,18 @@ class MonitoringApplicationController:
             ControllerEvent.FEATURE_SET_URI.value: feature_set_uri,
             ControllerEvent.ENDPOINT_POLICY.value: endpoint_policy,
         }
+        controller_stream = mlrun.model_monitoring.helpers.get_output_stream(
+            project=project,
+            function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
+            v3io_access_key=stream_access_key,
+        )
         logger.info(
             "Pushing data to controller stream",
             event=event,
             endpoint_id=endpoint_id,
-            stream_uri=stream_uri,
-        )
-        get_stream_pusher(stream_uri, access_key=stream_access_key).push(
-            [event], partition_key=endpoint_id
+            controller_stream_type=str(type(controller_stream)),
         )
+        controller_stream.push([event], partition_key=endpoint_id)
 
     def _push_to_main_stream(self, event: dict, endpoint_id: str) -> None:
         """
@@ -659,17 +658,18 @@ class MonitoringApplicationController:
         :param event: event dictionary to push to stream
         :param endpoint_id: endpoint id string
         """
-        stream_uri = get_stream_path(project=event.get(ControllerEvent.PROJECT))
-
+        mm_stream = mlrun.model_monitoring.helpers.get_output_stream(
+            project=event.get(ControllerEvent.PROJECT),
+            function_name=mm_constants.MonitoringFunctionNames.APPLICATION_CONTROLLER,
+            v3io_access_key=self.v3io_access_key,
+        )
         logger.info(
             "Pushing data to main stream, NOP event is been generated",
             event=json.dumps(event),
             endpoint_id=endpoint_id,
-            stream_uri=stream_uri,
-        )
-        get_stream_pusher(stream_uri, access_key=self.model_monitoring_access_key).push(
-            [event], partition_key=endpoint_id
+            mm_stream_type=str(type(mm_stream)),
         )
+        mm_stream.push([event], partition_key=endpoint_id)
 
 
 def handler(context: nuclio_sdk.Context, event: nuclio_sdk.Event) -> None:
@@ -685,12 +685,16 @@ def handler(context: nuclio_sdk.Context, event: nuclio_sdk.Event) -> None:
         trigger_kind=event.trigger.kind,
     )
 
-    if event.trigger.kind == "http":
+    if event.trigger.kind in mm_constants.CRON_TRIGGER_KINDS:
         # Runs controller chief:
         context.user_data.monitor_app_controller.push_regular_event_to_controller_stream()
-    else:
+    elif event.trigger.kind in mm_constants.STREAM_TRIGGER_KINDS:
         # Runs controller worker:
         context.user_data.monitor_app_controller.run(event)
+    else:
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            "Wrong trigger kind for model monitoring controller"
+        )
 
 
 def init_context(context):

mlrun/model_monitoring/db/tsdb/__init__.py

@@ -19,6 +19,7 @@ import mlrun.common.schemas.secret
 import mlrun.datastore.datastore_profile
 import mlrun.errors
 import mlrun.model_monitoring.helpers
+from mlrun.datastore.datastore_profile import DatastoreProfile
 
 from .base import TSDBConnector
 
@@ -29,10 +30,13 @@ class ObjectTSDBFactory(enum.Enum):
     v3io_tsdb = "v3io-tsdb"
     tdengine = "tdengine"
 
-    def to_tsdb_connector(self, project: str, **kwargs) -> TSDBConnector:
+    def to_tsdb_connector(
+        self, project: str, profile: DatastoreProfile, **kwargs
+    ) -> TSDBConnector:
         """
         Return a TSDBConnector object based on the provided enum value.
         :param project: The name of the project.
+        :param profile: Datastore profile containing DSN and credentials for TSDB connection
         :return: `TSDBConnector` object.
         """
 
@@ -51,7 +55,7 @@ class ObjectTSDBFactory(enum.Enum):
 
         from .tdengine.tdengine_connector import TDEngineConnector
 
-        return TDEngineConnector(project=project, **kwargs)
+        return TDEngineConnector(project=project, profile=profile, **kwargs)
 
     @classmethod
     def _missing_(cls, value: typing.Any):
@@ -87,12 +91,10 @@ def get_tsdb_connector(
     kwargs = {}
     if isinstance(profile, mlrun.datastore.datastore_profile.DatastoreProfileV3io):
         tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.V3IO_TSDB
-        kwargs["v3io_access_key"] = profile.v3io_access_key
     elif isinstance(
         profile, mlrun.datastore.datastore_profile.TDEngineDatastoreProfile
     ):
         tsdb_connector_type = mlrun.common.schemas.model_monitoring.TSDBTarget.TDEngine
-        kwargs["connection_string"] = profile.dsn()
     else:
         extra_message = (
             ""
@@ -109,4 +111,6 @@ def get_tsdb_connector(
     tsdb_connector_factory = ObjectTSDBFactory(tsdb_connector_type)
 
     # Convert into TSDB connector object
-    return tsdb_connector_factory.to_tsdb_connector(project=project, **kwargs)
+    return tsdb_connector_factory.to_tsdb_connector(
+        project=project, profile=profile, **kwargs
+    )