mlrun 1.8.0rc48__py3-none-any.whl → 1.8.0rc49__py3-none-any.whl

mlrun/model_monitoring/applications/context.py

@@ -23,12 +23,13 @@ import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.errors
 import mlrun.feature_store as fstore
+import mlrun.feature_store.feature_set as fs
 import mlrun.features
 import mlrun.serving
 import mlrun.utils
 from mlrun.artifacts import Artifact, DatasetArtifact, ModelArtifact, get_model
 from mlrun.common.model_monitoring.helpers import FeatureStats
-from mlrun.common.schemas import FeatureSet, ModelEndpoint
+from mlrun.common.schemas import ModelEndpoint
 from mlrun.model_monitoring.helpers import (
     calculate_inputs_statistics,
 )
@@ -41,7 +42,6 @@ class _ArtifactsLogger(Protocol):
 
     def log_artifact(self, *args, **kwargs) -> Artifact: ...
     def log_dataset(self, *args, **kwargs) -> DatasetArtifact: ...
-    def log_model(self, *args, **kwargs) -> ModelArtifact: ...
 
 
 class MonitoringApplicationContext:
@@ -59,7 +59,7 @@ class MonitoringApplicationContext:
         model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
-        feature_sets_dict: Optional[dict[str, FeatureSet]] = None,
+        feature_sets_dict: Optional[dict[str, fs.FeatureSet]] = None,
     ) -> None:
         """
         The :code:`MonitoringApplicationContext` object holds all the relevant information for the
@@ -124,9 +124,13 @@ class MonitoringApplicationContext:
         self._model_endpoint: Optional[ModelEndpoint] = (
             model_endpoint_dict.get(self.endpoint_id) if model_endpoint_dict else None
         )
-        self._feature_set: Optional[FeatureSet] = (
+        self._feature_set: Optional[fs.FeatureSet] = (
             feature_sets_dict.get(self.endpoint_id) if feature_sets_dict else None
         )
+        store, _, _ = mlrun.store_manager.get_or_create_store(
+            mlrun.mlconf.artifact_path
+        )
+        self.storage_options = store.get_storage_options()
 
     @classmethod
     def _from_ml_ctx(
@@ -169,7 +173,7 @@ class MonitoringApplicationContext:
         model_endpoint_dict: Optional[dict[str, ModelEndpoint]] = None,
         sample_df: Optional[pd.DataFrame] = None,
         feature_stats: Optional[FeatureStats] = None,
-        feature_sets_dict: Optional[dict[str, FeatureSet]] = None,
+        feature_sets_dict: Optional[dict[str, fs.FeatureSet]] = None,
     ) -> "MonitoringApplicationContext":
         nuclio_logger = graph_context.logger
         artifacts_logger = graph_context.project_obj
@@ -192,13 +196,14 @@ class MonitoringApplicationContext:
         )
 
     def _get_default_labels(self) -> dict[str, str]:
-        return {
+        labels = {
             mlrun_constants.MLRunInternalLabels.runner_pod: socket.gethostname(),
             mlrun_constants.MLRunInternalLabels.producer_type: "model-monitoring-app",
             mlrun_constants.MLRunInternalLabels.app_name: self.application_name,
             mlrun_constants.MLRunInternalLabels.endpoint_id: self.endpoint_id,
             mlrun_constants.MLRunInternalLabels.endpoint_name: self.endpoint_name,
         }
+        return {key: value for key, value in labels.items() if value is not None}
 
     def _add_default_labels(self, labels: Optional[dict[str, str]]) -> dict[str, str]:
         """Add the default labels to logged artifacts labels"""
@@ -221,22 +226,13 @@ class MonitoringApplicationContext:
                     "You can either provide the sample dataframe directly, the model endpoint's details and times, "
                     "or adapt the application's logic to not access the sample dataframe."
                 )
-            feature_set = self.feature_set
-            features = [f"{feature_set.metadata.name}.*"]
-            vector = fstore.FeatureVector(
-                name=f"{self.endpoint_id}_vector",
-                features=features,
-                with_indexes=True,
-            )
-            vector.metadata.tag = self.application_name
-            vector.feature_set_objects = {feature_set.metadata.name: feature_set}
-
-            offline_response = vector.get_offline_features(
+            df = self.feature_set.to_dataframe(
                 start_time=self.start_infer_time,
                 end_time=self.end_infer_time,
-                timestamp_for_filtering=mm_constants.FeatureSetFeatures.time_stamp(),
+                time_column=mm_constants.EventFieldType.TIMESTAMP,
+                storage_options=self.storage_options,
             )
-            self._sample_df = offline_response.to_dataframe().reset_index(drop=True)
+            self._sample_df = df.reset_index(drop=True)
         return self._sample_df
 
     @property
@@ -259,7 +255,7 @@ class MonitoringApplicationContext:
         return self._model_endpoint
 
     @property
-    def feature_set(self) -> FeatureSet:
+    def feature_set(self) -> fs.FeatureSet:
         if not self._feature_set and self.model_endpoint:
             self._feature_set = fstore.get_feature_set(
                 self.model_endpoint.spec.monitoring_feature_set_uri
@@ -329,13 +325,23 @@ class MonitoringApplicationContext:
         upload: Optional[bool] = None,
         labels: Optional[dict[str, str]] = None,
         target_path: Optional[str] = None,
+        unique_per_endpoint: bool = True,
         **kwargs,
     ) -> Artifact:
         """
         Log an artifact.
         See :func:`~mlrun.projects.MlrunProject.log_artifact` for the documentation.
+        :param unique_per_endpoint: by default True, we will log different artifact for each model endpoint,
+        set to False without changing item key will cause artifact override
         """
         labels = self._add_default_labels(labels)
+        # By default, we want to log different artifact for each model endpoint
+        endpoint_id = labels.get(mlrun_constants.MLRunInternalLabels.endpoint_id, "")
+        if unique_per_endpoint and isinstance(item, str):
+            item = f"{item}-{endpoint_id}" if endpoint_id else item
+        elif unique_per_endpoint:  # isinstance(item, Artifact) is True
+            item.key = f"{item.key}-{endpoint_id}" if endpoint_id else item.key
+
         return self._artifacts_logger.log_artifact(
             item,
             body=body,
@@ -364,13 +370,21 @@ class MonitoringApplicationContext:
         target_path="",
         extra_data=None,
         label_column: Optional[str] = None,
+        unique_per_endpoint: bool = True,
         **kwargs,
     ) -> DatasetArtifact:
         """
         Log a dataset artifact.
         See :func:`~mlrun.projects.MlrunProject.log_dataset` for the documentation.
+        :param unique_per_endpoint: by default True, we will log different dataset for each model endpoint,
+        set to False without changing item key will cause dataset override
         """
         labels = self._add_default_labels(labels)
+        # By default, we want to log different artifact for each model endpoint
+        endpoint_id = labels.get(mlrun_constants.MLRunInternalLabels.endpoint_id, "")
+        if unique_per_endpoint and isinstance(key, str):
+            key = f"{key}-{endpoint_id}" if endpoint_id else key
+
         return self._artifacts_logger.log_dataset(
             key,
             df,
@@ -387,54 +401,3 @@ class MonitoringApplicationContext:
             label_column=label_column,
             **kwargs,
         )
-
-    def log_model(
-        self,
-        key,
-        body=None,
-        framework="",
-        tag="",
-        model_dir=None,
-        model_file=None,
-        algorithm=None,
-        metrics=None,
-        parameters=None,
-        artifact_path=None,
-        upload=None,
-        labels=None,
-        inputs: Optional[list[mlrun.features.Feature]] = None,
-        outputs: Optional[list[mlrun.features.Feature]] = None,
-        feature_vector: Optional[str] = None,
-        feature_weights: Optional[list] = None,
-        training_set=None,
-        label_column=None,
-        extra_data=None,
-        **kwargs,
-    ) -> ModelArtifact:
-        """
-        Log a model artifact.
-        See :func:`~mlrun.projects.MlrunProject.log_model` for the documentation.
-        """
-        labels = self._add_default_labels(labels)
-        return self._artifacts_logger.log_model(
-            key,
-            body=body,
-            framework=framework,
-            tag=tag,
-            model_dir=model_dir,
-            model_file=model_file,
-            algorithm=algorithm,
-            metrics=metrics,
-            parameters=parameters,
-            artifact_path=artifact_path,
-            upload=upload,
-            labels=labels,
-            inputs=inputs,
-            outputs=outputs,
-            feature_vector=feature_vector,
-            feature_weights=feature_weights,
-            training_set=training_set,
-            label_column=label_column,
-            extra_data=extra_data,
-            **kwargs,
-        )

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -11,10 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import asyncio
 from datetime import datetime, timedelta
 from threading import Lock
-from typing import Callable, Literal, Optional, Union
+from typing import Callable, Final, Literal, Optional, Union
 
 import pandas as pd
 import taosws
@@ -24,6 +25,7 @@ from taoswswrap.tdengine_connection import (
 )
 
 import mlrun.common.schemas.model_monitoring as mm_schemas
+import mlrun.common.types
 import mlrun.model_monitoring.db.tsdb.tdengine.schemas as tdengine_schemas
 import mlrun.model_monitoring.db.tsdb.tdengine.stream_graph_steps
 from mlrun.datastore.datastore_profile import DatastoreProfile
@@ -35,6 +37,19 @@ _connection = None
 _connection_lock = Lock()
 
 
+class TDEngineTimestampPrecision(mlrun.common.types.StrEnum):
+    """
+    The timestamp precision for the TDEngine database.
+    For more information, see:
+    https://docs.tdengine.com/tdengine-reference/sql-manual/data-types/#timestamp
+    https://docs.tdengine.com/tdengine-reference/sql-manual/manage-databases/#create-database
+    """
+
+    MILLISECOND = "ms"  # TDEngine's default
+    MICROSECOND = "us"  # MLRun's default
+    NANOSECOND = "ns"
+
+
 class TDEngineConnector(TSDBConnector):
     """
     Handles the TSDB operations when the TSDB connector is of type TDEngine.
@@ -47,12 +62,17 @@ class TDEngineConnector(TSDBConnector):
         self,
         project: str,
         profile: DatastoreProfile,
+        timestamp_precision: TDEngineTimestampPrecision = TDEngineTimestampPrecision.MICROSECOND,
         **kwargs,
     ):
         super().__init__(project=project)
 
         self._tdengine_connection_profile = profile
 
+        self._timestamp_precision: Final = (  # cannot be changed after initialization
+            timestamp_precision
+        )
+
         self._init_super_tables()
 
         self._run_directly = (
@@ -105,7 +125,7 @@ class TDEngineConnector(TSDBConnector):
         """Create the database if it does not exist."""
         self.connection.prefix_statements = []
         self.connection.run(
-            statements=f"CREATE DATABASE IF NOT EXISTS {self.database}",
+            statements=f"CREATE DATABASE IF NOT EXISTS {self.database} PRECISION '{self._timestamp_precision}'",
             timeout=self._timeout,
             retries=self._retries,
         )
@@ -180,6 +200,7 @@ class TDEngineConnector(TSDBConnector):
             columns=columns,
             subtable=table_name,
             values=event,
+            timestamp_precision=self._timestamp_precision,
         )
 
         self.connection.run(

mlrun/serving/states.py

@@ -1833,7 +1833,7 @@ def params_to_step(
     class_args = class_args or {}
 
     if isinstance(class_name, QueueStep):
-        if not name or class_name.name:
+        if not (name or class_name.name):
             raise MLRunInvalidArgumentError("queue name must be specified")
 
         step = class_name
@@ -1854,7 +1854,11 @@ def params_to_step(
     elif class_name and hasattr(class_name, "to_dict"):
         struct = class_name.to_dict()
         kind = struct.get("kind", StepKinds.task)
-        name = name or struct.get("name", struct.get("class_name"))
+        name = (
+            name
+            or struct.get("name", struct.get("class_name"))
+            or class_name.to_dict(["name"]).get("name")
+        )
         cls = classes_map.get(kind, RootFlowStep)
         step = cls.from_dict(struct)
         step.function = function

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "c6cc999a67744a5b12083066cb733342de7a8700",
-  "version": "1.8.0-rc48"
+  "git_commit": "e4003fc398511d0bc1997ab263a55b9cd70d199a",
+  "version": "1.8.0-rc49"
 }

{mlrun-1.8.0rc48.dist-info → mlrun-1.8.0rc49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.8.0rc48
+Version: 1.8.0rc49
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -44,7 +44,7 @@ Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
 Requires-Dist: fsspec<2024.7,>=2023.9.2
 Requires-Dist: v3iofs~=0.1.17
-Requires-Dist: storey~=1.8.10
+Requires-Dist: storey~=1.8.11
 Requires-Dist: inflection~=0.5.0
 Requires-Dist: python-dotenv~=1.0
 Requires-Dist: setuptools>=75.2
@@ -99,7 +99,7 @@ Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
 Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
-Requires-Dist: taoswswrap~=0.3.4; extra == "tdengine"
+Requires-Dist: taoswswrap~=0.3.5; extra == "tdengine"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python~=3.7; extra == "snowflake"
 Provides-Extra: kfp18
@@ -152,7 +152,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
 Requires-Dist: sqlalchemy~=1.4; extra == "all"
 Requires-Dist: taos-ws-py==0.3.2; extra == "all"
-Requires-Dist: taoswswrap~=0.3.4; extra == "all"
+Requires-Dist: taoswswrap~=0.3.5; extra == "all"
 Provides-Extra: complete
 Requires-Dist: adlfs==2023.9.0; extra == "complete"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
@@ -184,7 +184,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
-Requires-Dist: taoswswrap~=0.3.4; extra == "complete"
+Requires-Dist: taoswswrap~=0.3.5; extra == "complete"
 Provides-Extra: complete-api
 Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
@@ -229,7 +229,7 @@ Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
 Requires-Dist: sqlalchemy~=1.4; extra == "complete-api"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete-api"
-Requires-Dist: taoswswrap~=0.3.4; extra == "complete-api"
+Requires-Dist: taoswswrap~=0.3.5; extra == "complete-api"
 Requires-Dist: timelength~=1.1; extra == "complete-api"
 Requires-Dist: uvicorn~=0.32.1; extra == "complete-api"
 Dynamic: author

{mlrun-1.8.0rc48.dist-info → mlrun-1.8.0rc49.dist-info}/RECORD

@@ -228,7 +228,7 @@ mlrun/model_monitoring/writer.py,sha256=ibbhvfSHb8Reqlb7RGFEAUNM4iTyK1gk8-2m46mP
 mlrun/model_monitoring/applications/__init__.py,sha256=xDBxkBjl-whHSG_4t1mLkxiypLH-fzn8TmAW9Mjo2uI,759
 mlrun/model_monitoring/applications/_application_steps.py,sha256=PxULZznKW66Oq-fKaraOAbsTuGnV0zgXh6_91wX3KUo,8367
 mlrun/model_monitoring/applications/base.py,sha256=7XL12idItWkoE3CJ_48F6cwVx5pJH3bgfG92hb8LcN8,24872
-mlrun/model_monitoring/applications/context.py,sha256=DKUDOfN4iY5wpOMjfsarx4pVN9A1sORyu7y2EEKEvMs,16964
+mlrun/model_monitoring/applications/context.py,sha256=l7HculkdcrWbMLjy9tsxEhm-1X6zxJE5Blk-U7yc-5I,16449
 mlrun/model_monitoring/applications/histogram_data_drift.py,sha256=09t0tfC35W0SeJA3fzN29pJiB6G-V_8GlcvULVq6H9Q,15179
 mlrun/model_monitoring/applications/results.py,sha256=_qmj6TWT0SR2bi7gUyRKBU418eGgGoLW2_hTJ7S-ock,5782
 mlrun/model_monitoring/applications/evidently/__init__.py,sha256=-DqdPnBSrjZhFvKOu_Ie3MiFvlur9sPTZpZ1u0_1AE8,690
@@ -242,7 +242,7 @@ mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsV
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=EslhaR65jfeNdD5Ibk-3Hb4e5r5qYPfHb9rTChX3sG0,12689
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Uadj0UvAmln2MxDWod-kAzau1uNlqZh981rPhbUH_5M,2857
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=5jgimfu2-omy8Cnnby7GpgB_MWEp9mmLX0zpbGC2JZ8,37934
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=rpE-RB5sqe5GAkd9MZB4JFzSP1skxs0loC5hhAkNgKk,38747
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=_-zo9relCDtjGgievxAcAP9gVN9nDWs8BzGtFwTjb9M,6284
 mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=IzdThNwWMBWo0D0VzXV-WVvGg-z7Y9e8ke8_LYJTeVA,46214
@@ -306,7 +306,7 @@ mlrun/serving/remote.py,sha256=gxJkj_J3j-sZcVUbUzbAmJafP6t6y4NVFsu0kWmYngA,18818
 mlrun/serving/routers.py,sha256=SY6AsaiSnh8ssXq8hQE2z9MYapOxFOFJBx9QomiZMO8,53915
 mlrun/serving/server.py,sha256=KiNhW0nTV5STZPzR6kEAUFVzCCAX8qv0g9AoCopARrM,23429
 mlrun/serving/serving_wrapper.py,sha256=R670-S6PX_d5ER6jiHtRvacuPyFzQH0mEf2K0sBIIOM,836
-mlrun/serving/states.py,sha256=UWiE85MB_SK3rgzWgNqQU2MKeyN2yF2BCvMcMAqLMTs,73247
+mlrun/serving/states.py,sha256=Hh3FBoQbHoO4KiofHfSwi_aUx7mQ26iXpKijcGiDJ6c,73341
 mlrun/serving/utils.py,sha256=k2EIYDWHUGkE-IBI6T0UNT32fw-KySsccIJM_LObI00,4171
 mlrun/serving/v1_serving.py,sha256=c6J_MtpE-Tqu00-6r4eJOCO6rUasHDal9W2eBIcrl50,11853
 mlrun/serving/v2_serving.py,sha256=b3C5Utv2_AOPrH_hPi3NarjNbAK3kRoeIfqMU4qNuUo,25362
@@ -340,11 +340,11 @@ mlrun/utils/notifications/notification/mail.py,sha256=ZyJ3eqd8simxffQmXzqd3bgbAq
 mlrun/utils/notifications/notification/slack.py,sha256=eQvmctTh6wIG5xVOesLLV9S1-UUCu5UEQ9JIJOor3ts,7183
 mlrun/utils/notifications/notification/webhook.py,sha256=NeyIMSBojjjTJaUHmPbxMByp34GxYkl1-16NqzU27fU,4943
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=VgW5oJpqfzAo9q08BKtzegYzji4XRJ88Lp0aBY1vEaw,89
+mlrun/utils/version/version.json,sha256=b0zzWDuszZWG3WY9mCKveicaqnMi13s_l0qH8nfwiWA,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.8.0rc48.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.8.0rc48.dist-info/METADATA,sha256=T7yAGdC4Yb-8_y8KkzMQ3oLLlK1vJNQ9sEbfMlZz7oM,26009
-mlrun-1.8.0rc48.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-mlrun-1.8.0rc48.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.8.0rc48.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.8.0rc48.dist-info/RECORD,,
+mlrun-1.8.0rc49.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.8.0rc49.dist-info/METADATA,sha256=Jznk4YH97II00VXkbNbjBzb2TybeurNx-a_S--ZYVHk,26009
+mlrun-1.8.0rc49.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+mlrun-1.8.0rc49.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.8.0rc49.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.8.0rc49.dist-info/RECORD,,