mlrun 1.7.0rc52__py3-none-any.whl → 1.7.0rc57__py3-none-any.whl

mlrun/errors.py

@@ -140,7 +140,13 @@ def err_to_str(err):
         error_strings.append(err_msg)
         err = err.__cause__
 
-    return ", caused by: ".join(error_strings)
+    err_msg = ", caused by: ".join(error_strings)
+
+    # in case the error string is longer than 32k, we truncate it
+    # the truncation takes the first 16k, then the last 16k characters
+    if len(err_msg) > 32_000:
+        err_msg = err_msg[:16_000] + "...truncated..." + err_msg[-16_000:]
+    return err_msg
 
 
 # Specific Errors

mlrun/model_monitoring/applications/_application_steps.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import traceback
 from typing import Any, Optional, Union
 
 import mlrun.common.schemas.alert as alert_objects
@@ -161,7 +162,10 @@ class _ApplicationErrorHandler(StepToDict):
         :param event: Application event.
         """
 
-        logger.error(f"Error in application step: {event}")
+        exception_with_trace = "".join(
+            traceback.format_exception(None, event.error, event.error.__traceback__)
+        )
+        logger.error(f"Error in application step: {exception_with_trace}")
 
         event_data = alert_objects.Event(
             kind=alert_objects.EventKind.MM_APP_FAILED,

mlrun/model_monitoring/controller.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import concurrent.futures
 import datetime
 import json
@@ -25,7 +24,9 @@ import nuclio
 import mlrun
 import mlrun.common.schemas.model_monitoring.constants as mm_constants
 import mlrun.data_types.infer
+import mlrun.feature_store as fstore
 import mlrun.model_monitoring.db.stores
+from mlrun.config import config as mlconf
 from mlrun.datastore import get_stream_pusher
 from mlrun.errors import err_to_str
 from mlrun.model_monitoring.helpers import (
@@ -286,9 +287,9 @@ class MonitoringApplicationController:
         )
 
         self.model_monitoring_access_key = self._get_model_monitoring_access_key()
-        self.tsdb_connector = mlrun.model_monitoring.get_tsdb_connector(
-            project=self.project
-        )
+        self.storage_options = None
+        if mlconf.artifact_path.startswith("s3://"):
+            self.storage_options = mlrun.mlconf.get_s3_storage_options()
 
     @staticmethod
     def _get_model_monitoring_access_key() -> Optional[str]:
@@ -375,7 +376,7 @@ class MonitoringApplicationController:
                         batch_window_generator=self._batch_window_generator,
                         project=self.project,
                         model_monitoring_access_key=self.model_monitoring_access_key,
-                        tsdb_connector=self.tsdb_connector,
+                        storage_options=self.storage_options,
                     )
 
     @classmethod
@@ -386,7 +387,7 @@ class MonitoringApplicationController:
         batch_window_generator: _BatchWindowGenerator,
         project: str,
         model_monitoring_access_key: str,
-        tsdb_connector: mlrun.model_monitoring.db.tsdb.TSDBConnector,
+        storage_options: Optional[dict] = None,
     ) -> None:
         """
         Process a model endpoint and trigger the monitoring applications. This function running on different process
@@ -398,11 +399,13 @@ class MonitoringApplicationController:
         :param batch_window_generator:      (_BatchWindowGenerator) An object that generates _BatchWindow objects.
         :param project:                     (str) Project name.
         :param model_monitoring_access_key: (str) Access key to apply the model monitoring process.
-        :param tsdb_connector:              (mlrun.model_monitoring.db.tsdb.TSDBConnector) TSDB connector
+        :param storage_options:             (dict) Storage options for reading the infer parquet files.
         """
         endpoint_id = endpoint[mm_constants.EventFieldType.UID]
-        # if false the endpoint represent batch infer step.
         has_stream = endpoint[mm_constants.EventFieldType.STREAM_PATH] != ""
+        m_fs = fstore.get_feature_set(
+            endpoint[mm_constants.EventFieldType.FEATURE_SET_URI]
+        )
         try:
             for application in applications_names:
                 batch_window = batch_window_generator.get_batch_window(
@@ -415,12 +418,13 @@ class MonitoringApplicationController:
                 )
 
                 for start_infer_time, end_infer_time in batch_window.get_intervals():
-                    prediction_metric = tsdb_connector.read_predictions(
-                        endpoint_id=endpoint_id,
-                        start=start_infer_time,
-                        end=end_infer_time,
+                    df = m_fs.to_dataframe(
+                        start_time=start_infer_time,
+                        end_time=end_infer_time,
+                        time_column=mm_constants.EventFieldType.TIMESTAMP,
+                        storage_options=storage_options,
                     )
-                    if not prediction_metric.data and has_stream:
+                    if len(df) == 0:
                         logger.info(
                             "No data found for the given interval",
                             start=start_infer_time,
@@ -442,6 +446,7 @@ class MonitoringApplicationController:
                             applications_names=[application],
                             model_monitoring_access_key=model_monitoring_access_key,
                         )
+
         except Exception:
             logger.exception(
                 "Encountered an exception",

mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py

@@ -217,7 +217,7 @@ class TDEngineConnector(TSDBConnector):
                 drop_statements.append(
                     self.tables[table]._drop_subtable_query(subtable=subtable[0])
                 )
-            self.connection.run(statements=[drop_statements])
+            self.connection.run(statements=drop_statements)
         logger.debug(
             "Deleted all project resources using the TDEngine connector",
             project=self.project,

mlrun/projects/pipelines.py

@@ -593,7 +593,7 @@ class _KFPRunner(_PipelineRunner):
             logger.warning(
                 "Setting notifications on kfp pipeline runner uses old notification behavior. "
                 "Notifications will only be sent if you wait for pipeline completion. "
-                "To use the new notification behavior, use the remote pipeline runner."
+                "Some of the features (like setting message or severity level) are not supported."
             )
             # for start message, fallback to old notification behavior
             for notification in notifications or []:

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "2c302d1e0b0ca25ab0b515fe43e3df47ebe8bd62",
-  "version": "1.7.0-rc52"
+  "git_commit": "e4ec8c2f525543b258cc0c190f2cf29d49f22032",
+  "version": "1.7.0-rc57"
 }

{mlrun-1.7.0rc52.dist-info → mlrun-1.7.0rc57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.0rc52
+Version: 1.7.0rc57
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -85,7 +85,7 @@ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'all'
 Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'all'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'all'
 Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'all'
-Requires-Dist: taoswswrap ~=0.1.0 ; extra == 'all'
+Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'all'
 Provides-Extra: api
 Requires-Dist: uvicorn ~=0.27.1 ; extra == 'api'
 Requires-Dist: dask-kubernetes ~=0.11.0 ; extra == 'api'
@@ -139,7 +139,7 @@ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'complete'
 Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete'
 Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'complete'
-Requires-Dist: taoswswrap ~=0.1.0 ; extra == 'complete'
+Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'complete'
 Provides-Extra: complete-api
 Requires-Dist: adlfs ==2023.9.0 ; extra == 'complete-api'
 Requires-Dist: aiobotocore <2.16,>=2.5.0 ; extra == 'complete-api'
@@ -177,7 +177,7 @@ Requires-Dist: s3fs <2024.7,>=2023.9.2 ; extra == 'complete-api'
 Requires-Dist: snowflake-connector-python ~=3.7 ; extra == 'complete-api'
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'complete-api'
 Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'complete-api'
-Requires-Dist: taoswswrap ~=0.1.0 ; extra == 'complete-api'
+Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'complete-api'
 Requires-Dist: timelength ~=1.1 ; extra == 'complete-api'
 Requires-Dist: uvicorn ~=0.27.1 ; extra == 'complete-api'
 Requires-Dist: memray ~=1.12 ; (sys_platform != "win32") and extra == 'complete-api'
@@ -213,7 +213,7 @@ Provides-Extra: sqlalchemy
 Requires-Dist: sqlalchemy ~=1.4 ; extra == 'sqlalchemy'
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py ==0.3.2 ; extra == 'tdengine'
-Requires-Dist: taoswswrap ~=0.1.0 ; extra == 'tdengine'
+Requires-Dist: taoswswrap ~=0.2.0 ; extra == 'tdengine'
 
 <a id="top"></a>
 [![Build Status](https://github.com/mlrun/mlrun/actions/workflows/build.yaml/badge.svg?branch=development)](https://github.com/mlrun/mlrun/actions/workflows/build.yaml?query=branch%3Adevelopment)

{mlrun-1.7.0rc52.dist-info → mlrun-1.7.0rc57.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mlrun/__init__.py,sha256=y08M1JcKXy5-9_5WaI9fn5aV5BxIQ5QkbduJK0OxWbA,7470
 mlrun/__main__.py,sha256=mC_Izs4kuHUHQi88QJFLN22n1kbygGM0wAirjNt7uj4,45938
 mlrun/config.py,sha256=NJG59Rl_5-mwgCdPDboRhjHD1ujW9ITYL7gtCbSMkM8,67308
-mlrun/errors.py,sha256=nY23dns_kTzbOrelJf0FyxLw5mglv7jo4Sx3efKS9Fs,7798
+mlrun/errors.py,sha256=G8GP4_wb3v2UEbiAS8OlamC7nYJNzbSvQ3sViZlyYhk,8063
 mlrun/execution.py,sha256=nXvvN8euzjuxhJouJD8VxfK0keTTA6UoMrcD_17AL-4,44252
 mlrun/features.py,sha256=m17K_3l9Jktwb9dOwlHLTAPTlemsWrRF7dJhXUX0iJU,15429
 mlrun/k8s_utils.py,sha256=mRQMs6NzPq36vx1n5_2BfFapXysc8wv3NcrZ77_2ANA,8949
@@ -213,7 +213,7 @@ mlrun/launcher/local.py,sha256=pP9-ZrNL8OnNDEiXTAKAZQnmLpS_mCc2v-mJw329eks,11269
 mlrun/launcher/remote.py,sha256=tGICSfWtvUHeR31mbzy6gqHejmDxjPUgjtxXTWhRubg,7699
 mlrun/model_monitoring/__init__.py,sha256=dm5_j0_pwqrdzFwTaEtGnKfv2nVpNaM56nBI-oqLbNU,879
 mlrun/model_monitoring/api.py,sha256=2EHCzB_5sCDgalYPkrFbI01cSO7LVWBv9yWoooJ-a0g,28106
-mlrun/model_monitoring/controller.py,sha256=dvqEyoE-iCd2jqDeoUpcrQFUeoTME58i3Wa2MhYi57k,20444
+mlrun/model_monitoring/controller.py,sha256=m2Z2Nwqj3A3byxrV6PAbkqzT0AsNxmlNqOk61nNJxOc,20637
 mlrun/model_monitoring/features_drift_table.py,sha256=c6GpKtpOJbuT1u5uMWDL_S-6N4YPOmlktWMqPme3KFY,25308
 mlrun/model_monitoring/helpers.py,sha256=KsbSH0kEjCPajvLUpv3q5GWyvx0bZj-JkghGJlzbLZI,12757
 mlrun/model_monitoring/model_endpoint.py,sha256=7VX0cBATqLsA4sSinDzouf41ndxqh2mf5bO9BW0G5Z4,4017
@@ -221,7 +221,7 @@ mlrun/model_monitoring/stream_processing.py,sha256=0eu1Gq1Obq87LFno6eIZ55poXoFae
 mlrun/model_monitoring/tracking_policy.py,sha256=sQq956akAQpntkrJwIgFWcEq-JpyVcg0FxgNa4h3V70,5502
 mlrun/model_monitoring/writer.py,sha256=TrBwngRmdwr67De71UCcCFsJOfcqQe8jDp0vkBvGf0o,10177
 mlrun/model_monitoring/applications/__init__.py,sha256=QYvzgCutFdAkzqKPD3mvkX_3c1X4tzd-kW8ojUOE9ic,889
-mlrun/model_monitoring/applications/_application_steps.py,sha256=fvZbtat7eXe5mo927_jyhq4BqWCapKZn7OVjptepIAI,7055
+mlrun/model_monitoring/applications/_application_steps.py,sha256=i6gyVaZHif0xEWl60QoU4VJCJx5-KRVdR1djUKuLBjI,7222
 mlrun/model_monitoring/applications/base.py,sha256=uzc14lFlwTJnL0p2VBCzmp-CNoHd73cK_Iz0YHC1KAs,4380
 mlrun/model_monitoring/applications/context.py,sha256=vOZ_ZgUuy5UsNe22-puJSt7TB32HiZtqBdN1hegykuQ,12436
 mlrun/model_monitoring/applications/evidently_base.py,sha256=FSzmoDZP8EiSQ3tq5RmU7kJ6edh8bWaKQh0rBORjODY,5099
@@ -245,7 +245,7 @@ mlrun/model_monitoring/db/tsdb/helpers.py,sha256=0oUXc4aUkYtP2SGP6jTb3uPPKImIUsV
 mlrun/model_monitoring/db/tsdb/tdengine/__init__.py,sha256=vgBdsKaXUURKqIf3M0y4sRatmSVA4CQiJs7J5dcVBkQ,620
 mlrun/model_monitoring/db/tsdb/tdengine/schemas.py,sha256=LqWJebDXrXo0mteKVG6LnvPYlWBYh2lMCXEcv-lWoKA,10551
 mlrun/model_monitoring/db/tsdb/tdengine/stream_graph_steps.py,sha256=Hb0vcCBP-o0ET78mU4P32fnhUL65QZv-pMuv2lnCby4,1586
-mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=drNq_EpERU4lUNr_HyKBHVE4gB3CTz34Oi49XcO64EQ,18990
+mlrun/model_monitoring/db/tsdb/tdengine/tdengine_connector.py,sha256=tx8W40cqKKnwuAvFVa7pbX8mv6P-gS1dWNifjPuIG7c,18988
 mlrun/model_monitoring/db/tsdb/v3io/__init__.py,sha256=aL3bfmQsUQ-sbvKGdNihFj8gLCK3mSys0qDcXtYOwgc,616
 mlrun/model_monitoring/db/tsdb/v3io/stream_graph_steps.py,sha256=mbmhN4f_F58ptVjhwoMF6ifZSdnZWhK7x8eNsWS39IA,6217
 mlrun/model_monitoring/db/tsdb/v3io/v3io_connector.py,sha256=1H-IBXPNJPRAaxDMGWpUU25QqfR87LpZbJ03vaJkICs,32858
@@ -272,7 +272,7 @@ mlrun/platforms/__init__.py,sha256=ggSGF7inITs6S-vj9u4S9X_5psgbA0G3GVqf7zu8qYc,2
 mlrun/platforms/iguazio.py,sha256=1h5BpdAEQJBg2vIt7ySjUADU0ip5OkaMYr0_VREi9ys,13084
 mlrun/projects/__init__.py,sha256=Lv5rfxyXJrw6WGOWJKhBz66M6t3_zsNMCfUD6waPwx4,1153
 mlrun/projects/operations.py,sha256=gtqSU9OvYOV-b681uQtWgnW7YSnX6qfa1Mt1Xm4f1ZI,19752
-mlrun/projects/pipelines.py,sha256=9IZjfm9ccBO5xPW6FFY0em9w2ETNBP0hTvzfUf_YDjM,40951
+mlrun/projects/pipelines.py,sha256=tJsoqHYJw0jsB9rawUtTHQ71xnT03b_JVyPROPX-EGU,40962
 mlrun/projects/project.py,sha256=FjgkBBBP6geuxOGGp1Es5EFqsrs3M6PNWejBdoM08ng,190769
 mlrun/runtimes/__init__.py,sha256=egLM94cDMUyQ1GVABdFGXUQcDhU70lP3k7qSnM_UnHY,9008
 mlrun/runtimes/base.py,sha256=JXWmTIcm3b0klGUOHDlyFNa3bUgsNzQIgWhUQpSZoE0,37692
@@ -341,11 +341,11 @@ mlrun/utils/notifications/notification/ipython.py,sha256=ZtVL30B_Ha0VGoo4LxO-voT
 mlrun/utils/notifications/notification/slack.py,sha256=wqpFGr5BTvFO5KuUSzFfxsgmyU1Ohq7fbrGeNe9TXOk,7006
 mlrun/utils/notifications/notification/webhook.py,sha256=cb9w1Mc8ENfJBdgan7iiVHK9eVls4-R3tUxmXM-P-8I,4746
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=aebwGt4rZOROuMZJhVbQLjSTK6WxjEeI0k5kbplPv10,89
+mlrun/utils/version/version.json,sha256=2gLjdFu6FTXJq5Qo9gVAozN80SS7B7ymDVVut5IiECM,89
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.7.0rc52.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.7.0rc52.dist-info/METADATA,sha256=4xKf0fr7nekI1lzBspdArputjtfZNMTiXwjBWx0dQHo,24485
-mlrun-1.7.0rc52.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-mlrun-1.7.0rc52.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.7.0rc52.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.7.0rc52.dist-info/RECORD,,
+mlrun-1.7.0rc57.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.7.0rc57.dist-info/METADATA,sha256=X8t8a2srkg8jb147WeKPA5In1ZAZz73qd6_tuoR1New,24485
+mlrun-1.7.0rc57.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+mlrun-1.7.0rc57.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.7.0rc57.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.7.0rc57.dist-info/RECORD,,

{mlrun-1.7.0rc52.dist-info → mlrun-1.7.0rc57.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any