mlrun 1.7.1rc10__py3-none-any.whl → 1.7.2rc3__py3-none-any.whl

mlrun/common/schemas/artifact.py

@@ -47,6 +47,12 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
                 True,
             )
 
+    @classmethod
+    def from_kind(cls, kind: str) -> "ArtifactCategories":
+        if kind in [cls.model.value, cls.dataset.value]:
+            return cls(kind)
+        return cls.other
+
 
 class ArtifactIdentifier(pydantic.BaseModel):
     # artifact kind

mlrun/config.py

@@ -120,6 +120,14 @@ default_config = {
         "projects": {
             "summaries": {
                 "cache_interval": "30",
+                "feature_gates": {
+                    "artifacts": "enabled",
+                    "schedules": "enabled",
+                    "feature_sets": "enabled",
+                    "models": "enabled",
+                    "runs": "enabled",
+                    "pipelines": "enabled",
+                },
             },
         },
     },

mlrun/data_types/data_types.py

@@ -124,6 +124,7 @@ def spark_to_value_type(data_type):
         "double": ValueType.DOUBLE,
         "boolean": ValueType.BOOL,
         "timestamp": ValueType.DATETIME,
+        "timestamp_ntz": ValueType.DATETIME,
         "string": ValueType.STRING,
         "array": "list",
         "map": "dict",

mlrun/data_types/spark.py

@@ -18,7 +18,7 @@ from os import environ
 import numpy as np
 import pytz
 from pyspark.sql.functions import to_utc_timestamp
-from pyspark.sql.types import BooleanType, DoubleType, TimestampType
+from pyspark.sql.types import BooleanType, DoubleType
 
 from mlrun.feature_store.retrieval.spark_merger import spark_df_to_pandas
 from mlrun.utils import logger
@@ -143,7 +143,8 @@ def get_df_stats_spark(df, options, num_bins=20, sample_size=None):
     timestamp_columns = set()
     boolean_columns = set()
     for field in df_after_type_casts.schema.fields:
-        is_timestamp = isinstance(field.dataType, TimestampType)
+        # covers TimestampType and TimestampNTZType, which was added in PySpark 3.4.0
+        is_timestamp = field.dataType.typeName().startswith("timestamp")
         is_boolean = isinstance(field.dataType, BooleanType)
         if is_timestamp:
             df_after_type_casts = df_after_type_casts.withColumn(

mlrun/data_types/to_pandas.py

@@ -244,6 +244,15 @@ def _to_corrected_pandas_type(dt):
 
 
 def spark_df_to_pandas(spark_df):
+    import pyspark
+
+    if semver.parse(pyspark.__version__) >= semver.Version(3, 5, 0):
+
+        def to_pandas(spark_df_inner):
+            return spark_df_inner.toPandas()
+    else:
+        to_pandas = _to_pandas
+
     # as of pyspark 3.2.3, toPandas fails to convert timestamps unless we work around the issue
     # when we upgrade pyspark, we should check whether this workaround is still necessary
     # see https://stackoverflow.com/questions/76389694/transforming-pyspark-to-pandas-dataframe
@@ -262,9 +271,9 @@ def spark_df_to_pandas(spark_df):
                 )
                 type_conversion_dict[field.name] = "datetime64[ns]"
 
-        df = _to_pandas(spark_df)
+        df = to_pandas(spark_df)
         if type_conversion_dict:
             df = df.astype(type_conversion_dict)
         return df
     else:
-        return _to_pandas(spark_df)
+        return to_pandas(spark_df)

mlrun/datastore/targets.py

@@ -1136,7 +1136,8 @@ class CSVTarget(BaseStoreTarget):
         import pyspark.sql.functions as funcs
 
         for col_name, col_type in df.dtypes:
-            if col_type == "timestamp":
+            # covers TimestampType and TimestampNTZType, which was added in PySpark 3.4.0
+            if col_type.startswith("timestamp"):
                 # df.write.csv saves timestamps with millisecond precision, but we want microsecond precision
                 # for compatibility with storey.
                 df = df.withColumn(

mlrun/db/factory.py

@@ -54,9 +54,6 @@ class RunDBFactory(
             self._run_db = self._rundb_container.nop(url)
 
         else:
-            # TODO: this practically makes the SQLRunDB a singleton, which mean that its session is shared, needs
-            #  to be refreshed frequently and cannot be used concurrently.
-            #  The SQLRunDB should always get its session from the FastAPI dependency injection.
             self._run_db = self._rundb_container.run_db(url)
 
         self._run_db.connect(secrets=secrets)

mlrun/runtimes/nuclio/function.py

@@ -1035,9 +1035,10 @@ class RemoteRuntime(KubeResource):
         if args and sidecar.get("command"):
             sidecar["args"] = mlrun.utils.helpers.as_list(args)
 
-        # populate the sidecar resources from the function spec
+        # put the configured resources on the sidecar container instead of the reverse proxy container
         if self.spec.resources:
             sidecar["resources"] = self.spec.resources
+            self.spec.resources = None
 
     def _set_sidecar(self, name: str) -> dict:
         self.spec.config.setdefault("spec.sidecars", [])

mlrun/utils/notifications/notification/webhook.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 import typing
 
 import aiohttp
@@ -93,7 +94,6 @@ class WebhookNotification(NotificationBase):
 
     @staticmethod
     def _serialize_runs_in_request_body(override_body, runs):
-        str_parsed_runs = ""
         runs = runs or []
 
         def parse_runs():
@@ -105,22 +105,26 @@ class WebhookNotification(NotificationBase):
                     parsed_run = {
                         "project": run["metadata"]["project"],
                         "name": run["metadata"]["name"],
-                        "host": run["metadata"]["labels"]["host"],
                         "status": {"state": run["status"]["state"]},
                     }
-                    if run["status"].get("error", None):
-                        parsed_run["status"]["error"] = run["status"]["error"]
-                    elif run["status"].get("results", None):
-                        parsed_run["status"]["results"] = run["status"]["results"]
+                    if host := run["metadata"].get("labels", {}).get("host", ""):
+                        parsed_run["host"] = host
+                    if error := run["status"].get("error"):
+                        parsed_run["status"]["error"] = error
+                    elif results := run["status"].get("results"):
+                        parsed_run["status"]["results"] = results
                     parsed_runs.append(parsed_run)
             return str(parsed_runs)
 
         if isinstance(override_body, dict):
             for key, value in override_body.items():
-                if "{{ runs }}" or "{{runs}}" in value:
-                    if not str_parsed_runs:
-                        str_parsed_runs = parse_runs()
-                    override_body[key] = value.replace(
-                        "{{ runs }}", str_parsed_runs
-                    ).replace("{{runs}}", str_parsed_runs)
+                if not isinstance(value, str):
+                    # If the value is not a string, we don't want to parse it
+                    continue
+                if re.search(r"{{\s*runs\s*}}", value):
+                    str_parsed_runs = parse_runs()
+                    override_body[key] = re.sub(
+                        r"{{\s*runs\s*}}", str_parsed_runs, value
+                    )
+
         return override_body

mlrun/utils/notifications/notification_pusher.py

@@ -168,6 +168,11 @@ class NotificationPusher(_NotificationPusherBase):
                     logger.warning(
                         "Failed to push notification async",
                         error=mlrun.errors.err_to_str(result),
+                        traceback=traceback.format_exception(
+                            etype=type(result),
+                            value=result,
+                            tb=result.__traceback__,
+                        ),
                     )
 
         logger.debug(

mlrun/utils/version/version.json

@@ -1,4 +1,4 @@
 {
-  "git_commit": "837dd64ab75493fe34986c6203d884a0ae33843b",
-  "version": "1.7.1-rc10"
+  "git_commit": "3440c50863afd3b3dd850ade4ee237e8a1eb3211",
+  "version": "1.7.2-rc3"
 }

{mlrun-1.7.1rc10.dist-info → mlrun-1.7.2rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mlrun
-Version: 1.7.1rc10
+Version: 1.7.2rc3
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv

{mlrun-1.7.1rc10.dist-info → mlrun-1.7.2rc3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 mlrun/__init__.py,sha256=y08M1JcKXy5-9_5WaI9fn5aV5BxIQ5QkbduJK0OxWbA,7470
 mlrun/__main__.py,sha256=mC_Izs4kuHUHQi88QJFLN22n1kbygGM0wAirjNt7uj4,45938
-mlrun/config.py,sha256=DbVVAW1kgjtW0Bkm5bymE2_Wxd-0iHV_hoAlRDnQZDA,68743
+mlrun/config.py,sha256=wSwpgwg3dCTogMSLX5AX0tcZKuafSnv1RaoL1Ws7vlk,69056
 mlrun/errors.py,sha256=G8GP4_wb3v2UEbiAS8OlamC7nYJNzbSvQ3sViZlyYhk,8063
 mlrun/execution.py,sha256=nXvvN8euzjuxhJouJD8VxfK0keTTA6UoMrcD_17AL-4,44252
 mlrun/features.py,sha256=1VlN5mdSvUrLSJJlJWk4mXp9YoNxkFTu36IGn9AbN7s,15539
@@ -40,7 +40,7 @@ mlrun/common/runtimes/constants.py,sha256=Rl0Sd8n_L7Imo-uF1LL9CJ5Szi0W1gUm36yrF8
 mlrun/common/schemas/__init__.py,sha256=QZMyVHjIoa88JmyVy45JGkNGz5K39XX7A72TUnXrLNA,5267
 mlrun/common/schemas/alert.py,sha256=qWYCISNYMdkgAARVQNxshVr9d-s8LGscfLKpczkTBms,6749
 mlrun/common/schemas/api_gateway.py,sha256=9ilorgLOiWxFZbv89-dbPNfVdaChlGOIdC4SLTxQwNI,7118
-mlrun/common/schemas/artifact.py,sha256=V3ngobnzI1v2eoOroWBEedjAZu0ntCSIQ-LzsOK1Z9k,3570
+mlrun/common/schemas/artifact.py,sha256=lX0tAjvIxrMDmubvTUKKKl-BAt0FKDG0s7lzfvi1jb0,3758
 mlrun/common/schemas/auth.py,sha256=7XpEXICjDhHHkAppOp0mHvEtCwG68L3mhgSHPqqTBMk,6584
 mlrun/common/schemas/background_task.py,sha256=2qZxib2qrF_nPZj0ncitCG-2jxz2hg1qj0hFc8eswWQ,1707
 mlrun/common/schemas/client_spec.py,sha256=wqzQ5R4Zc7FL-8lV_BRN6nLrD0jK1kon05-JQ3fy2KY,2892
@@ -73,10 +73,10 @@ mlrun/common/schemas/model_monitoring/constants.py,sha256=Wha21Iev3Nr9ugB1Ms_wrm
 mlrun/common/schemas/model_monitoring/grafana.py,sha256=SG13MFUUz_tk6-mWeSx17qcdEW4ekicxqNtnMSwRTCY,1559
 mlrun/common/schemas/model_monitoring/model_endpoints.py,sha256=5vvjNX1bV98VSGdT4jwHr5ArKC9v_c1iHlaTf82fSUY,13198
 mlrun/data_types/__init__.py,sha256=EkxfkFoHb91zz3Aymq-KZfCHlPMzEc3bBqgzPUwmHWY,1087
-mlrun/data_types/data_types.py,sha256=uB9qJusSvPRK2PTvrFBXrS5jcDXMuwqXokJGToDg4VA,4953
+mlrun/data_types/data_types.py,sha256=0_oKLC6-sXL2_nnaDMP_HSXB3fD1nJAG4J2Jq6sGNNw,4998
 mlrun/data_types/infer.py,sha256=z2EbSpR6xWEE5-HRUtDZkapHQld3xMbzXtTX83K-690,6134
-mlrun/data_types/spark.py,sha256=xfcr6lcaLcHepnrHavx_vacMJK7BC8FWsUKjwrjjn6w,9509
-mlrun/data_types/to_pandas.py,sha256=-ZbJBg00x4xxyqqqu3AVbEh-HaO2--DrChyPuedRhHA,11215
+mlrun/data_types/spark.py,sha256=ADlhaPPRDbHFrQA1jmMldGx8SP4dNBCL8Pyt0WyXWmU,9588
+mlrun/data_types/to_pandas.py,sha256=KOy0FLXPJirsgH6szcC5BI6t70yVDCjuo6LmuYHNTuI,11429
 mlrun/datastore/__init__.py,sha256=y2_NkHUiz9WKJ1XWeUHX-MKErwmIag6nxZ7Z06EcSk0,4180
 mlrun/datastore/alibaba_oss.py,sha256=-RMA4vCE4rar-D57Niy3tY_6bXKHLFpMp28z5YR7-jI,4888
 mlrun/datastore/azure_blob.py,sha256=9qkgrEMXGiuYYcc6b6HkuHlRHDbl0p7tIzeWxAAcEVs,12724
@@ -96,7 +96,7 @@ mlrun/datastore/spark_udf.py,sha256=NnnB3DZxZb-rqpRy7b-NC7QWXuuqFn3XkBDc86tU4mQ,
 mlrun/datastore/spark_utils.py,sha256=_AsVoU5Ix_-W7Gyq8io8V-2GTk0m8THJNDP3WGGaWJY,2865
 mlrun/datastore/store_resources.py,sha256=rcLoG506AMmR8qPJU_gE-G5d34VJVV_vNlZ3VHqho6c,6869
 mlrun/datastore/storeytargets.py,sha256=uNYG4nCBD3JIfa51CG4cDe9ryc9oIcqUdUXKvCPB6uE,5086
-mlrun/datastore/targets.py,sha256=TkG2HG4h7SaQ3qG2sKAHAuJJyj_gnE-eChaIsyjlq1o,80450
+mlrun/datastore/targets.py,sha256=R-w-_pDlg_fzM5zdzBfdyBQ7PW8q2MqO8Z4k-Xuaw90,80549
 mlrun/datastore/utils.py,sha256=l9dLZb_VCbHs_htqMFRv4qiestZ8z8K-4eY1MxHS8wE,7720
 mlrun/datastore/v3io.py,sha256=HxP6mygiYM6leDAbQ9KdTxObLCt9yGMro0YhfdU6KUo,8157
 mlrun/datastore/wasbfs/__init__.py,sha256=s5Ul-0kAhYqFjKDR2X0O2vDGDbLQQduElb32Ev56Te4,1343
@@ -104,7 +104,7 @@ mlrun/datastore/wasbfs/fs.py,sha256=MnSj7Q4OKA2L55ihCmUnj2t3GA3B77oLMdAw-yxvN9w,
 mlrun/db/__init__.py,sha256=WqJ4x8lqJ7ZoKbhEyFqkYADd9P6E3citckx9e9ZLcIU,1163
 mlrun/db/auth_utils.py,sha256=hpg8D2r82oN0BWabuWN04BTNZ7jYMAF242YSUpK7LFM,5211
 mlrun/db/base.py,sha256=lUfJrCWbuRUErIrUUXAKI2sSlrwfB-dHDz-Ck_cnZHU,24297
-mlrun/db/factory.py,sha256=ibIrE5QkIIyzDU1FXKrfbc31cZiRLYKDZb8dqCpQwyU,2397
+mlrun/db/factory.py,sha256=yP2vVmveUE7LYTCHbS6lQIxP9rW--zdISWuPd_I3d_4,2111
 mlrun/db/httpdb.py,sha256=VSk5lCrxBQydla9Cw4lYLA7W9o0Ge4WNfmmKFB4x3WM,184966
 mlrun/db/nopdb.py,sha256=1oCZR2EmQQDkwXUgmyI3SB76zvOwA6Ml3Lk_xvuwHfc,21620
 mlrun/feature_store/__init__.py,sha256=FhHRc8NdqL_HWpCs7A8dKruxJS5wEm55Gs3dcgBiRUg,1522
@@ -294,7 +294,7 @@ mlrun/runtimes/mpijob/abstract.py,sha256=kDWo-IY1FKLZhI30j38Xx9HMhlUvHezfd1DT2Sh
 mlrun/runtimes/mpijob/v1.py,sha256=1XQZC7AIMGX_AQCbApcwpH8I7y39-v0v2O35MvxjXoo,3213
 mlrun/runtimes/nuclio/__init__.py,sha256=gx1kizzKv8pGT5TNloN1js1hdbxqDw3rM90sLVYVffY,794
 mlrun/runtimes/nuclio/api_gateway.py,sha256=oQRSOvqtODKCzT2LqlqSXZbq2vcZ7epsFZwO9jvarhc,26899
-mlrun/runtimes/nuclio/function.py,sha256=TQt6RyxK_iyzNJr2r57BRtVXuy2GMrhdeFOlFjb2AZg,52106
+mlrun/runtimes/nuclio/function.py,sha256=jtwtD9xfH3261cuekuHT97oBhNDqjBXCzHgQ2Y6IYs8,52184
 mlrun/runtimes/nuclio/nuclio.py,sha256=sLK8KdGO1LbftlL3HqPZlFOFTAAuxJACZCVl1c0Ha6E,2942
 mlrun/runtimes/nuclio/serving.py,sha256=L1Tz5EZyo8JZmUBNmIRYL9AoWfqSm4zLQQ9DWbnlmp8,29726
 mlrun/runtimes/nuclio/application/__init__.py,sha256=rRs5vasy_G9IyoTpYIjYDafGoL6ifFBKgBtsXn31Atw,614
@@ -332,20 +332,20 @@ mlrun/utils/singleton.py,sha256=p1Y-X0mPSs_At092GS-pZCA8CTR62HOqPU07_ZH6-To,869
 mlrun/utils/v3io_clients.py,sha256=0aCFiQFBmgdSeLzJr_nEP6SG-zyieSgH8RdtcUq4dc0,1294
 mlrun/utils/vault.py,sha256=xUiKL17dCXjwQJ33YRzQj0oadUXATlFWPzKKYAESoQk,10447
 mlrun/utils/notifications/__init__.py,sha256=eUzQDBxSQmMZASRY-YAnYS6tL5801P0wEjycp3Dvoe0,990
-mlrun/utils/notifications/notification_pusher.py,sha256=4ecV6JfCtvYpb0kl1-sdg4Cw6XTrAjmmh2olhUenesY,26752
+mlrun/utils/notifications/notification_pusher.py,sha256=ASBnYSjroV4y2BDK93w_0ZDSp92F6oDZWEax07uvAdY,26984
 mlrun/utils/notifications/notification/__init__.py,sha256=o1OgBKFSQoD6g8Lh20Cw-_CLa-FPVaL33Kv6YwKiLGA,2154
 mlrun/utils/notifications/notification/base.py,sha256=hf3BDZ4-bq92MsqofQHt8DZqqlcKbWHscZFvzHdMcw4,4265
 mlrun/utils/notifications/notification/console.py,sha256=MAVk7v5PJ52vdGRv76YcEPixWgV0licBPWGpR01uR40,2643
 mlrun/utils/notifications/notification/git.py,sha256=g_8RksjCboGrKKjyhkePk5nSWrfdT61JkhMeg9EeGcY,6119
 mlrun/utils/notifications/notification/ipython.py,sha256=ZtVL30B_Ha0VGoo4LxO-voT1U41IYwyytovv5X_LsI4,2066
 mlrun/utils/notifications/notification/slack.py,sha256=wqpFGr5BTvFO5KuUSzFfxsgmyU1Ohq7fbrGeNe9TXOk,7006
-mlrun/utils/notifications/notification/webhook.py,sha256=cb9w1Mc8ENfJBdgan7iiVHK9eVls4-R3tUxmXM-P-8I,4746
+mlrun/utils/notifications/notification/webhook.py,sha256=ux-KGV0mjFtpgq5NvgMhNfjjMZwlVjpQKNBDFvRJKBo,4841
 mlrun/utils/version/__init__.py,sha256=7kkrB7hEZ3cLXoWj1kPoDwo4MaswsI2JVOBpbKgPAgc,614
-mlrun/utils/version/version.json,sha256=rEraRuFIouYMRuXUyMajVSxbq-8ul-ZfqZGCQFrheTg,89
+mlrun/utils/version/version.json,sha256=tXyLyaj-oqIunewjazCj0NU4jQGIy1cB9Kh9K9g_5AY,88
 mlrun/utils/version/version.py,sha256=eEW0tqIAkU9Xifxv8Z9_qsYnNhn3YH7NRAfM-pPLt1g,1878
-mlrun-1.7.1rc10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mlrun-1.7.1rc10.dist-info/METADATA,sha256=dFxhKtPDI0LjkbvOl4pjTIilyBqBmhaHhFuuXPFqVTg,24168
-mlrun-1.7.1rc10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-mlrun-1.7.1rc10.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
-mlrun-1.7.1rc10.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
-mlrun-1.7.1rc10.dist-info/RECORD,,
+mlrun-1.7.2rc3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mlrun-1.7.2rc3.dist-info/METADATA,sha256=eFKRh0Nnt0IZ6fOD7VIUf-iRxCZEUHt0ywZLbqaHu7w,24167
+mlrun-1.7.2rc3.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
+mlrun-1.7.2rc3.dist-info/entry_points.txt,sha256=1Owd16eAclD5pfRCoJpYC2ZJSyGNTtUr0nCELMioMmU,46
+mlrun-1.7.2rc3.dist-info/top_level.txt,sha256=NObLzw3maSF9wVrgSeYBv-fgnHkAJ1kEkh12DLdd5KM,6
+mlrun-1.7.2rc3.dist-info/RECORD,,

{mlrun-1.7.1rc10.dist-info → mlrun-1.7.2rc3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.7.0)
 Root-Is-Purelib: true
 Tag: py3-none-any