mlrun 1.10.0rc24__tar.gz → 1.10.0rc25__tar.gz

{mlrun-1.10.0rc24/mlrun.egg-info → mlrun-1.10.0rc25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlrun
-Version: 1.10.0rc24
+Version: 1.10.0rc25
 Summary: Tracking and config of machine learning runs
 Home-page: https://github.com/mlrun/mlrun
 Author: Yaron Haviv
@@ -43,7 +43,7 @@ Requires-Dist: v3io-frames~=0.10.15; python_version < "3.11"
 Requires-Dist: v3io-frames>=0.13.0; python_version >= "3.11"
 Requires-Dist: semver~=3.0
 Requires-Dist: dependency-injector~=4.41
-Requires-Dist: fsspec<2024.7,>=2023.9.2
+Requires-Dist: fsspec<=2025.7.0,>=2025.5.1
 Requires-Dist: v3iofs~=0.1.17
 Requires-Dist: storey~=1.10.11
 Requires-Dist: inflection~=0.5.0
@@ -56,14 +56,15 @@ Requires-Dist: mlrun-pipelines-kfp-common~=0.5.8
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7
 Requires-Dist: docstring_parser~=0.16
 Requires-Dist: aiosmtplib~=3.0
+Requires-Dist: deepdiff~=7.0
 Provides-Extra: s3
 Requires-Dist: boto3<1.36,>=1.28.0; extra == "s3"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "s3"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "s3"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "s3"
 Provides-Extra: azure-blob-storage
 Requires-Dist: msrest~=0.6.21; extra == "azure-blob-storage"
 Requires-Dist: azure-core~=1.24; extra == "azure-blob-storage"
-Requires-Dist: adlfs==2023.9.0; extra == "azure-blob-storage"
+Requires-Dist: adlfs==2024.12.0; extra == "azure-blob-storage"
 Requires-Dist: pyopenssl>=23; extra == "azure-blob-storage"
 Provides-Extra: azure-key-vault
 Requires-Dist: azure-identity~=1.5; extra == "azure-key-vault"
@@ -78,7 +79,7 @@ Requires-Dist: google-cloud-storage==2.14.0; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "google-cloud"
 Requires-Dist: google-cloud==0.34; extra == "google-cloud"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "google-cloud"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "google-cloud"
 Provides-Extra: kafka
 Requires-Dist: kafka-python~=2.1.0; extra == "kafka"
 Requires-Dist: avro~=1.11; extra == "kafka"
@@ -96,8 +97,8 @@ Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "da
 Requires-Dist: dask~=2023.12.1; python_version < "3.11" and extra == "dask"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "dask"
 Provides-Extra: alibaba-oss
-Requires-Dist: ossfs==2023.12.0; extra == "alibaba-oss"
-Requires-Dist: oss2==2.18.1; extra == "alibaba-oss"
+Requires-Dist: ossfs==2025.5.0; extra == "alibaba-oss"
+Requires-Dist: oss2==2.18.4; extra == "alibaba-oss"
 Provides-Extra: tdengine
 Requires-Dist: taos-ws-py==0.3.2; extra == "tdengine"
 Provides-Extra: snowflake
@@ -126,7 +127,7 @@ Requires-Dist: pydantic<2,>=1; extra == "api"
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "api"
 Requires-Dist: grpcio~=1.70.0; extra == "api"
 Provides-Extra: all
-Requires-Dist: adlfs==2023.9.0; extra == "all"
+Requires-Dist: adlfs==2024.12.0; extra == "all"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "all"
 Requires-Dist: avro~=1.11; extra == "all"
 Requires-Dist: azure-core~=1.24; extra == "all"
@@ -138,7 +139,7 @@ Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "all"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "all"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "all"
 Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "all"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "all"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "all"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "all"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "all"
 Requires-Dist: google-cloud-storage==2.14.0; extra == "all"
@@ -147,17 +148,17 @@ Requires-Dist: graphviz~=0.20.0; extra == "all"
 Requires-Dist: kafka-python~=2.1.0; extra == "all"
 Requires-Dist: mlflow~=2.22; extra == "all"
 Requires-Dist: msrest~=0.6.21; extra == "all"
-Requires-Dist: oss2==2.18.1; extra == "all"
-Requires-Dist: ossfs==2023.12.0; extra == "all"
+Requires-Dist: oss2==2.18.4; extra == "all"
+Requires-Dist: ossfs==2025.5.0; extra == "all"
 Requires-Dist: plotly~=5.23; extra == "all"
 Requires-Dist: pyopenssl>=23; extra == "all"
 Requires-Dist: redis~=4.3; extra == "all"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "all"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "all"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "all"
 Requires-Dist: sqlalchemy~=2.0; extra == "all"
 Requires-Dist: taos-ws-py==0.3.2; extra == "all"
 Provides-Extra: complete
-Requires-Dist: adlfs==2023.9.0; extra == "complete"
+Requires-Dist: adlfs==2024.12.0; extra == "complete"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete"
 Requires-Dist: avro~=1.11; extra == "complete"
 Requires-Dist: azure-core~=1.24; extra == "complete"
@@ -169,7 +170,7 @@ Requires-Dist: dask~=2024.12.1; python_version >= "3.11" and extra == "complete"
 Requires-Dist: databricks-sdk~=0.20.0; extra == "complete"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "complete"
 Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "complete"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "complete"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "complete"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "complete"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "complete"
 Requires-Dist: google-cloud-storage==2.14.0; extra == "complete"
@@ -178,17 +179,17 @@ Requires-Dist: graphviz~=0.20.0; extra == "complete"
 Requires-Dist: kafka-python~=2.1.0; extra == "complete"
 Requires-Dist: mlflow~=2.22; extra == "complete"
 Requires-Dist: msrest~=0.6.21; extra == "complete"
-Requires-Dist: oss2==2.18.1; extra == "complete"
-Requires-Dist: ossfs==2023.12.0; extra == "complete"
+Requires-Dist: oss2==2.18.4; extra == "complete"
+Requires-Dist: ossfs==2025.5.0; extra == "complete"
 Requires-Dist: plotly~=5.23; extra == "complete"
 Requires-Dist: pyopenssl>=23; extra == "complete"
 Requires-Dist: redis~=4.3; extra == "complete"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "complete"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete"
 Requires-Dist: sqlalchemy~=2.0; extra == "complete"
 Requires-Dist: taos-ws-py==0.3.2; extra == "complete"
 Provides-Extra: complete-api
-Requires-Dist: adlfs==2023.9.0; extra == "complete-api"
+Requires-Dist: adlfs==2024.12.0; extra == "complete-api"
 Requires-Dist: aiobotocore<2.16,>=2.5.0; extra == "complete-api"
 Requires-Dist: aiosmtplib~=3.0; extra == "complete-api"
 Requires-Dist: alembic~=1.14; extra == "complete-api"
@@ -205,7 +206,7 @@ Requires-Dist: databricks-sdk~=0.20.0; extra == "complete-api"
 Requires-Dist: distributed~=2023.12.1; python_version < "3.11" and extra == "complete-api"
 Requires-Dist: distributed~=2024.12.1; python_version >= "3.11" and extra == "complete-api"
 Requires-Dist: fastapi~=0.116.0; extra == "complete-api"
-Requires-Dist: gcsfs<2024.7,>=2023.9.2; extra == "complete-api"
+Requires-Dist: gcsfs<=2025.7.0,>=2025.5.1; extra == "complete-api"
 Requires-Dist: google-cloud-bigquery-storage~=2.17; extra == "complete-api"
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]==3.14.1; extra == "complete-api"
 Requires-Dist: google-cloud-storage==2.14.0; extra == "complete-api"
@@ -220,15 +221,15 @@ Requires-Dist: mlflow~=2.22; extra == "complete-api"
 Requires-Dist: mlrun-pipelines-kfp-v1-8~=0.5.7; extra == "complete-api"
 Requires-Dist: msrest~=0.6.21; extra == "complete-api"
 Requires-Dist: objgraph~=3.6; extra == "complete-api"
-Requires-Dist: oss2==2.18.1; extra == "complete-api"
-Requires-Dist: ossfs==2023.12.0; extra == "complete-api"
+Requires-Dist: oss2==2.18.4; extra == "complete-api"
+Requires-Dist: ossfs==2025.5.0; extra == "complete-api"
 Requires-Dist: plotly~=5.23; extra == "complete-api"
 Requires-Dist: psycopg2-binary~=2.9; extra == "complete-api"
 Requires-Dist: pydantic<2,>=1; extra == "complete-api"
 Requires-Dist: pymysql~=1.1; extra == "complete-api"
 Requires-Dist: pyopenssl>=23; extra == "complete-api"
 Requires-Dist: redis~=4.3; extra == "complete-api"
-Requires-Dist: s3fs<2024.7,>=2023.9.2; extra == "complete-api"
+Requires-Dist: s3fs<=2025.7.0,>=2025.5.1; extra == "complete-api"
 Requires-Dist: snowflake-connector-python~=3.7; extra == "complete-api"
 Requires-Dist: sqlalchemy-utils~=0.41.2; extra == "complete-api"
 Requires-Dist: sqlalchemy~=2.0; extra == "complete-api"

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/dependencies.py

@@ -29,12 +29,12 @@ def extra_requirements() -> dict[str, list[str]]:
         "s3": [
             "boto3>=1.28.0,<1.36",
             "aiobotocore>=2.5.0,<2.16",
-            "s3fs>=2023.9.2, <2024.7",
+            "s3fs>=2025.5.1, <=2025.7.0",
         ],
         "azure-blob-storage": [
             "msrest~=0.6.21",
             "azure-core~=1.24",
-            "adlfs==2023.9.0",
+            "adlfs==2024.12.0",
             "pyopenssl>=23",
         ],
         "azure-key-vault": [
@@ -53,7 +53,7 @@ def extra_requirements() -> dict[str, list[str]]:
             # (https://github.com/pypa/setuptools/issues/4476) with setuptools (ML-7273)
             "google-cloud-bigquery-storage~=2.17",
             "google-cloud==0.34",
-            "gcsfs>=2023.9.2, <2024.7",
+            "gcsfs>=2025.5.1, <=2025.7.0",
         ],
         "kafka": [
             "kafka-python~=2.1.0",
@@ -72,7 +72,7 @@ def extra_requirements() -> dict[str, list[str]]:
             'dask~=2023.12.1; python_version < "3.11"',
             'distributed~=2023.12.1; python_version < "3.11"',
         ],
-        "alibaba-oss": ["ossfs==2023.12.0", "oss2==2.18.1"],
+        "alibaba-oss": ["ossfs==2025.5.0", "oss2==2.18.4"],
         "tdengine": ["taos-ws-py==0.3.2"],
         "snowflake": ["snowflake-connector-python~=3.7"],
     }

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/extras-requirements.txt

@@ -10,18 +10,18 @@
 # we have a test test_extras_requirement_file_aligned to verify this file is aligned to setup.py
 # The versions of gcsfs and s3fs should be identical to the fsspec version;
 # Therefore, they must be equal to each other.
-s3fs>=2023.9.2, <2024.7
-gcsfs>=2023.9.2, <2024.7
-ossfs==2023.12.0
+s3fs>=2025.5.1, <=2025.7.0
+gcsfs>=2025.5.1, <=2025.7.0
+ossfs==2025.5.0
 boto3>=1.28.0,<1.36
-oss2==2.18.1
+oss2==2.18.4
 aiobotocore>=2.5.0,<2.16
 # https://github.com/Azure/azure-sdk-for-python/issues/24765#issuecomment-1150310498
 msrest~=0.6.21
 azure-core~=1.24
 # This version is required in order to support the self._strip_protocol
 # function in Azure Blob File System functions (such as _ls and more).
-adlfs==2023.9.0
+adlfs==2024.12.0
 azure-identity~=1.5
 azure-keyvault-secrets~=4.2
 # cryptography>=39, which is required by azure, needs this, or else we get

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/mlrun/common/schemas/hub.py

@@ -15,6 +15,7 @@
 from datetime import datetime, timezone
 from typing import Optional
 
+import deepdiff
 from pydantic.v1 import BaseModel, Extra, Field
 
 import mlrun.common.types
@@ -83,6 +84,19 @@ class HubSource(BaseModel):
             status=ObjectStatus(state="created"),
         )
 
+    def diff(self, another_source: "HubSource") -> dict:
+        """
+        Compare this HubSource with another one.
+        Returns a dict of differences (metadata, spec, status).
+        """
+        exclude_paths = [
+            "root['metadata']['updated']",
+            "root['metadata']['created']",
+        ]
+        return deepdiff.DeepDiff(
+            self.dict(), another_source.dict(), exclude_paths=exclude_paths
+        )
+
 
 last_source_index = -1
 

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/mlrun/common/schemas/model_monitoring/constants.py

@@ -331,6 +331,7 @@ class EndpointType(IntEnum):
 class EndpointMode(IntEnum):
     REAL_TIME = 0
     BATCH = 1
+    BATCH_LEGACY = 2  # legacy batch mode, used for endpoints created through the batch inference job
 
 
 class MonitoringFunctionNames(MonitoringStrEnum):

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -119,7 +119,7 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
     project: constr(regex=PROJECT_PATTERN)
     endpoint_type: EndpointType = EndpointType.NODE_EP
     uid: Optional[constr(regex=MODEL_ENDPOINT_ID_PATTERN)]
-    mode: EndpointMode = EndpointMode.REAL_TIME
+    mode: Optional[EndpointMode] = None
 
     @classmethod
     def mutable_fields(cls):
@@ -131,6 +131,15 @@ class ModelEndpointMetadata(ObjectMetadata, ModelEndpointParser):
             return str(v)
         return v
 
+    @validator("mode", pre=True, always=True)
+    def _set_mode_based_on_endpoint_type(cls, v, values):  # noqa: N805
+        if v is None:
+            if values.get("endpoint_type") == EndpointType.BATCH_EP:
+                return EndpointMode.BATCH_LEGACY
+            else:
+                return EndpointMode.REAL_TIME
+        return v
+
 
 class ModelEndpointSpec(ObjectSpec, ModelEndpointParser):
     model_class: Optional[str] = ""

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/mlrun/datastore/azure_blob.py

@@ -229,18 +229,25 @@ class AzureBlobStore(DataStore):
         st = self.storage_options
         service = "blob"
         primary_url = None
-        if st.get("connection_string"):
+
+        # Parse connection string (fills account_name/account_key or SAS)
+        connection_string = st.get("connection_string")
+        if connection_string:
             primary_url, _, parsed_credential = parse_connection_str(
-                st.get("connection_string"), credential=None, service=service
+                connection_string, credential=None, service=service
             )
-            for key in ["account_name", "account_key"]:
-                parsed_value = parsed_credential.get(key)
-                if parsed_value:
+
+            if isinstance(parsed_credential, str):
+                # SharedAccessSignature as raw string
+                parsed_credential = {"sas_token": parsed_credential}
+
+            for key in ["account_name", "account_key", "sas_token"]:
+                if parsed_value := parsed_credential.get(key):
                     if key in st and st[key] != parsed_value:
                         if key == "account_name":
                             raise mlrun.errors.MLRunInvalidArgumentError(
-                                f"Storage option for '{key}' is '{st[key]}',\
-                                    which does not match corresponding connection string '{parsed_value}'"
+                                f"Storage option for '{key}' is '{st[key]}', "
+                                f"which does not match corresponding connection string '{parsed_value}'"
                             )
                         else:
                             raise mlrun.errors.MLRunInvalidArgumentError(
@@ -249,6 +256,7 @@ class AzureBlobStore(DataStore):
                     st[key] = parsed_value
 
         account_name = st.get("account_name")
+        # Derive host (prefer connection string primary URL)
         if primary_url:
             if primary_url.startswith("http://"):
                 primary_url = primary_url[len("http://") :]
@@ -258,48 +266,63 @@ class AzureBlobStore(DataStore):
         elif account_name:
             host = f"{account_name}.{service}.core.windows.net"
         else:
+            # nothing to configure yet
             return res
 
-        if "account_key" in st:
+        host = host.rstrip("/")
+
+        # Account key (optional; WASB supports it)
+        if "account_key" in st and st["account_key"]:
             res[f"spark.hadoop.fs.azure.account.key.{host}"] = st["account_key"]
 
-        if "client_secret" in st or "client_id" in st or "tenant_id" in st:
-            res[f"spark.hadoop.fs.azure.account.auth.type.{host}"] = "OAuth"
-            res[f"spark.hadoop.fs.azure.account.oauth.provider.type.{host}"] = (
-                "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
-            )
-            if "client_id" in st:
-                res[f"spark.hadoop.fs.azure.account.oauth2.client.id.{host}"] = st[
-                    "client_id"
-                ]
-            if "client_secret" in st:
-                res[f"spark.hadoop.fs.azure.account.oauth2.client.secret.{host}"] = st[
-                    "client_secret"
-                ]
-            if "tenant_id" in st:
-                tenant_id = st["tenant_id"]
-                res[f"spark.hadoop.fs.azure.account.oauth2.client.endpoint.{host}"] = (
-                    f"https://login.microsoftonline.com/{tenant_id}/oauth2/token"
-                )
+        # --- WASB + SAS (container-scoped key; no provider classes needed) ---
+        if "sas_token" in st and st["sas_token"]:
+            sas = st["sas_token"].lstrip("?")
+            if container := getattr(self, "endpoint", None) or st.get("container"):
+                # fs.azure.sas.<container>.<account>.blob.core.windows.net = <sas>
+                res[f"spark.hadoop.fs.azure.sas.{container}.{host}"] = sas
 
-        if "sas_token" in st:
-            res[f"spark.hadoop.fs.azure.account.auth.type.{host}"] = "SAS"
-            res[f"spark.hadoop.fs.azure.sas.token.provider.type.{host}"] = (
-                "org.apache.hadoop.fs.azurebfs.sas.FixedSASTokenProvider"
-            )
-            res[f"spark.hadoop.fs.azure.sas.fixed.token.{host}"] = st["sas_token"]
+            else:
+                raise mlrun.errors.MLRunInvalidArgumentError(
+                    "Container name is required for WASB SAS. "
+                    "Set self.endpoint or storage_options['container']."
+                )
         return res
 
     @property
     def spark_url(self):
-        spark_options = self.get_spark_options()
-        url = f"wasbs://{self.endpoint}"
-        prefix = "spark.hadoop.fs.azure.account.key."
-        if spark_options:
-            for key in spark_options:
-                if key.startswith(prefix):
-                    account_key = key[len(prefix) :]
-                    if not url.endswith(account_key):
-                        url += f"@{account_key}"
-                    break
-        return url
+        # Build: wasbs://<container>@<host>
+        st = self.storage_options
+        service = "blob"
+
+        container = getattr(self, "endpoint", None) or st.get("container")
+        if not container:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "Container is required to build the WASB URL "
+                "(self.endpoint or storage_options['container'])."
+            )
+
+        # Prefer host from connection string; else synthesize from account_name
+        host = None
+        account_name = st.get("account_name")
+        connection_string = st.get("connection_string")
+
+        if connection_string:
+            primary_url, _, _ = parse_connection_str(
+                connection_string, credential=None, service=service
+            )
+            if primary_url.startswith("http://"):
+                primary_url = primary_url[len("http://") :]
+            if primary_url.startswith("https://"):
+                primary_url = primary_url[len("https://") :]
+            host = primary_url.rstrip("/")
+
+        if not host and account_name:
+            host = f"{account_name}.{service}.core.windows.net"
+
+        if not host:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                "account_name is required (or provide a connection_string) to build the WASB URL."
+            )
+
+        return f"wasbs://{container}@{host}"

{mlrun-1.10.0rc24 → mlrun-1.10.0rc25}/mlrun/datastore/datastore_profile.py

@@ -333,7 +333,9 @@ class DatastoreProfileGCS(DatastoreProfile):
             #  in gcs the path after schema is starts with bucket, wherefore it should not start with "/".
             subpath = subpath[1:]
         if self.bucket:
-            return f"gcs://{self.bucket}/{subpath}"
+            return (
+                f"gcs://{self.bucket}/{subpath}" if subpath else f"gcs://{self.bucket}"
+            )
         else:
             return f"gcs://{subpath}"
 
@@ -370,7 +372,11 @@ class DatastoreProfileAzureBlob(DatastoreProfile):
             #  in azure the path after schema is starts with container, wherefore it should not start with "/".
             subpath = subpath[1:]
         if self.container:
-            return f"az://{self.container}/{subpath}"
+            return (
+                f"az://{self.container}/{subpath}"
+                if subpath
+                else f"az://{self.container}"
+            )
         else:
             return f"az://{subpath}"