mlrun 1.10.0rc40__py3-none-any.whl → 1.11.0rc16__py3-none-any.whl

mlrun/common/schemas/client_spec.py

@@ -67,3 +67,7 @@ class ClientSpec(pydantic.v1.BaseModel):
     alerts_mode: typing.Optional[str]
     system_id: typing.Optional[str]
     model_endpoint_monitoring_store_prefixes: typing.Optional[dict[str, str]]
+    authentication_mode: typing.Optional[str]
+    # Iguazio V4 OAuth token provider configuration
+    oauth_internal_token_endpoint: typing.Optional[str]
+    oauth_external_token_endpoint: typing.Optional[str]

mlrun/common/schemas/constants.py

@@ -95,6 +95,21 @@ headers_prefix = "x-mlrun-"
 
 class HeaderNames:
     projects_role = "x-projects-role"
+    remote_user = "x-remote-user"
+    forwarded_host = "x-forwarded-host"
+    data_session_override = "x-data-session-override"
+    user_id = "x-user-id"
+    user_group_ids = "x-user-group-ids"
+    unix_uid = "x-unix-uid"
+    v3io_session_key = "x-v3io-session-key"
+    v3io_access_key = "x-v3io-access-key"
+    v3io_user_id = "x-v3io-user-id"
+    v3io_session_planes = "x-v3io-session-planes"
+    authorization = "authorization"
+    igz_authenticator_kind = "x-igz-authenticator-kind"
+    cookies = "cookies"
+    cookie = "cookie"
+    x_request_id = "x-request-id"
     patch_mode = f"{headers_prefix}patch-mode"
     deletion_strategy = f"{headers_prefix}deletion-strategy"
     secret_store_token = f"{headers_prefix}secret-store-token"
@@ -106,6 +121,16 @@ class HeaderNames:
     ui_clear_cache = f"{headers_prefix}ui-clear-cache"
 
 
+class AuthorizationHeaderPrefixes:
+    basic = "Basic "
+    bearer = "Bearer "
+
+
+class CookieNames:
+    oauth2_proxy = "_oauth2_proxy"
+    iguazio = "session"
+
+
 class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
     name = "name"  # Supported for feature-store objects
 

mlrun/common/schemas/frontend_spec.py

@@ -31,13 +31,6 @@ class PreemptionNodesFeatureFlag(mlrun.common.types.StrEnum):
     disabled = "disabled"
 
 
-class AuthenticationFeatureFlag(mlrun.common.types.StrEnum):
-    none = "none"
-    basic = "basic"
-    bearer = "bearer"
-    iguazio = "iguazio"
-
-
 class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
     enabled = "enabled"
     disabled = "disabled"
@@ -45,7 +38,7 @@ class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
 
 class FeatureFlags(pydantic.v1.BaseModel):
     project_membership: ProjectMembershipFeatureFlag
-    authentication: AuthenticationFeatureFlag
+    authentication: mlrun.common.types.AuthenticationMode
     nuclio_streams: NuclioStreamsFeatureFlag
     preemption_nodes: PreemptionNodesFeatureFlag
 

mlrun/common/schemas/function.py

@@ -140,3 +140,27 @@ class Function(pydantic.v1.BaseModel):
 
     class Config:
         extra = pydantic.v1.Extra.allow
+
+
+class BatchingSpec(pydantic.v1.BaseModel):
+    # Set to True to enable batching
+    enabled: bool
+    # Maximal events to batch together. Default size is 10.
+    batch_size: typing.Optional[int]
+    # The maximum amount of time to wait before processing the batch. Default timeout is 1s.
+    # Once this time passes, the batch is processed even if it hasn’t reached the full batch size.
+    timeout: typing.Optional[str]
+
+    def get_nuclio_batch_config(self):
+        if not self.enabled:
+            return None
+
+        config = {"mode": "enable"}
+
+        if self.batch_size:
+            config["batchSize"] = self.batch_size
+
+        if self.timeout:
+            config["timeout"] = self.timeout
+
+        return config

mlrun/common/schemas/hub.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import Optional
 
 import deepdiff
@@ -40,6 +40,7 @@ class HubObjectMetadata(BaseModel):
 class HubSourceType(mlrun.common.types.StrEnum):
     functions = "functions"
     modules = "modules"
+    steps = "steps"
 
 
 # Sources-related objects
@@ -66,7 +67,7 @@ class HubSource(BaseModel):
         if not mlrun.mlconf.hub.default_source.create:
             return None
 
-        now = datetime.now(timezone.utc)
+        now = datetime.now(UTC)
         hub_metadata = HubObjectMetadata(
             name=mlrun.mlconf.hub.default_source.name,
             description=mlrun.mlconf.hub.default_source.description,

mlrun/common/schemas/model_monitoring/__init__.py

@@ -40,7 +40,7 @@ from .constants import (
     ResultStatusApp,
     SpecialApps,
     StreamProcessingEvent,
-    TDEngineSuperTables,
+    TimescaleDBTables,
     TSDBTarget,
     V3IOTSDBTables,
     VersionedModel,

mlrun/common/schemas/model_monitoring/constants.py

@@ -274,7 +274,7 @@ class EventKeyMetrics:
 
 class TSDBTarget(MonitoringStrEnum):
     V3IO_TSDB = "v3io-tsdb"
-    TDEngine = "tdengine"
+    TimescaleDB = "postgresql"
 
 
 class ProjectSecretKeys:
@@ -353,7 +353,7 @@ class V3IOTSDBTables(MonitoringStrEnum):
     PREDICTIONS = "predictions"
 
 
-class TDEngineSuperTables(MonitoringStrEnum):
+class TimescaleDBTables(MonitoringStrEnum):
     APP_RESULTS = "app_results"
     METRICS = "metrics"
     PREDICTIONS = "predictions"

mlrun/common/schemas/secret.py

@@ -49,5 +49,20 @@ class SecretKeysData(BaseModel):
     secret_keys: Optional[list] = []
 
 
-class UserSecretCreationRequest(SecretsData):
-    user: str
+class SecretToken(BaseModel):
+    name: str
+    token: str
+
+
+class StoreSecretTokensResponse(BaseModel):
+    created_tokens: list[str] = []
+    updated_tokens: list[str] = []
+
+
+class SecretTokenInfo(BaseModel):
+    name: str
+    expiration: int
+
+
+class ListSecretTokensResponse(BaseModel):
+    secret_tokens: list[SecretTokenInfo]

mlrun/common/secrets.py

@@ -11,10 +11,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import re
+import typing
 from abc import ABC, abstractmethod
 
 import mlrun.common.schemas
+from mlrun.config import config as mlconf
+
+_AUTH_SECRET_NAME_TEMPLATE = re.escape(
+    mlconf.secret_stores.kubernetes.auth_secret_name.format(
+        hashed_access_key="",
+    )
+)
+AUTH_SECRET_PATTERN = re.compile(f"^{_AUTH_SECRET_NAME_TEMPLATE}.*")
+
+
+def validate_not_forbidden_secret(secret_name: str) -> None:
+    """
+    Forbid client-supplied references to internal MLRun auth/project secrets.
+    No-op when running inside the API server (API enrichments are allowed).
+    """
+    if not secret_name or mlrun.config.is_running_as_api():
+        return
+    if AUTH_SECRET_PATTERN.match(secret_name):
+        raise mlrun.errors.MLRunInvalidArgumentError(
+            f"Forbidden secret '{secret_name}' matches MLRun auth-secret pattern."
+        )
 
 
 class SecretProviderInterface(ABC):
@@ -54,6 +76,44 @@ class SecretProviderInterface(ABC):
     def get_secret_data(self, secret_name, namespace=""):
         pass
 
+    @abstractmethod
+    def store_user_token_secret(
+        self,
+        username: str,
+        token_name: str,
+        token: str,
+        expiration: int,
+        force: bool = False,
+        namespace: typing.Optional[str] = None,
+    ) -> typing.Optional[mlrun.common.schemas.SecretEventActions]:
+        pass
+
+    @abstractmethod
+    def get_user_token_secret_value(
+        self,
+        username: str,
+        token_name: str,
+        namespace: typing.Optional[str] = None,
+    ) -> str:
+        pass
+
+    @abstractmethod
+    def list_user_token_secrets(
+        self,
+        username: str,
+        namespace: typing.Optional[str] = None,
+    ) -> list[mlrun.common.schemas.SecretTokenInfo]:
+        pass
+
+    @abstractmethod
+    def delete_user_token_secret(
+        self,
+        username: str,
+        token_name: str,
+        namespace: typing.Optional[str] = None,
+    ) -> None:
+        pass
+
 
 class InMemorySecretProvider(SecretProviderInterface):
     def __init__(self):
@@ -130,6 +190,40 @@ class InMemorySecretProvider(SecretProviderInterface):
     def get_secret_data(self, secret_name, namespace=""):
         return self.secrets_map[secret_name]
 
+    def store_user_token_secret(
+        self,
+        username: str,
+        token_name: str,
+        token: str,
+        expiration: int,
+        force: bool = False,
+        namespace: typing.Optional[str] = None,
+    ) -> typing.Optional[mlrun.common.schemas.SecretEventActions]:
+        raise NotImplementedError()
+
+    def get_user_token_secret_value(
+        self,
+        username: str,
+        token_name: str,
+        namespace: typing.Optional[str] = None,
+    ) -> str:
+        raise NotImplementedError()
+
+    def list_user_token_secrets(
+        self,
+        username: str,
+        namespace: typing.Optional[str] = None,
+    ) -> list[mlrun.common.schemas.SecretTokenInfo]:
+        raise NotImplementedError()
+
+    def delete_user_token_secret(
+        self,
+        username: str,
+        token_name: str,
+        namespace: typing.Optional[str] = None,
+    ) -> None:
+        raise NotImplementedError()
+
     @staticmethod
     def _generate_auth_secret_data(username: str, access_key: str):
         return {

mlrun/common/types.py

@@ -14,18 +14,10 @@
 
 import enum
 
+# Alias to Python's built-in StrEnum (Python 3.11+)
+StrEnum = enum.StrEnum
 
-# TODO: From python 3.11 StrEnum is built-in and this will not be needed
-class StrEnum(str, enum.Enum):
-    def __str__(self):
-        return self.value
 
-    def __repr__(self):
-        return self.value
-
-
-# Partial backport from Python 3.11
-# https://docs.python.org/3/library/http.html#http.HTTPMethod
 class HTTPMethod(StrEnum):
     GET = "GET"
     POST = "POST"
@@ -37,3 +29,11 @@ class HTTPMethod(StrEnum):
 class Operation(StrEnum):
     ADD = "add"
     REMOVE = "remove"
+
+
+class AuthenticationMode(StrEnum):
+    NONE = "none"
+    BASIC = "basic"
+    BEARER = "bearer"
+    IGUAZIO = "iguazio"
+    IGUAZIO_V4 = "iguazio-v4"

mlrun/config.py

@@ -40,6 +40,7 @@ import yaml
 
 import mlrun.common.constants
 import mlrun.common.schemas
+import mlrun.common.types
 import mlrun.errors
 
 env_prefix = "MLRUN_"
@@ -66,7 +67,6 @@ default_config = {
     "nuclio_version": "",
     "default_nuclio_runtime": "python:3.11",
     "nest_asyncio_enabled": "",  # enable import of nest_asyncio for corner cases with old jupyter, set "1"
-    "ui_url": "",  # remote/external mlrun UI url (for hyperlinks) (This is deprecated in favor of the ui block)
     "remote_host": "",
     "api_base_version": "v1",
     "version": "",  # will be set to current version
@@ -85,7 +85,9 @@ default_config = {
     "kfp_image": "mlrun/mlrun-kfp",  # image to use for KFP runner
     "dask_kfp_image": "mlrun/mlrun",  # image to use for dask KFP runner
     "igz_version": "",  # the version of the iguazio system the API is running on
-    "iguazio_api_url": "",  # the url to iguazio api
+    "iguazio_api_url": "",  # the url to iguazio api (internal / external access with priority to internal)
+    "iguazio_api_url_ingress": "",  # the url to iguazio api ingress (for external access)
+    "iguazio_api_ssl_verify": True,  # verify ssl certificate of iguazio api
     "spark_app_image": "",  # image to use for spark operator app runtime
     "spark_app_image_tag": "",  # image tag to use for spark operator app runtime
     "spark_history_server_path": "",  # spark logs directory for spark history server
@@ -422,11 +424,17 @@ default_config = {
             "allow_local_run": False,
         },
         "authentication": {
-            "mode": "none",  # one of none, basic, bearer, iguazio
+            "mode": "none",  # one of none, basic, bearer, iguazio, iguazio-v4
             "basic": {"username": "", "password": ""},
             "bearer": {"token": ""},
             "iguazio": {
                 "session_verification_endpoint": "data_sessions/verifications/app_service",
+                "authentication_endpoint": "api/v1/authentication/refresh-access-token",
+            },
+            "service_account": {
+                # the following are the default values for k8s service accounts, but may be changed per deployment
+                "token_expiration_seconds": 600,
+                "token_path": "/var/run/secrets/kubernetes.io/serviceaccount/token",
             },
         },
         "nuclio": {
@@ -481,6 +489,10 @@ default_config = {
         },
         "authorization": {
             "mode": "none",  # one of none, opa
+            "namespaces": {
+                "resources": "",
+                "mgmt": "mgmt",
+            },
             "opa": {
                 "address": "",
                 "request_timeout": 10,
@@ -653,7 +665,7 @@ default_config = {
         "writer_graph": {
             "max_events": 1000,
             "flush_after_seconds": 30,
-            "writer_version": "v1",  # v1 is the sync version while v2 is async
+            "writer_version": "v2",  # v1 is the sync version while v2 is async
             "parquet_batching_max_events": 10,
             "parquet_batching_timeout_secs": 30,
         },
@@ -670,6 +682,15 @@ default_config = {
         "parquet_batching_max_events": 10_000,
         "parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
         "model_endpoint_creation_check_period": 15,
+        # TSDB (TimescaleDB) configuration
+        "tsdb": {
+            # When True, automatically create/generate database name using system_id if not explicitly
+            # specified in the connection string. When False, use the database from connection string as-is.
+            "auto_create_database": True,
+            # Connection pool timeout in seconds. This is the maximum time to wait for a connection
+            # from the pool before raising an error.
+            "connection_pool_timeout": 120,
+        },
     },
     "secret_stores": {
         # Use only in testing scenarios (such as integration tests) to avoid using k8s for secrets (will use in-memory
@@ -725,7 +746,7 @@ default_config = {
             # Set false to avoid creating a global source (for example in a dark site)
             "create": True,
             "name": "default",
-            "description": "MLRun global function hub",
+            "description": "MLRun hub",
             "url": "https://mlrun.github.io/marketplace",
             "channel": "master",
         },
@@ -868,6 +889,19 @@ default_config = {
         "enabled": False,
         "request_timeout": 5,
     },
+    "auth_with_oauth_token": {
+        "enabled": False,
+        "request_timeout": 5,
+        "refresh_threshold": 0.75,
+        # Default is empty. automatically set based on configuration (end client vs jupyter vs runtime, etc)
+        # can be set manually set using envvars
+        "token_file": "",
+        # Default is empty because if set, searches for the specific token name in the file, if empty, it will look
+        # for a token named "default", if "default" does not exist, it will use the first token in the file
+        "token_name": "",
+    },
+    # a runtime computed value. Do not set it manually.
+    "auth_token_endpoint": "",
     "services": {
         # The running service name. One of: "api", "alerts"
         "service_name": "api",
@@ -965,7 +999,7 @@ class Config:
                     try:
                         config_value.update(value)
                     except AttributeError as exc:
-                        if not isinstance(config_value, (dict, Config)):
+                        if not isinstance(config_value, dict | Config):
                             raise ValueError(
                                 f"Can not update `{key}` config. "
                                 f"Expected a configuration but received {type(value)}"
@@ -1280,10 +1314,7 @@ class Config:
 
     @staticmethod
     def resolve_ui_url():
-        # ui_url is deprecated in favor of the ui.url (we created the ui block)
-        # since the config class is used in a "recursive" way, we can't use property like we used in other places
-        # since the property will need to be url, which exists in other structs as well
-        return config.ui.url or config.ui_url
+        return config.ui.url
 
     def is_api_running_on_k8s(self):
         # determine if the API service is attached to K8s cluster
@@ -1403,6 +1434,18 @@ class Config:
             ver in mlrun.mlconf.ce.mode for ver in ["lite", "full"]
         )
 
+    def is_iguazio_mode(self):
+        return (
+            mlrun.mlconf.httpdb.authentication.mode
+            == mlrun.common.types.AuthenticationMode.IGUAZIO
+        )
+
+    def is_iguazio_v4_mode(self):
+        return (
+            config.httpdb.authentication.mode
+            == mlrun.common.types.AuthenticationMode.IGUAZIO_V4
+        )
+
     def is_explicit_ack_enabled(self) -> bool:
         return self.httpdb.nuclio.explicit_ack == "enabled" and (
             not self.nuclio_version
@@ -1570,7 +1613,6 @@ def read_env(env=None, prefix=env_prefix):
             "https://mlrun-api.", "https://framesd."
         )
 
-    uisvc = env.get("MLRUN_UI_SERVICE_HOST")
     igz_domain = env.get("IGZ_NAMESPACE_DOMAIN")
 
     # workaround to try and detect IGZ domain
@@ -1596,10 +1638,6 @@ def read_env(env=None, prefix=env_prefix):
     if config.get("nuclio_dashboard_url") == "disabled":
         config["nuclio_dashboard_url"] = ""
 
-    if uisvc and not config.get("ui_url"):
-        if igz_domain:
-            config["ui_url"] = f"https://mlrun-ui.{igz_domain}"
-
     if log_level := config.get("log_level"):
         import mlrun.utils.logger
 

mlrun/data_types/infer.py

@@ -134,9 +134,9 @@ def get_df_stats(df, options, num_bins=None, sample_size=None):
     for col, values in df.describe(include="all", **kwargs).items():
         stats_dict = {}
         for stat, val in values.dropna().items():
-            if isinstance(val, (float, np.floating, np.float64)):
+            if isinstance(val, float | np.floating | np.float64):
                 stats_dict[stat] = float(val)
-            elif isinstance(val, (int, np.integer, np.int64)):
+            elif isinstance(val, int | np.integer | np.int64):
                 # boolean values are considered subclass of int
                 if isinstance(val, bool):
                     stats_dict[stat] = bool(val)

mlrun/datastore/__init__.py

@@ -59,7 +59,6 @@ from ..utils import logger
 from .base import DataItem
 from .datastore import StoreManager, in_memory_store, uri_to_ipython
 from .dbfs_store import DatabricksFileBugFixed, DatabricksFileSystemDisableCache
-from .s3 import parse_s3_bucket_and_key
 from .sources import (
     BigQuerySource,
     CSVSource,
@@ -75,7 +74,7 @@ from .store_resources import (
     parse_store_uri,
 )
 from .targets import CSVTarget, NoSqlTarget, ParquetTarget, StreamTarget
-from .utils import get_kafka_brokers_from_dict, parse_kafka_url
+from .utils import get_kafka_brokers_from_dict, parse_kafka_url, parse_s3_bucket_and_key
 
 store_manager = StoreManager()
 
@@ -123,7 +122,7 @@ def get_stream_pusher(stream_path: str, **kwargs):
         )
         if isinstance(
             datastore_profile,
-            (DatastoreProfileKafkaStream, DatastoreProfileKafkaTarget),
+            DatastoreProfileKafkaStream | DatastoreProfileKafkaTarget,
         ):
             attributes = datastore_profile.attributes()
             brokers = attributes.pop("brokers", None)