mlrun 1.3.3rc1__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/config.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,6 +48,10 @@ default_env_file = os.getenv("MLRUN_DEFAULT_ENV_FILE", "~/.mlrun.env")
 
 default_config = {
     "namespace": "",  # default kubernetes namespace
+    "kubernetes": {
+        "kubeconfig_path": "",  # local path to kubeconfig file (for development purposes),
+        # empty by default as the API already running inside k8s cluster
+    },
     "dbpath": "",  # db/api url
     # url to nuclio dashboard api (can be with user & token, e.g. https://username:password@dashboard-url.com)
     "nuclio_dashboard_url": "",
@@ -74,9 +78,10 @@ default_config = {
     "spark_app_image_tag": "",  # image tag to use for spark operator app runtime
     "spark_history_server_path": "",  # spark logs directory for spark history server
     "spark_operator_version": "spark-3",  # the version of the spark operator in use
-    "builder_alpine_image": "alpine:3.13.1",  # builder alpine image (as kaniko's initContainer)
     "package_path": "mlrun",  # mlrun pip package
     "default_base_image": "mlrun/mlrun",  # default base image when doing .deploy()
+    # template for project default image name. Parameter {name} will be replaced with project name
+    "default_project_image_name": ".mlrun-project-image-{name}",
     "default_project": "default",  # default project name
     "default_archive": "",  # default remote archive URL (for build tar.gz)
     "mpijob_crd_version": "",  # mpijob crd version (e.g: "v1alpha1". must be in: mlrun.runtime.MPIJobCRDVersions)
@@ -155,7 +160,7 @@ default_config = {
                 # default security context to be applied to all functions - json string base64 encoded format
                 # in camelCase format: {"runAsUser": 1000, "runAsGroup": 3000}
                 "default": "e30=",  # encoded empty dict
-                # see mlrun.api.schemas.function.SecurityContextEnrichmentModes for available options
+                # see mlrun.common.schemas.function.SecurityContextEnrichmentModes for available options
                 "enrichment_mode": "disabled",
                 # default 65534 (nogroup), set to -1 to use the user unix id or
                 # function.spec.security_context.pipelines.kfp_pod_user_unix_id for kfp pods
@@ -178,7 +183,7 @@ default_config = {
             "mpijob": "mlrun/ml-models",
         },
         # see enrich_function_preemption_spec for more info,
-        # and mlrun.api.schemas.function.PreemptionModes for available options
+        # and mlrun.common.schemas.function.PreemptionModes for available options
         "preemption_mode": "prevent",
     },
     "httpdb": {
@@ -219,7 +224,7 @@ default_config = {
         "allowed_file_paths": "s3://,gcs://,gs://,az://",
         "db_type": "sqldb",
         "max_workers": 64,
-        # See mlrun.api.schemas.APIStates for options
+        # See mlrun.common.schemas.APIStates for options
         "state": "online",
         "retry_api_call_on_exception": "enabled",
         "http_connection_timeout_keep_alive": 11,
@@ -230,10 +235,10 @@ default_config = {
             "conflict_retry_interval": None,
             # Whether to perform data migrations on initialization. enabled or disabled
             "data_migrations_mode": "enabled",
-            # Whether or not to perform database migration from sqlite to mysql on initialization
+            # Whether to perform database migration from sqlite to mysql on initialization
             "database_migration_mode": "enabled",
             "backup": {
-                # Whether or not to use db backups on initialization
+                # Whether to use db backups on initialization
                 "mode": "enabled",
                 "file_format": "db_backup_%Y%m%d%H%M.db",
                 "use_rotation": True,
@@ -244,6 +249,14 @@ default_config = {
             # None will set this to be equal to the httpdb.max_workers
             "connections_pool_size": None,
             "connections_pool_max_overflow": None,
+            # below is a db-specific configuration
+            "mysql": {
+                # comma separated mysql modes (globally) to set on runtime
+                # optional values (as per https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html#sql-mode-full):
+                #
+                # if set to "nil" or "none", nothing would be set
+                "modes": "STRICT_TRANS_TABLES",
+            },
         },
         "jobs": {
             # whether to allow to run local runtimes in the API - configurable to allow the scheduler testing to work
@@ -357,9 +370,12 @@ default_config = {
             # git+https://github.com/mlrun/mlrun@development. by default uses the version
             "mlrun_version_specifier": "",
             "kaniko_image": "gcr.io/kaniko-project/executor:v1.8.0",  # kaniko builder image
-            "kaniko_init_container_image": "alpine:3.13.1",
+            "kaniko_init_container_image": "alpine:3.18",
             # image for kaniko init container when docker registry is ECR
             "kaniko_aws_cli_image": "amazon/aws-cli:2.7.10",
+            # kaniko sometimes fails to get filesystem from image, this is a workaround to retry the process
+            # a known issue in Kaniko - https://github.com/GoogleContainerTools/kaniko/issues/1717
+            "kaniko_image_fs_extraction_retries": "3",
             # additional docker build args in json encoded base64 format
             "build_args": "",
             "pip_ca_secret_name": "",
@@ -372,18 +388,37 @@ default_config = {
         },
         "v3io_api": "",
         "v3io_framesd": "",
+        # If running from sdk and MLRUN_DBPATH is not set, the db will fallback to a nop db which will not preform any
+        # run db operations.
+        "nop_db": {
+            # if set to true, will raise an error for trying to use run db functionality
+            # if set to false, will use a nop db which will not preform any run db operations
+            "raise_error": False,
+            # if set to true, will log a warning for trying to use run db functionality while in nop db mode
+            "verbose": True,
+        },
     },
     "model_endpoint_monitoring": {
         "serving_stream_args": {"shard_count": 1, "retention_period_hours": 24},
         "drift_thresholds": {"default": {"possible_drift": 0.5, "drift_detected": 0.7}},
+        # Store prefixes are used to handle model monitoring storing policies based on project and kind, such as events,
+        # stream, and endpoints.
         "store_prefixes": {
             "default": "v3io:///users/pipelines/{project}/model-endpoints/{kind}",
             "user_space": "v3io:///projects/{project}/model-endpoints/{kind}",
+            "stream": "",
         },
+        # Offline storage path can be either relative or a full path. This path is used for general offline data
+        # storage such as the parquet file which is generated from the monitoring stream function for the drift analysis
+        "offline_storage_path": "model-endpoints/{kind}",
+        # Default http path that points to the monitoring stream nuclio function. Will be used as a stream path
+        # when the user is working in CE environment and has not provided any stream path.
+        "default_http_sink": "http://nuclio-{project}-model-monitoring-stream.mlrun.svc.cluster.local:8080",
         "batch_processing_function_branch": "master",
         "parquet_batching_max_events": 10000,
-        # See mlrun.api.schemas.ModelEndpointStoreType for available options
-        "store_type": "kv",
+        # See mlrun.common.schemas.ModelEndpointStoreType for available options
+        "store_type": "v3io-nosql",
+        "endpoint_store_connection": "",
     },
     "secret_stores": {
         "vault": {
@@ -427,15 +462,16 @@ default_config = {
         "projects_prefix": "projects",  # The UI link prefix for projects
         "url": "",  # remote/external mlrun UI url (for hyperlinks)
     },
-    "marketplace": {
-        "k8s_secrets_project_name": "-marketplace-secrets",
+    "hub": {
+        "k8s_secrets_project_name": "-hub-secrets",
         "catalog_filename": "catalog.json",
         "default_source": {
-            # Set to false to avoid creating a global source (for example in a dark site)
+            # Set false to avoid creating a global source (for example in a dark site)
             "create": True,
             "name": "mlrun_global_hub",
             "description": "MLRun global function hub",
-            "url": "https://raw.githubusercontent.com/mlrun/marketplace",
+            "url": "https://raw.githubusercontent.com/mlrun/marketplace/master",
+            "object_type": "functions",
             "channel": "master",
         },
     },
@@ -507,6 +543,27 @@ default_config = {
         # interval for stopping log collection for runs which are in a terminal state
         "stop_logs_interval": 3600,
     },
+    # Configurations for the `mlrun.package` sub-package involving packagers - logging returned outputs and parsing
+    # inputs data items:
+    "packagers": {
+        # Whether to enable packagers. True will wrap each run in the `mlrun.package.handler` decorator to log and parse
+        # using packagers.
+        "enabled": True,
+        # Whether to treat returned tuples from functions as a tuple and not as multiple returned items. If True, all
+        # returned values will be packaged together as the tuple they are returned in. Default is False to enable
+        # logging multiple returned items.
+        "pack_tuples": False,
+    },
+    # Events are currently (and only) used to audit changes and record access to MLRun entities (such as secrets)
+    "events": {
+        # supported modes "enabled", "disabled".
+        # "enabled" - events are emitted.
+        # "disabled" - a nop client is used (aka doing nothing).
+        "mode": "disabled",
+        "verbose": False,
+        # used for igz client when emitting events
+        "access_key": "",
+    },
 }
 
 _is_running_as_api = None
@@ -517,8 +574,7 @@ def is_running_as_api():
     global _is_running_as_api
 
     if _is_running_as_api is None:
-        # os.getenv will load the env var as string, and json.loads will convert it to a bool
-        _is_running_as_api = json.loads(os.getenv("MLRUN_IS_API_SERVER", "false"))
+        _is_running_as_api = os.getenv("MLRUN_IS_API_SERVER", "false").lower() == "true"
 
     return _is_running_as_api
 
@@ -926,6 +982,68 @@ class Config:
         # Get v3io access key from the environment
         return os.environ.get("V3IO_ACCESS_KEY")
 
+    def get_model_monitoring_file_target_path(
+        self,
+        project: str = "",
+        kind: str = "",
+        target: str = "online",
+        artifact_path: str = None,
+    ) -> str:
+        """Get the full path from the configuration based on the provided project and kind.
+
+        :param project:        Project name.
+        :param kind:           Kind of target path (e.g. events, log_stream, endpoints, etc.)
+        :param target:         Can be either online or offline. If the target is online, then we try to get a specific
+                               path for the provided kind. If it doesn't exist, use the default path.
+                               If the target path is offline and the offline path is already a full path in the
+                               configuration, then the result will be that path as-is. If the offline path is a
+                               relative path, then the result will be based on the project artifact path and the offline
+                               relative path. If project artifact path wasn't provided, then we use MLRun artifact
+                               path instead.
+        :param artifact_path:  Optional artifact path that will be used as a relative path. If not provided, the
+                               relative artifact path will be taken from the global MLRun artifact path.
+
+        :return: Full configured path for the provided kind.
+        """
+
+        if target != "offline":
+            store_prefix_dict = (
+                mlrun.mlconf.model_endpoint_monitoring.store_prefixes.to_dict()
+            )
+            if store_prefix_dict.get(kind):
+                # Target exist in store prefix and has a valid string value
+                return store_prefix_dict[kind].format(project=project)
+            return mlrun.mlconf.model_endpoint_monitoring.store_prefixes.default.format(
+                project=project, kind=kind
+            )
+
+        # Get the current offline path from the configuration
+        file_path = mlrun.mlconf.model_endpoint_monitoring.offline_storage_path.format(
+            project=project, kind=kind
+        )
+
+        # Absolute path
+        if any(value in file_path for value in ["://", ":///"]) or os.path.isabs(
+            file_path
+        ):
+            return file_path
+
+        # Relative path
+        else:
+            artifact_path = artifact_path or config.artifact_path
+            if artifact_path[-1] != "/":
+                artifact_path += "/"
+
+            return mlrun.utils.helpers.fill_artifact_path_template(
+                artifact_path=artifact_path + file_path, project=project
+            )
+
+    def is_ce_mode(self) -> bool:
+        # True if the setup is in CE environment
+        return isinstance(mlrun.mlconf.ce, mlrun.config.Config) and any(
+            ver in mlrun.mlconf.ce.mode for ver in ["lite", "full"]
+        )
+
 
 # Global configuration
 config = Config.from_dict(default_config)
@@ -946,7 +1064,7 @@ def _populate(skip_errors=False):
 def _do_populate(env=None, skip_errors=False):
     global config
 
-    if not os.environ.get("MLRUN_IGNORE_ENV_FILE") and not is_running_as_api():
+    if not os.environ.get("MLRUN_IGNORE_ENV_FILE"):
         if "MLRUN_ENV_FILE" in os.environ:
             env_file = os.path.expanduser(os.environ["MLRUN_ENV_FILE"])
             dotenv.load_dotenv(env_file, override=True)
@@ -983,12 +1101,10 @@ def _do_populate(env=None, skip_errors=False):
 
 
 def _validate_config(config):
-    import mlrun.k8s_utils
-
     try:
         limits_gpu = config.default_function_pod_resources.limits.gpu
         requests_gpu = config.default_function_pod_resources.requests.gpu
-        mlrun.k8s_utils.verify_gpu_requests_and_limits(
+        _verify_gpu_requests_and_limits(
             requests_gpu=requests_gpu,
             limits_gpu=limits_gpu,
         )
@@ -998,6 +1114,19 @@ def _validate_config(config):
     config.verify_security_context_enrichment_mode_is_allowed()
 
 
+def _verify_gpu_requests_and_limits(requests_gpu: str = None, limits_gpu: str = None):
+    # https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/
+    if requests_gpu and not limits_gpu:
+        raise mlrun.errors.MLRunConflictError(
+            "You cannot specify GPU requests without specifying limits"
+        )
+    if requests_gpu and limits_gpu and requests_gpu != limits_gpu:
+        raise mlrun.errors.MLRunConflictError(
+            f"When specifying both GPU requests and limits these two values must be equal, "
+            f"requests_gpu={requests_gpu}, limits_gpu={limits_gpu}"
+        )
+
+
 def _convert_resources_to_str(config: dict = None):
     resources_types = ["cpu", "memory", "gpu"]
     resource_requirements = ["requests", "limits"]
@@ -1048,15 +1177,18 @@ def read_env(env=None, prefix=env_prefix):
         cfg[path[0]] = value
 
     env_dbpath = env.get("MLRUN_DBPATH", "")
+    # expected format: https://mlrun-api.tenant.default-tenant.app.some-system.some-namespace.com
     is_remote_mlrun = (
         env_dbpath.startswith("https://mlrun-api.") and "tenant." in env_dbpath
     )
+
     # It's already a standard to set this env var to configure the v3io api, so we're supporting it (instead
     # of MLRUN_V3IO_API), in remote usage this can be auto detected from the DBPATH
     v3io_api = env.get("V3IO_API")
     if v3io_api:
         config["v3io_api"] = v3io_api
     elif is_remote_mlrun:
+        # in remote mlrun we can't use http, so we'll use https
         config["v3io_api"] = env_dbpath.replace("https://mlrun-api.", "https://webapi.")
 
     # It's already a standard to set this env var to configure the v3io framesd, so we're supporting it (instead

mlrun/data_types/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +14,12 @@
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
-from .data_types import InferOptions, ValueType, pd_schema_to_value_type
+from .data_types import (
+    InferOptions,
+    ValueType,
+    pd_schema_to_value_type,
+    python_type_to_value_type,
+)
 from .infer import DFDataInfer
 
 

mlrun/data_types/data_types.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -79,7 +79,9 @@ def pa_type_to_value_type(type_):
 
 
 def python_type_to_value_type(value_type):
-    type_name = value_type.__name__
+    type_name = (
+        value_type.__name__ if hasattr(value_type, "__name__") else str(value_type)
+    )
     type_map = {
         "int": ValueType.INT64,
         "str": ValueType.STRING,

mlrun/data_types/infer.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/data_types/spark.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@ from datetime import datetime
 from os import environ
 
 import numpy as np
+import pytz
+from pyspark.sql.functions import to_utc_timestamp
 from pyspark.sql.types import BooleanType, DoubleType, TimestampType
 
 from mlrun.utils import logger
@@ -143,6 +145,9 @@ def get_df_stats_spark(df, options, num_bins=20, sample_size=None):
         is_timestamp = isinstance(field.dataType, TimestampType)
         is_boolean = isinstance(field.dataType, BooleanType)
         if is_timestamp:
+            df_after_type_casts = df_after_type_casts.withColumn(
+                field.name, to_utc_timestamp(df_after_type_casts[field.name], "UTC")
+            )
             timestamp_columns.add(field.name)
         if is_boolean:
             boolean_columns.add(field.name)
@@ -210,11 +215,13 @@ def get_df_stats_spark(df, options, num_bins=20, sample_size=None):
         if col in timestamp_columns:
             for stat, val in stats.items():
                 if stat == "mean" or stat in original_type_stats:
-                    stats[stat] = datetime.fromtimestamp(val).isoformat()
+                    stats[stat] = datetime.fromtimestamp(val, tz=pytz.UTC).isoformat()
                 elif stat == "hist":
                     values = stats[stat][1]
                     for i in range(len(values)):
-                        values[i] = datetime.fromtimestamp(values[i]).isoformat()
+                        values[i] = datetime.fromtimestamp(
+                            values[i], tz=pytz.UTC
+                        ).isoformat()
         # for boolean values, keep mean and histogram values numeric (0 to 1 representation)
         if col in boolean_columns:
             for stat, val in stats.items():

mlrun/datastore/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,7 +33,12 @@ __all__ = [
 
 import mlrun.datastore.wasbfs
 
-from ..platforms.iguazio import KafkaOutputStream, OutputStream, parse_path
+from ..platforms.iguazio import (
+    HTTPOutputStream,
+    KafkaOutputStream,
+    OutputStream,
+    parse_path,
+)
 from ..utils import logger
 from .base import DataItem
 from .datastore import StoreManager, in_memory_store, uri_to_ipython
@@ -69,7 +74,7 @@ def get_in_memory_items():
 
 
 def get_stream_pusher(stream_path: str, **kwargs):
-    """get a stream pusher object from URL, currently only support v3io stream
+    """get a stream pusher object from URL.
 
     common kwargs::
 
@@ -87,6 +92,8 @@ def get_stream_pusher(stream_path: str, **kwargs):
         return KafkaOutputStream(
             topic, bootstrap_servers, kwargs.get("kafka_producer_options")
         )
+    elif stream_path.startswith("http://") or stream_path.startswith("https://"):
+        return HTTPOutputStream(stream_path=stream_path)
     elif "://" not in stream_path:
         return OutputStream(stream_path, **kwargs)
     elif stream_path.startswith("v3io"):

mlrun/datastore/azure_blob.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.