mlrun 1.9.0rc3__py3-none-any.whl → 1.9.0rc5__py3-none-any.whl

mlrun/__main__.py

@@ -13,6 +13,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import functools
+import importlib.metadata
 import json
 import pathlib
 import socket
@@ -25,12 +27,14 @@ from pprint import pprint
 import click
 import dotenv
 import pandas as pd
+import semver
 import yaml
 from tabulate import tabulate
 
 import mlrun
 import mlrun.common.constants as mlrun_constants
 import mlrun.common.schemas
+import mlrun.platforms
 import mlrun.utils.helpers
 from mlrun.common.helpers import parse_versioned_object_uri
 from mlrun.runtimes.mounts import auto_mount as auto_mount_modifier
@@ -63,16 +67,22 @@ from .utils.version import Version
 pd.set_option("mode.chained_assignment", None)
 
 
-def validate_base_argument(ctx, param, value):
+def validate_base_argument(ctx: click.Context, param: click.Parameter, value: str):
+    # click 8.2 expects the context to be passed to make_metavar
+    if semver.VersionInfo.parse(
+        importlib.metadata.version("click")
+    ) < semver.VersionInfo.parse("8.2.0"):
+        metavar_func = functools.partial(param.make_metavar)
+    else:
+        metavar_func = functools.partial(param.make_metavar, ctx)
     if value and value.startswith("-"):
         raise click.BadParameter(
             f"{param.human_readable_name} ({value}) cannot start with '-', ensure the command options are typed "
             f"correctly. Preferably use '--' to separate options and arguments "
-            f"e.g. 'mlrun run --option1 --option2 -- {param.make_metavar()} [--arg1|arg1] [--arg2|arg2]'",
+            f"e.g. 'mlrun run --option1 --option2 -- {metavar_func()} [--arg1|arg1] [--arg2|arg2]'",
             ctx=ctx,
             param=param,
         )
-
     return value
 
 

mlrun/artifacts/base.py

@@ -219,7 +219,7 @@ class Artifact(ModelObj):
         project=None,
         src_path: typing.Optional[str] = None,
         # All params up until here are legacy params for compatibility with legacy artifacts.
-        # TODO: remove them in 1.9.0.
+        # TODO: remove them in 1.10.0.
         metadata: ArtifactMetadata = None,
         spec: ArtifactSpec = None,
     ):
@@ -235,7 +235,7 @@ class Artifact(ModelObj):
             or src_path
         ):
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )
@@ -758,13 +758,13 @@ class LinkArtifact(Artifact):
         link_tree=None,
         project=None,
         # All params up until here are legacy params for compatibility with legacy artifacts.
-        # TODO: remove them in 1.9.0.
+        # TODO: remove them in 1.10.0.
         metadata: ArtifactMetadata = None,
         spec: LinkArtifactSpec = None,
     ):
         if key or target_path or link_iteration or link_key or link_tree or project:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )
@@ -907,7 +907,7 @@ def convert_legacy_artifact_to_new_format(
         artifact_key = f"{artifact_key}:{artifact_tag}"
     # TODO: Remove once data migration v5 is obsolete
     warnings.warn(
-        f"Converting legacy artifact '{artifact_key}' to new format. This will not be supported in MLRun 1.9.0. "
+        f"Converting legacy artifact '{artifact_key}' to new format. This will not be supported in MLRun 1.10.0. "
         f"Make sure to save the artifact/project in the new format.",
         FutureWarning,
     )

mlrun/artifacts/dataset.py

@@ -163,7 +163,7 @@ class DatasetArtifact(Artifact):
     ):
         if key or format or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )

mlrun/artifacts/model.py

@@ -152,7 +152,7 @@ class ModelArtifact(Artifact):
     ):
         if key or body or format or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )

mlrun/artifacts/plots.py

@@ -37,7 +37,7 @@ class PlotArtifact(Artifact):
     ):
         if key or body or is_inline or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )
@@ -96,7 +96,7 @@ class PlotlyArtifact(Artifact):
         """
         if key or target_path:
             warnings.warn(
-                "Artifact constructor parameters are deprecated and will be removed in 1.9.0. "
+                "Artifact constructor parameters are deprecated in 1.7.0 and will be removed in 1.10.0. "
                 "Use the metadata and spec parameters instead.",
                 DeprecationWarning,
             )

mlrun/common/constants.py

@@ -90,6 +90,13 @@ class MLRunInternalLabels:
             if not key.startswith("__") and isinstance(value, str)
         ]
 
+    @staticmethod
+    def default_run_labels_to_enrich():
+        return [
+            MLRunInternalLabels.owner,
+            MLRunInternalLabels.v3io_user,
+        ]
+
 
 class DeployStatusTextKind(mlrun.common.types.StrEnum):
     logs = "logs"

mlrun/common/runtimes/constants.py

@@ -237,7 +237,7 @@ class RunStates:
         }[pipeline_run_status]
 
 
-# TODO: remove this class in 1.9.0 - use only MlrunInternalLabels
+# TODO: remove this class in 1.10.0 - use only MlrunInternalLabels
 class RunLabels(enum.Enum):
     owner = mlrun_constants.MLRunInternalLabels.owner
     v3io_user = mlrun_constants.MLRunInternalLabels.v3io_user

mlrun/common/schemas/artifact.py

@@ -80,7 +80,7 @@ class ArtifactIdentifier(pydantic.v1.BaseModel):
 
 @deprecated(
     version="1.7.0",
-    reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.9.0. "
+    reason="mlrun.common.schemas.ArtifactsFormat is deprecated and will be removed in 1.10.0. "
     "Use mlrun.common.formatters.ArtifactFormat instead.",
     category=FutureWarning,
 )

mlrun/common/schemas/model_monitoring/model_endpoints.py

@@ -88,13 +88,18 @@ class ModelEndpointParser(abc.ABC, BaseModel):
 
     @classmethod
     def from_flat_dict(
-        cls, endpoint_dict: dict, json_parse_values: Optional[list] = None
+        cls,
+        endpoint_dict: dict,
+        json_parse_values: Optional[list] = None,
+        validate: bool = True,
     ) -> "ModelEndpointParser":
         """Create a `ModelEndpointParser` object from an endpoint dictionary
 
         :param endpoint_dict:     Model endpoint dictionary.
         :param json_parse_values: List of dictionary keys with a JSON string value that will be parsed into a
                                   dictionary using json.loads().
+        :param validate:          Whether to validate the flattened dictionary.
+                                  Skip validation to optimize performance when it is safe to do so.
         """
         if json_parse_values is None:
             json_parse_values = cls.json_parse_values()
@@ -103,6 +108,7 @@ class ModelEndpointParser(abc.ABC, BaseModel):
             model_class=cls,
             flattened_dictionary=endpoint_dict,
             json_parse_values=json_parse_values,
+            validate=validate,
         )
 
 
@@ -213,17 +219,27 @@ class ModelEndpoint(BaseModel):
         return flatten_dict
 
     @classmethod
-    def from_flat_dict(cls, endpoint_dict: dict) -> "ModelEndpoint":
+    def from_flat_dict(
+        cls, endpoint_dict: dict, validate: bool = True
+    ) -> "ModelEndpoint":
         """Create a `ModelEndpoint` object from an endpoint flattened dictionary. Because the provided dictionary
         is flattened, we pass it as is to the subclasses without splitting the keys into spec, metadata, and status.
 
         :param endpoint_dict:     Model endpoint dictionary.
+        :param validate:          Whether to validate the flattened dictionary.
+                                  Skip validation to optimize performance when it is safe to do so.
         """
 
         return cls(
-            metadata=ModelEndpointMetadata.from_flat_dict(endpoint_dict=endpoint_dict),
-            spec=ModelEndpointSpec.from_flat_dict(endpoint_dict=endpoint_dict),
-            status=ModelEndpointStatus.from_flat_dict(endpoint_dict=endpoint_dict),
+            metadata=ModelEndpointMetadata.from_flat_dict(
+                endpoint_dict=endpoint_dict, validate=validate
+            ),
+            spec=ModelEndpointSpec.from_flat_dict(
+                endpoint_dict=endpoint_dict, validate=validate
+            ),
+            status=ModelEndpointStatus.from_flat_dict(
+                endpoint_dict=endpoint_dict, validate=validate
+            ),
         )
 
     def get(self, field, default=None):
@@ -311,7 +327,10 @@ class ModelEndpointMonitoringMetricNoData(_ModelEndpointMonitoringMetricValuesBa
 
 
 def _mapping_attributes(
-    model_class: type[Model], flattened_dictionary: dict, json_parse_values: list
+    model_class: type[Model],
+    flattened_dictionary: dict,
+    json_parse_values: list,
+    validate: bool = True,
 ) -> Model:
     """Generate a `BaseModel` object with the provided dictionary attributes.
 
@@ -319,8 +338,10 @@ def _mapping_attributes(
     :param flattened_dictionary: Flattened dictionary that contains the model endpoint attributes.
     :param json_parse_values:    List of dictionary keys with a JSON string value that will be parsed into a
                                  dictionary using json.loads().
+    :param validate:             Whether to validate the flattened dictionary.
+                                 Skip validation to optimize performance when it is safe to do so.
     """
-    # Get the fields of the provided base model object. These fields will be used to filter to relevent keys
+    # Get the fields of the provided base model object. These fields will be used to filter to relevant keys
     # from the flattened dictionary.
     wanted_keys = model_class.__fields__.keys()
 
@@ -338,7 +359,10 @@ def _mapping_attributes(
             else:
                 dict_to_parse[field_key] = None
 
-    return model_class.parse_obj(dict_to_parse)
+    if validate:
+        return model_class.parse_obj(dict_to_parse)
+
+    return model_class.construct(**dict_to_parse)
 
 
 def _json_loads_if_not_none(field: Any) -> Any:

mlrun/common/schemas/pipeline.py

@@ -22,7 +22,7 @@ import mlrun.common.types
 
 @deprecated(
     version="1.7.0",
-    reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.9.0. "
+    reason="mlrun.common.schemas.PipelinesFormat is deprecated and will be removed in 1.10.0. "
     "Use mlrun.common.formatters.PipelineFormat instead.",
     category=FutureWarning,
 )

mlrun/common/schemas/project.py

@@ -26,7 +26,7 @@ from .object import ObjectKind, ObjectStatus
 
 @deprecated(
     version="1.7.0",
-    reason="mlrun.common.schemas.ProjectsFormat is deprecated and will be removed in 1.9.0. "
+    reason="mlrun.common.schemas.ProjectsFormat is deprecated and will be removed in 1.10.0. "
     "Use mlrun.common.formatters.ProjectFormat instead.",
     category=FutureWarning,
 )

mlrun/common/schemas/runs.py

@@ -28,7 +28,7 @@ class RunIdentifier(pydantic.v1.BaseModel):
 
 @deprecated(
     version="1.7.0",
-    reason="mlrun.common.schemas.RunsFormat is deprecated and will be removed in 1.9.0. "
+    reason="mlrun.common.schemas.RunsFormat is deprecated and will be removed in 1.10.0. "
     "Use mlrun.common.formatters.RunFormat instead.",
     category=FutureWarning,
 )

mlrun/config.py

@@ -482,7 +482,7 @@ default_config = {
             "project_owners_cache_ttl": "30 seconds",
             # access key to be used when the leader is iguazio and polling is done from it
             "iguazio_access_key": "",
-            "iguazio_list_projects_default_page_size": 200,
+            "iguazio_list_projects_default_page_size": 500,
             "iguazio_client_job_cache_ttl": "20 minutes",
             "nuclio_project_deletion_verification_timeout": "300 seconds",
             "nuclio_project_deletion_verification_interval": "5 seconds",
@@ -591,17 +591,17 @@ default_config = {
         },
         "writer_stream_args": {
             "v3io": {
-                "shard_count": 1,
+                "shard_count": 4,
                 "retention_period_hours": 24,
-                "num_workers": 1,
+                "num_workers": 4,
                 "min_replicas": 1,
                 "max_replicas": 1,
             },
             "kafka": {
-                "partition_count": 1,
+                "partition_count": 4,
                 # TODO: add retention period configuration
                 "replication_factor": 1,
-                "num_workers": 1,
+                "num_workers": 4,
                 "min_replicas": 1,
                 "max_replicas": 1,
             },
@@ -634,12 +634,6 @@ default_config = {
         "offline_storage_path": "model-endpoints/{kind}",
         "parquet_batching_max_events": 10_000,
         "parquet_batching_timeout_secs": timedelta(minutes=1).total_seconds(),
-        "tdengine": {
-            "run_directly": True,
-            # timeout and retry are ignored when run_directly is set to True
-            "timeout": 10,
-            "retries": 1,
-        },
     },
     "secret_stores": {
         # Use only in testing scenarios (such as integration tests) to avoid using k8s for secrets (will use in-memory

mlrun/datastore/datastore.py

@@ -111,7 +111,7 @@ def schema_to_store(schema):
 
 def uri_to_ipython(link):
     schema, endpoint, parsed_url = parse_url(link)
-    if schema in [DB_SCHEMA, "memory"]:
+    if schema in [DB_SCHEMA, "memory", "ds"]:
         return ""
     return schema_to_store(schema).uri_to_ipython(endpoint, parsed_url.path)
 

mlrun/datastore/datastore_profile.py

@@ -165,9 +165,9 @@ class DatastoreProfileKafkaTarget(DatastoreProfile):
                 self.brokers = self.bootstrap_servers
                 self.bootstrap_servers = None
             warnings.warn(
-                "'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.9.0, "
+                "'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.10.0, "
                 "use 'brokers' instead.",
-                # TODO: Remove this in 1.9.0
+                # TODO: Remove this in 1.10.0
                 FutureWarning,
             )
 

mlrun/datastore/sources.py

@@ -794,12 +794,12 @@ class SnowflakeSource(BaseSourceDriver):
         warehouse: Optional[str] = None,
         **kwargs,
     ):
-        # TODO: Remove in 1.9.0
+        # TODO: Remove in 1.10.0
         if schema:
             warnings.warn(
-                "schema is deprecated in 1.7.0, and will be removed in 1.9.0, please use db_schema"
+                "schema is deprecated in 1.7.0, and will be removed in 1.10.0, please use db_schema"
             )
-        db_schema = db_schema or schema  # TODO: Remove in 1.9.0
+        db_schema = db_schema or schema  # TODO: Remove in 1.10.0
 
         attributes = attributes or {}
         if url:

mlrun/datastore/targets.py

@@ -443,8 +443,8 @@ class BaseStoreTarget(DataTargetBase):
         self.credentials_prefix = credentials_prefix
         if credentials_prefix:
             warnings.warn(
-                "The 'credentials_prefix' parameter is deprecated and will be removed in "
-                "1.9.0. Please use datastore profiles instead.",
+                "The 'credentials_prefix' parameter is deprecated in 1.7.0 and will be removed in "
+                "1.10.0. Please use datastore profiles instead.",
                 FutureWarning,
             )
 
@@ -1671,7 +1671,7 @@ class KafkaTarget(BaseStoreTarget):
     ):
         attrs = {}
 
-        # TODO: Remove this in 1.9.0
+        # TODO: Remove this in 1.10.0
         if bootstrap_servers:
             if brokers:
                 raise mlrun.errors.MLRunInvalidArgumentError(
@@ -1679,7 +1679,7 @@ class KafkaTarget(BaseStoreTarget):
                     "'bootstrap_servers' parameter. Please use 'brokers' only."
                 )
             warnings.warn(
-                "'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.9.0, "
+                "'bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in 1.10.0, "
                 "use 'brokers' instead.",
                 FutureWarning,
             )

mlrun/datastore/utils.py

@@ -176,8 +176,8 @@ def get_kafka_brokers_from_dict(options: dict, pop=False) -> typing.Optional[str
     kafka_bootstrap_servers = get_or_pop("kafka_bootstrap_servers", None)
     if kafka_bootstrap_servers:
         warnings.warn(
-            "The 'kafka_bootstrap_servers' parameter is deprecated and will be removed in "
-            "1.9.0. Please pass the 'kafka_brokers' parameter instead.",
+            "The 'kafka_bootstrap_servers' parameter is deprecated in 1.7.0 and will be removed in "
+            "1.10.0. Please pass the 'kafka_brokers' parameter instead.",
             FutureWarning,
         )
     return kafka_bootstrap_servers

mlrun/db/base.py

@@ -441,10 +441,10 @@ class RunDBInterface(ABC):
     ) -> dict:
         pass
 
-    # TODO: remove in 1.9.0
+    # TODO: remove in 1.10.0
     @deprecated(
-        version="1.9.0",
-        reason="'list_features' will be removed in 1.9.0, use 'list_features_v2' instead",
+        version="1.7.0",
+        reason="'list_features' will be removed in 1.10.0, use 'list_features_v2' instead",
         category=FutureWarning,
     )
     @abstractmethod
@@ -469,10 +469,10 @@ class RunDBInterface(ABC):
     ) -> mlrun.common.schemas.FeaturesOutputV2:
         pass
 
-    # TODO: remove in 1.9.0
+    # TODO: remove in 1.10.0
     @deprecated(
-        version="1.9.0",
-        reason="'list_entities' will be removed in 1.9.0, use 'list_entities_v2' instead",
+        version="1.7.0",
+        reason="'list_entities' will be removed in 1.10.0, use 'list_entities_v2' instead",
         category=FutureWarning,
     )
     @abstractmethod
@@ -734,7 +734,8 @@ class RunDBInterface(ABC):
         labels: Optional[Union[str, dict[str, Optional[str]], list[str]]] = None,
         start: Optional[datetime.datetime] = None,
         end: Optional[datetime.datetime] = None,
-        tsdb_metrics: bool = True,
+        tsdb_metrics: bool = False,
+        metric_list: Optional[list[str]] = None,
         top_level: bool = False,
         uids: Optional[list[str]] = None,
         latest_only: bool = False,
@@ -750,6 +751,7 @@ class RunDBInterface(ABC):
         function_tag: Optional[str] = None,
         endpoint_id: Optional[str] = None,
         tsdb_metrics: bool = True,
+        metric_list: Optional[list[str]] = None,
         feature_analysis: bool = False,
     ) -> mlrun.common.schemas.ModelEndpoint:
         pass