mlrun 1.6.4rc2__py3-none-any.whl → 1.7.0rc20__py3-none-any.whl

mlrun/common/schemas/api_gateway.py

@@ -0,0 +1,148 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import typing
+from typing import Optional
+
+import pydantic
+
+import mlrun.common.types
+from mlrun.common.constants import MLRUN_FUNCTIONS_ANNOTATION
+
+
+class APIGatewayAuthenticationMode(mlrun.common.types.StrEnum):
+    basic = "basicAuth"
+    none = "none"
+    access_key = "accessKey"
+
+    @classmethod
+    def from_str(cls, authentication_mode: str):
+        if authentication_mode == "none":
+            return cls.none
+        elif authentication_mode == "basicAuth":
+            return cls.basic
+        elif authentication_mode == "accessKey":
+            return cls.access_key
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Authentication mode `{authentication_mode}` is not supported",
+            )
+
+
+class APIGatewayState(mlrun.common.types.StrEnum):
+    none = ""
+    ready = "ready"
+    error = "error"
+    waiting_for_provisioning = "waitingForProvisioning"
+
+
+class _APIGatewayBaseModel(pydantic.BaseModel):
+    class Config:
+        extra = pydantic.Extra.allow
+
+
+class APIGatewayMetadata(_APIGatewayBaseModel):
+    name: str
+    namespace: Optional[str]
+    labels: Optional[dict] = {}
+    annotations: Optional[dict] = {}
+
+
+class APIGatewayBasicAuth(_APIGatewayBaseModel):
+    username: str
+    password: str
+
+
+class APIGatewayUpstream(_APIGatewayBaseModel):
+    kind: Optional[str] = "nucliofunction"
+    nucliofunction: dict[str, str]
+    percentage: Optional[int] = 0
+    port: Optional[int] = 0
+
+
+class APIGatewaySpec(_APIGatewayBaseModel):
+    name: str
+    description: Optional[str]
+    path: Optional[str] = "/"
+    authenticationMode: Optional[APIGatewayAuthenticationMode] = (
+        APIGatewayAuthenticationMode.none
+    )
+    upstreams: list[APIGatewayUpstream]
+    authentication: Optional[dict[str, Optional[APIGatewayBasicAuth]]]
+    host: Optional[str]
+
+
+class APIGatewayStatus(_APIGatewayBaseModel):
+    name: Optional[str]
+    state: Optional[APIGatewayState]
+
+
+class APIGateway(_APIGatewayBaseModel):
+    metadata: APIGatewayMetadata
+    spec: APIGatewaySpec
+    status: Optional[APIGatewayStatus]
+
+    def get_function_names(self):
+        return [
+            upstream.nucliofunction.get("name")
+            for upstream in self.spec.upstreams
+            if upstream.nucliofunction.get("name")
+        ]
+
+    def enrich_mlrun_function_names(self):
+        upstream_with_nuclio_names = []
+        mlrun_function_uris = []
+        for upstream in self.spec.upstreams:
+            uri = upstream.nucliofunction.get("name")
+            project, function_name, tag, _ = (
+                mlrun.common.helpers.parse_versioned_object_uri(uri)
+            )
+            upstream.nucliofunction["name"] = (
+                mlrun.runtimes.nuclio.function.get_fullname(function_name, project, tag)
+            )
+
+            upstream_with_nuclio_names.append(upstream)
+            mlrun_function_uris.append(uri)
+
+        self.spec.upstreams = upstream_with_nuclio_names
+        if len(mlrun_function_uris) == 1:
+            self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = mlrun_function_uris[
+                0
+            ]
+        elif len(mlrun_function_uris) == 2:
+            self.metadata.annotations[MLRUN_FUNCTIONS_ANNOTATION] = "&".join(
+                mlrun_function_uris
+            )
+        return self
+
+    def replace_nuclio_names_with_mlrun_uri(self):
+        mlrun_functions = self.metadata.annotations.get(MLRUN_FUNCTIONS_ANNOTATION)
+        if mlrun_functions:
+            mlrun_function_uris = (
+                mlrun_functions.split("&")
+                if "&" in mlrun_functions
+                else [mlrun_functions]
+            )
+            if len(mlrun_function_uris) != len(self.spec.upstreams):
+                raise mlrun.errors.MLRunValueError(
+                    "Error when translating nuclio names to mlrun names in api gateway:"
+                    "  number of functions doesn't match the mlrun functions in annotation"
+                )
+            for i in range(len(mlrun_function_uris)):
+                self.spec.upstreams[i].nucliofunction["name"] = mlrun_function_uris[i]
+        return self
+
+
+class APIGatewaysOutput(_APIGatewayBaseModel):
+    api_gateways: typing.Optional[dict[str, APIGateway]] = {}

mlrun/common/schemas/artifact.py

@@ -30,7 +30,7 @@ class ArtifactCategories(mlrun.common.types.StrEnum):
     # and should not be used as such
     link = "link"
 
-    def to_kinds_filter(self) -> typing.Tuple[typing.List[str], bool]:
+    def to_kinds_filter(self) -> tuple[list[str], bool]:
         link_kind = ArtifactCategories.link.value
 
         if self.value == ArtifactCategories.model.value:
@@ -58,11 +58,6 @@ class ArtifactIdentifier(pydantic.BaseModel):
     # hash: typing.Optional[str]
 
 
-class ArtifactsFormat(mlrun.common.types.StrEnum):
-    # TODO: add a format that returns a minimal response
-    full = "full"
-
-
 class ArtifactMetadata(pydantic.BaseModel):
     key: str
     project: str
@@ -81,8 +76,8 @@ class ArtifactSpec(pydantic.BaseModel):
     inline: typing.Optional[str]
     size: typing.Optional[int]
     db_key: typing.Optional[str]
-    extra_data: typing.Optional[typing.Dict[str, typing.Any]]
-    unpackaging_instructions: typing.Optional[typing.Dict[str, typing.Any]]
+    extra_data: typing.Optional[dict[str, typing.Any]]
+    unpackaging_instructions: typing.Optional[dict[str, typing.Any]]
 
     class Config:
         extra = pydantic.Extra.allow
@@ -93,3 +88,18 @@ class Artifact(pydantic.BaseModel):
     metadata: ArtifactMetadata
     spec: ArtifactSpec
     status: ObjectStatus
+
+
+class ArtifactsDeletionStrategies(mlrun.common.types.StrEnum):
+    """Artifacts deletion strategies types."""
+
+    metadata_only = "metadata-only"
+    """Only removes the artifact db record, leaving all related artifact data in-place"""
+
+    data_optional = "data-optional"
+    """Delete the artifact data of the artifact as a best-effort.
+    If artifact data deletion fails still try to delete the artifact db record"""
+
+    data_force = "data-force"
+    """Delete the artifact data, and if cannot delete it fail the deletion
+    and don’t delete the artifact db record"""

mlrun/common/schemas/auth.py

@@ -58,8 +58,11 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
     pipeline = "pipeline"
     hub_source = "hub-source"
     workflow = "workflow"
+    alert = "alert"
+    alert_templates = "alert-templates"
+    event = "event"
     datastore_profile = "datastore-profile"
-    api_gateways = "api-gateways"
+    api_gateway = "api-gateway"
 
     def to_resource_string(
         self,
@@ -83,6 +86,9 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             AuthorizationResourceTypes.schedule: "/projects/{project_name}/schedules/{resource_name}",
             AuthorizationResourceTypes.secret: "/projects/{project_name}/secrets/{resource_name}",
             AuthorizationResourceTypes.run: "/projects/{project_name}/runs/{resource_name}",
+            AuthorizationResourceTypes.event: "/projects/{project_name}/events/{resource_name}",
+            AuthorizationResourceTypes.alert: "/projects/{project_name}/alerts/{resource_name}",
+            AuthorizationResourceTypes.alert_templates: "/alert-templates/{resource_name}",
             # runtime resource doesn't have an identifier, we don't need any auth granularity behind project level
             AuthorizationResourceTypes.runtime_resource: "/projects/{project_name}/runtime-resources",
             AuthorizationResourceTypes.model_endpoint: "/projects/{project_name}/model-endpoints/{resource_name}",
@@ -94,7 +100,7 @@ class AuthorizationResourceTypes(mlrun.common.types.StrEnum):
             AuthorizationResourceTypes.hub_source: "/marketplace/sources",
             # workflow define how to run a pipeline and can be considered as the specification of a pipeline.
             AuthorizationResourceTypes.workflow: "/projects/{project_name}/workflows/{resource_name}",
-            AuthorizationResourceTypes.api_gateways: "/projects/{project_name}/api-gateways",
+            AuthorizationResourceTypes.api_gateway: "/projects/{project_name}/api-gateways/{resource_name}",
         }[self].format(project_name=project_name, resource_name=resource_name)
 
 
@@ -115,17 +121,17 @@ class AuthInfo(pydantic.BaseModel):
     data_session: typing.Optional[str] = None
     access_key: typing.Optional[str] = None
     user_id: typing.Optional[str] = None
-    user_group_ids: typing.List[str] = []
+    user_group_ids: list[str] = []
     user_unix_id: typing.Optional[int] = None
     projects_role: typing.Optional[ProjectsRole] = None
-    planes: typing.List[str] = []
+    planes: list[str] = []
 
     def to_nuclio_auth_info(self):
         if self.session != "":
             return NuclioAuthInfo(password=self.session, mode=NuclioAuthKinds.iguazio)
         return None
 
-    def get_member_ids(self) -> typing.List[str]:
+    def get_member_ids(self) -> list[str]:
         member_ids = []
         if self.user_id:
             member_ids.append(self.user_id)

mlrun/common/schemas/background_task.py

@@ -61,4 +61,4 @@ class BackgroundTask(pydantic.BaseModel):
 
 
 class BackgroundTaskList(pydantic.BaseModel):
-    background_tasks: typing.List[BackgroundTask]
+    background_tasks: list[BackgroundTask]

mlrun/common/schemas/client_spec.py

@@ -28,7 +28,8 @@ class ClientSpec(pydantic.BaseModel):
     mpijob_crd_version: typing.Optional[str]
     ui_url: typing.Optional[str]
     artifact_path: typing.Optional[str]
-    feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
+    feature_store_data_prefixes: typing.Optional[dict[str, str]]
+    feature_store_default_targets: typing.Optional[str]
     spark_app_image: typing.Optional[str]
     spark_app_image_tag: typing.Optional[str]
     spark_history_server_path: typing.Optional[str]
@@ -58,6 +59,7 @@ class ClientSpec(pydantic.BaseModel):
     sql_url: typing.Optional[str]
     model_endpoint_monitoring_store_type: typing.Optional[str]
     model_endpoint_monitoring_endpoint_store_connection: typing.Optional[str]
+    model_monitoring_tsdb_connection: typing.Optional[str]
     ce: typing.Optional[dict]
     # not passing them as one object as it possible client user would like to override only one of the params
     calculate_artifact_hash: typing.Optional[str]
@@ -65,3 +67,4 @@ class ClientSpec(pydantic.BaseModel):
     logs: typing.Optional[dict]
     packagers: typing.Optional[dict]
     external_platform_tracking: typing.Optional[dict]
+    alerts_mode: typing.Optional[str]

mlrun/common/schemas/feature_store.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -46,8 +46,8 @@ class Entity(BaseModel):
 
 
 class FeatureSetSpec(ObjectSpec):
-    entities: List[Entity] = []
-    features: List[Feature] = []
+    entities: list[Entity] = []
+    features: list[Feature] = []
     engine: Optional[str] = Field(default="storey")
 
 
@@ -65,7 +65,7 @@ class FeatureSet(BaseModel):
 class EntityRecord(BaseModel):
     name: str
     value_type: str
-    labels: List[LabelRecord]
+    labels: list[LabelRecord]
 
     class Config:
         orm_mode = True
@@ -74,31 +74,31 @@ class EntityRecord(BaseModel):
 class FeatureRecord(BaseModel):
     name: str
     value_type: str
-    labels: List[LabelRecord]
+    labels: list[LabelRecord]
 
     class Config:
         orm_mode = True
 
 
 class FeatureSetRecord(ObjectRecord):
-    entities: List[EntityRecord]
-    features: List[FeatureRecord]
+    entities: list[EntityRecord]
+    features: list[FeatureRecord]
 
     class Config:
         orm_mode = True
 
 
 class FeatureSetsOutput(BaseModel):
-    feature_sets: List[FeatureSet]
+    feature_sets: list[FeatureSet]
 
 
 class FeatureSetsTagsOutput(BaseModel):
-    tags: List[str] = []
+    tags: list[str] = []
 
 
 class FeatureSetDigestSpec(BaseModel):
-    entities: List[Entity]
-    features: List[Feature]
+    entities: list[Entity]
+    features: list[Feature]
 
 
 class FeatureSetDigestOutput(BaseModel):
@@ -112,7 +112,7 @@ class FeatureListOutput(BaseModel):
 
 
 class FeaturesOutput(BaseModel):
-    features: List[FeatureListOutput]
+    features: list[FeatureListOutput]
 
 
 class EntityListOutput(BaseModel):
@@ -121,7 +121,7 @@ class EntityListOutput(BaseModel):
 
 
 class EntitiesOutput(BaseModel):
-    entities: List[EntityListOutput]
+    entities: list[EntityListOutput]
 
 
 class FeatureVector(BaseModel):
@@ -140,11 +140,11 @@ class FeatureVectorRecord(ObjectRecord):
 
 
 class FeatureVectorsOutput(BaseModel):
-    feature_vectors: List[FeatureVector]
+    feature_vectors: list[FeatureVector]
 
 
 class FeatureVectorsTagsOutput(BaseModel):
-    tags: List[str] = []
+    tags: list[str] = []
 
 
 class DataSource(BaseModel):
@@ -167,7 +167,7 @@ class DataTarget(BaseModel):
 
 class FeatureSetIngestInput(BaseModel):
     source: Optional[DataSource]
-    targets: Optional[List[DataTarget]]
+    targets: Optional[list[DataTarget]]
     infer_options: Optional[int]
     credentials: Credentials = Credentials()
 

mlrun/common/schemas/frontend_spec.py

@@ -53,20 +53,21 @@ class FeatureFlags(pydantic.BaseModel):
 class FrontendSpec(pydantic.BaseModel):
     jobs_dashboard_url: typing.Optional[str]
     model_monitoring_dashboard_url: typing.Optional[str]
-    abortable_function_kinds: typing.List[str] = []
+    abortable_function_kinds: list[str] = []
     feature_flags: FeatureFlags
     default_function_priority_class_name: typing.Optional[str]
-    valid_function_priority_class_names: typing.List[str] = []
-    default_function_image_by_kind: typing.Dict[str, str] = {}
+    valid_function_priority_class_names: list[str] = []
+    default_function_image_by_kind: dict[str, str] = {}
     function_deployment_target_image_template: typing.Optional[str]
     function_deployment_target_image_name_prefix_template: str
-    function_deployment_target_image_registries_to_enforce_prefix: typing.List[str] = []
+    function_deployment_target_image_registries_to_enforce_prefix: list[str] = []
     function_deployment_mlrun_requirement: typing.Optional[str]
     auto_mount_type: typing.Optional[str]
-    auto_mount_params: typing.Dict[str, str] = {}
+    auto_mount_params: dict[str, str] = {}
     default_artifact_path: str
     default_function_pod_resources: Resources = Resources()
     default_function_preemption_mode: str
-    feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
-    allowed_artifact_path_prefixes_list: typing.List[str]
+    feature_store_data_prefixes: typing.Optional[dict[str, str]]
+    allowed_artifact_path_prefixes_list: list[str]
     ce: typing.Optional[dict]
+    internal_labels: list[str] = []

mlrun/common/schemas/function.py

@@ -45,6 +45,9 @@ class FunctionState:
     # same goes for the build which is not coming from the pod, but is used and we can't just omit it for BC reasons
     build = "build"
 
+    # for pipeline steps
+    skipped = "skipped"
+
     @classmethod
     def get_function_state_from_pod_state(cls, pod_state: str):
         if pod_state == "succeeded":
@@ -60,6 +63,7 @@ class FunctionState:
         return [
             cls.ready,
             cls.error,
+            cls.skipped,
         ]
 
 
@@ -106,7 +110,7 @@ class ServiceAccount(pydantic.BaseModel):
 
 
 class StateThresholds(pydantic.BaseModel):
-    default: typing.Optional[typing.Dict[str, str]]
+    default: typing.Optional[dict[str, str]]
 
 
 class FunctionSpec(pydantic.BaseModel):

mlrun/common/schemas/hub.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime, timezone
-from typing import Dict, List, Optional
+from typing import Optional
 
 from pydantic import BaseModel, Extra, Field
 
@@ -56,36 +56,29 @@ class HubSource(BaseModel):
     status: Optional[ObjectStatus] = ObjectStatus(state="created")
 
     def get_full_uri(self, relative_path):
-        return "{base}/{object_type}/{channel}/{relative_path}".format(
-            base=self.spec.path,
-            object_type=self.spec.object_type,
-            channel=self.spec.channel,
-            relative_path=relative_path,
-        )
+        return f"{self.spec.path}/{self.spec.object_type}/{self.spec.channel}/{relative_path}"
 
     def get_catalog_uri(self):
-        return self.get_full_uri(mlrun.config.config.hub.catalog_filename)
+        return self.get_full_uri(mlrun.mlconf.hub.catalog_filename)
 
     @classmethod
     def generate_default_source(cls):
-        if not mlrun.config.config.hub.default_source.create:
+        if not mlrun.mlconf.hub.default_source.create:
             return None
 
         now = datetime.now(timezone.utc)
         hub_metadata = HubObjectMetadata(
-            name=mlrun.config.config.hub.default_source.name,
-            description=mlrun.config.config.hub.default_source.description,
+            name=mlrun.mlconf.hub.default_source.name,
+            description=mlrun.mlconf.hub.default_source.description,
             created=now,
             updated=now,
         )
         return cls(
             metadata=hub_metadata,
             spec=HubSourceSpec(
-                path=mlrun.config.config.hub.default_source.url,
-                channel=mlrun.config.config.hub.default_source.channel,
-                object_type=HubSourceType(
-                    mlrun.config.config.hub.default_source.object_type
-                ),
+                path=mlrun.mlconf.hub.default_source.url,
+                channel=mlrun.mlconf.hub.default_source.channel,
+                object_type=HubSourceType(mlrun.mlconf.hub.default_source.object_type),
             ),
             status=ObjectStatus(state="created"),
         )
@@ -120,7 +113,7 @@ class HubItemMetadata(HubObjectMetadata):
 
 class HubItemSpec(ObjectSpec):
     item_uri: str
-    assets: Dict[str, str] = {}
+    assets: dict[str, str] = {}
 
 
 class HubItem(BaseModel):
@@ -133,4 +126,4 @@ class HubItem(BaseModel):
 class HubCatalog(BaseModel):
     kind: ObjectKind = Field(ObjectKind.hub_catalog, const=True)
     channel: str
-    catalog: List[HubItem]
+    catalog: list[HubItem]

mlrun/common/schemas/memory_reports.py

@@ -19,7 +19,7 @@ import pydantic
 
 
 class MostCommonObjectTypesReport(pydantic.BaseModel):
-    object_types: typing.List[typing.Tuple[str, int]]
+    object_types: list[tuple[str, int]]
 
 
 class ObjectTypeReport(pydantic.BaseModel):
@@ -27,4 +27,4 @@ class ObjectTypeReport(pydantic.BaseModel):
     sample_size: int
     start_index: typing.Optional[int]
     max_depth: int
-    object_report: typing.List[typing.Dict[str, typing.Any]]
+    object_report: list[dict[str, typing.Any]]

mlrun/common/schemas/model_monitoring/__init__.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
 
 from .constants import (
     ControllerPolicy,
@@ -25,17 +23,29 @@ from .constants import (
     FeatureSetFeatures,
     FileTargetKind,
     FunctionURI,
+    MetricData,
     ModelEndpointTarget,
     ModelMonitoringMode,
     ModelMonitoringStoreKinds,
     MonitoringFunctionNames,
+    PredictionsQueryConstants,
     ProjectSecretKeys,
+    PrometheusEndpoints,
     PrometheusMetric,
-    TimeSeriesTarget,
+    ResultData,
+    ResultKindApp,
+    SchedulingKeys,
+    SpecialApps,
+    TDEngineSuperTables,
+    TSDBTarget,
+    V3IOTSDBTables,
     VersionedModel,
+    WriterEvent,
+    WriterEventKind,
 )
 from .grafana import (
     GrafanaColumn,
+    GrafanaColumnType,
     GrafanaDataPoint,
     GrafanaNumberColumn,
     GrafanaStringColumn,
@@ -48,6 +58,11 @@ from .model_endpoints import (
     ModelEndpoint,
     ModelEndpointList,
     ModelEndpointMetadata,
+    ModelEndpointMonitoringMetric,
+    ModelEndpointMonitoringMetricNoData,
+    ModelEndpointMonitoringMetricType,
+    ModelEndpointMonitoringMetricValues,
+    ModelEndpointMonitoringResultValues,
     ModelEndpointSpec,
     ModelEndpointStatus,
 )