mlrun 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

mlrun/{api → common}/schemas/background_task.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,12 +17,12 @@ import typing
 
 import pydantic
 
-import mlrun.api.utils.helpers
+import mlrun.common.types
 
 from .object import ObjectKind
 
 
-class BackgroundTaskState(mlrun.api.utils.helpers.StrEnum):
+class BackgroundTaskState(mlrun.common.types.StrEnum):
     succeeded = "succeeded"
     failed = "failed"
     running = "running"

mlrun/{api → common}/schemas/client_spec.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mlrun/{api → common}/schemas/clusterization_spec.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ import typing
 
 import pydantic
 
-import mlrun.api.utils.helpers
+import mlrun.common.types
 
 
 class ClusterizationSpec(pydantic.BaseModel):
@@ -24,6 +24,6 @@ class ClusterizationSpec(pydantic.BaseModel):
     chief_version: typing.Optional[str]
 
 
-class WaitForChiefToReachOnlineStateFeatureFlag(mlrun.api.utils.helpers.StrEnum):
+class WaitForChiefToReachOnlineStateFeatureFlag(mlrun.common.types.StrEnum):
     enabled = "enabled"
     disabled = "disabled"

mlrun/{api → common}/schemas/constants.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,11 @@
 #
 import mergedeep
 
-import mlrun.api.utils.helpers
+import mlrun.common.types
 import mlrun.errors
 
 
-class PatchMode(mlrun.api.utils.helpers.StrEnum):
+class PatchMode(mlrun.common.types.StrEnum):
     replace = "replace"
     additive = "additive"
 
@@ -33,7 +33,7 @@ class PatchMode(mlrun.api.utils.helpers.StrEnum):
             )
 
 
-class DeletionStrategy(mlrun.api.utils.helpers.StrEnum):
+class DeletionStrategy(mlrun.common.types.StrEnum):
     restrict = "restrict"
     restricted = "restricted"
     cascade = "cascade"
@@ -97,7 +97,7 @@ class HeaderNames:
     ui_clear_cache = f"{headers_prefix}ui-clear-cache"
 
 
-class FeatureStorePartitionByField(mlrun.api.utils.helpers.StrEnum):
+class FeatureStorePartitionByField(mlrun.common.types.StrEnum):
     name = "name"  # Supported for feature-store objects
 
     def to_partition_by_db_field(self, db_cls):
@@ -109,7 +109,7 @@ class FeatureStorePartitionByField(mlrun.api.utils.helpers.StrEnum):
             )
 
 
-class RunPartitionByField(mlrun.api.utils.helpers.StrEnum):
+class RunPartitionByField(mlrun.common.types.StrEnum):
     name = "name"  # Supported for runs objects
 
     def to_partition_by_db_field(self, db_cls):
@@ -121,7 +121,7 @@ class RunPartitionByField(mlrun.api.utils.helpers.StrEnum):
             )
 
 
-class SortField(mlrun.api.utils.helpers.StrEnum):
+class SortField(mlrun.common.types.StrEnum):
     created = "created"
     updated = "updated"
 
@@ -139,7 +139,7 @@ class SortField(mlrun.api.utils.helpers.StrEnum):
             )
 
 
-class OrderType(mlrun.api.utils.helpers.StrEnum):
+class OrderType(mlrun.common.types.StrEnum):
     asc = "asc"
     desc = "desc"
 
@@ -170,6 +170,19 @@ class APIStates:
     def terminal_states():
         return [APIStates.online, APIStates.offline]
 
+    @staticmethod
+    def description(state: str):
+        return {
+            APIStates.online: "API is online",
+            APIStates.waiting_for_migrations: "API is waiting for migrations to be triggered. "
+            "Send POST request to /api/operations/migrations to trigger it",
+            APIStates.migrations_in_progress: "Migrations are in progress",
+            APIStates.migrations_failed: "Migrations failed, API can't be started",
+            APIStates.migrations_completed: "Migrations completed, API is waiting to become online",
+            APIStates.offline: "API is offline",
+            APIStates.waiting_for_chief: "API is waiting for chief to be ready",
+        }.get(state, f"Unknown API state '{state}'")
+
 
 class ClusterizationRole:
     chief = "chief"

mlrun/common/schemas/events.py

@@ -0,0 +1,36 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import mlrun.common.types
+
+
+class EventsModes(mlrun.common.types.StrEnum):
+    enabled = "enabled"
+    disabled = "disabled"
+
+
+class EventClientKinds(mlrun.common.types.StrEnum):
+    iguazio = "iguazio"
+    nop = "nop"
+
+
+class SecretEventActions(mlrun.common.types.StrEnum):
+    created = "created"
+    updated = "updated"
+    deleted = "deleted"
+
+
+class AuthSecretEventActions(mlrun.common.types.StrEnum):
+    created = "created"
+    updated = "updated"

mlrun/{api → common}/schemas/feature_store.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,6 +48,7 @@ class Entity(BaseModel):
 class FeatureSetSpec(ObjectSpec):
     entities: List[Entity] = []
     features: List[Feature] = []
+    engine: Optional[str] = Field(default="storey")
 
 
 class FeatureSet(BaseModel):

mlrun/{api → common}/schemas/frontend_spec.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,29 +16,29 @@ import typing
 
 import pydantic
 
-import mlrun.api.utils.helpers
+import mlrun.common.types
 
 from .k8s import Resources
 
 
-class ProjectMembershipFeatureFlag(mlrun.api.utils.helpers.StrEnum):
+class ProjectMembershipFeatureFlag(mlrun.common.types.StrEnum):
     enabled = "enabled"
     disabled = "disabled"
 
 
-class PreemptionNodesFeatureFlag(mlrun.api.utils.helpers.StrEnum):
+class PreemptionNodesFeatureFlag(mlrun.common.types.StrEnum):
     enabled = "enabled"
     disabled = "disabled"
 
 
-class AuthenticationFeatureFlag(mlrun.api.utils.helpers.StrEnum):
+class AuthenticationFeatureFlag(mlrun.common.types.StrEnum):
     none = "none"
     basic = "basic"
     bearer = "bearer"
     iguazio = "iguazio"
 
 
-class NuclioStreamsFeatureFlag(mlrun.api.utils.helpers.StrEnum):
+class NuclioStreamsFeatureFlag(mlrun.common.types.StrEnum):
     enabled = "enabled"
     disabled = "disabled"
 
@@ -67,6 +67,7 @@ class FrontendSpec(pydantic.BaseModel):
     default_function_pod_resources: Resources = Resources()
     default_function_preemption_mode: str
     feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
+    allowed_artifact_path_prefixes_list: typing.List[str]
 
     # ce_mode is deprecated, we will use the full ce config instead and ce_mode will be removed in 1.6.0
     ce_mode: typing.Optional[str]

mlrun/{api → common}/schemas/function.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,10 +16,10 @@ import typing
 
 import pydantic
 
-import mlrun.api.utils.helpers
+import mlrun.common.types
 
 
-# Ideally we would want this to be class FunctionState(mlrun.api.utils.helpers.StrEnum) which is the
+# Ideally we would want this to be class FunctionState(mlrun.common.types.StrEnum) which is the
 # "FastAPI-compatible" way of creating schemas
 # But, when we save a function to the DB, we pickle the body, which saves the state as an instance of this class (and
 # not just a string), then if for some reason we downgrade to 0.6.4, before we had this class, we fail reading (pickle
@@ -46,7 +46,7 @@ class FunctionState:
     build = "build"
 
 
-class PreemptionModes(mlrun.api.utils.helpers.StrEnum):
+class PreemptionModes(mlrun.common.types.StrEnum):
     # makes function pods be able to run on preemptible nodes
     allow = "allow"
     # makes the function pods run on preemptible nodes only
@@ -59,7 +59,7 @@ class PreemptionModes(mlrun.api.utils.helpers.StrEnum):
 
 # used when running in Iguazio (otherwise use disabled mode)
 # populates mlrun.mlconf.function.spec.security_context.enrichment_mode
-class SecurityContextEnrichmentModes(mlrun.api.utils.helpers.StrEnum):
+class SecurityContextEnrichmentModes(mlrun.common.types.StrEnum):
     # always use the user id of the user that triggered the 1st run / created the function
     # NOTE: this mode is incomplete and not fully supported yet
     retain = "retain"

mlrun/{api → common}/schemas/http.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import mlrun.api.utils.helpers
+import mlrun.common.types
 
 
-class HTTPSessionRetryMode(mlrun.api.utils.helpers.StrEnum):
+class HTTPSessionRetryMode(mlrun.common.types.StrEnum):
     enabled = "enabled"
     disabled = "disabled"

mlrun/common/schemas/hub.py

@@ -0,0 +1,134 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Extra, Field
+
+import mlrun.common.types
+import mlrun.errors
+from mlrun.common.schemas.object import ObjectKind, ObjectSpec, ObjectStatus
+from mlrun.config import config
+
+
+# Defining a different base class (not ObjectMetadata), as there's no project, and it differs enough to
+# justify a new class
+class HubObjectMetadata(BaseModel):
+    name: str
+    description: str = ""
+    labels: Optional[dict] = {}
+    updated: Optional[datetime]
+    created: Optional[datetime]
+
+    class Config:
+        extra = Extra.allow
+
+
+# Currently only functions are supported. Will add more in the future.
+class HubSourceType(mlrun.common.types.StrEnum):
+    functions = "functions"
+
+
+# Sources-related objects
+class HubSourceSpec(ObjectSpec):
+    path: str  # URL to base directory, should include schema (s3://, etc...)
+    channel: str
+    credentials: Optional[dict] = {}
+    object_type: HubSourceType = Field(HubSourceType.functions, const=True)
+
+
+class HubSource(BaseModel):
+    kind: ObjectKind = Field(ObjectKind.hub_source, const=True)
+    metadata: HubObjectMetadata
+    spec: HubSourceSpec
+    status: Optional[ObjectStatus] = ObjectStatus(state="created")
+
+    def get_full_uri(self, relative_path):
+        return "{base}/{object_type}/{channel}/{relative_path}".format(
+            base=self.spec.path,
+            object_type=self.spec.object_type,
+            channel=self.spec.channel,
+            relative_path=relative_path,
+        )
+
+    def get_catalog_uri(self):
+        return self.get_full_uri(config.hub.catalog_filename)
+
+    @classmethod
+    def generate_default_source(cls):
+        if not config.hub.default_source.create:
+            return None
+
+        now = datetime.now(timezone.utc)
+        hub_metadata = HubObjectMetadata(
+            name=config.hub.default_source.name,
+            description=config.hub.default_source.description,
+            created=now,
+            updated=now,
+        )
+        return cls(
+            metadata=hub_metadata,
+            spec=HubSourceSpec(
+                path=config.hub.default_source.url,
+                channel=config.hub.default_source.channel,
+                object_type=HubSourceType(config.hub.default_source.object_type),
+            ),
+            status=ObjectStatus(state="created"),
+        )
+
+
+last_source_index = -1
+
+
+class IndexedHubSource(BaseModel):
+    index: int = last_source_index  # Default last. Otherwise, must be > 0
+    source: HubSource
+
+
+# Item-related objects
+class HubItemMetadata(HubObjectMetadata):
+    source: HubSourceType = Field(HubSourceType.functions, const=True)
+    version: str
+    tag: Optional[str]
+
+    def get_relative_path(self) -> str:
+        if self.source == HubSourceType.functions:
+            # This is needed since the hub deployment script modifies the paths to use _ instead of -.
+            modified_name = self.name.replace("-", "_")
+            # Prefer using the tag if exists. Otherwise, use version.
+            version = self.tag or self.version
+            return f"{modified_name}/{version}/"
+        else:
+            raise mlrun.errors.MLRunInvalidArgumentError(
+                f"Bad source for hub item - {self.source}"
+            )
+
+
+class HubItemSpec(ObjectSpec):
+    item_uri: str
+    assets: Dict[str, str] = {}
+
+
+class HubItem(BaseModel):
+    kind: ObjectKind = Field(ObjectKind.hub_item, const=True)
+    metadata: HubItemMetadata
+    spec: HubItemSpec
+    status: ObjectStatus
+
+
+class HubCatalog(BaseModel):
+    kind: ObjectKind = Field(ObjectKind.hub_catalog, const=True)
+    channel: str
+    catalog: List[HubItem]

mlrun/{api → common}/schemas/k8s.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ import typing
 
 import pydantic
 
-import mlrun.api.utils.helpers
+import mlrun.common.types
 
 
 class ResourceSpec(pydantic.BaseModel):
@@ -30,7 +30,7 @@ class Resources(pydantic.BaseModel):
     limits: ResourceSpec = ResourceSpec()
 
 
-class NodeSelectorOperator(mlrun.api.utils.helpers.StrEnum):
+class NodeSelectorOperator(mlrun.common.types.StrEnum):
     """
     A node selector operator is the set of operators that can be used in a node selector requirement
     https://github.com/kubernetes/api/blob/b754a94214be15ffc8d648f9fe6481857f1fc2fe/core/v1/types.go#L2765

mlrun/{api → common}/schemas/memory_reports.py

@@ -1,4 +1,4 @@
-# Copyright 2018 Iguazio
+# Copyright 2023 Iguazio
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.