dtlpy 1.92.19__py3-none-any.whl → 1.93.11__py3-none-any.whl

dtlpy/__init__.py

@@ -97,14 +97,18 @@ from .entities import (
     ItemLink, UrlLink, LinkTypeEnum,
     Modality, ModalityTypeEnum, ModalityRefTypeEnum,
     Workload, WorkloadUnit, ItemAction,
-    PipelineExecution, CycleRerunMethod, PipelineExecutionNode, Pipeline, PipelineConnection,
+    PipelineExecution, PipelineExecutionStatus, CycleRerunMethod, PipelineExecutionNode, Pipeline, PipelineConnection,
     PipelineNode, TaskNode, CodeNode, PipelineStats, PipelineSettings,
     PipelineNodeType, PipelineNameSpace, PipelineResumeOption, Variable, CompositionStatus,
     FunctionNode, DatasetNode, PipelineConnectionPort, PipelineNodeIO, Organization, OrganizationsPlans, Integration,
     Driver, S3Driver, GcsDriver, AzureBlobDriver, CacheAction, PodType,
     ExternalStorage, IntegrationType, Role, PlatformEntityType, SettingsValueTypes, SettingsTypes, SettingsSectionNames,
     SettingScope, BaseSetting, UserSetting, Setting, ServiceSample, ExecutionSample, PipelineExecutionSample,
-    ResourceExecution, Message, NotificationEventContext
+    ResourceExecution, Message, NotificationEventContext,
+    # compute
+    ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources,
+    NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute,
+    ServiceDriver
 )
 from .ml import BaseModelAdapter
 from .utilities import Converter, BaseServiceRunner, Progress, Context, AnnotationFormat
@@ -163,6 +167,8 @@ apps = repositories.Apps(client_api=client_api)
 dpks = repositories.Dpks(client_api=client_api)
 messages = repositories.Messages(client_api=client_api)
 compositions = repositories.Compositions(client_api=client_api)
+computes = repositories.Computes(client_api=client_api)
+service_drivers = repositories.ServiceDrivers(client_api=client_api)
 
 try:
     check_sdk.check(version=__version__, client_api=client_api)

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.92.19'
+version = '1.93.11'

dtlpy/entities/__init__.py

@@ -58,7 +58,7 @@ from .driver import Driver, S3Driver, GcsDriver, AzureBlobDriver
 from .pipeline import Pipeline, PipelineStats, PipelineResumeOption, PipelineSettings, Variable, CompositionStatus
 from .node import PipelineConnection, PipelineNode, PipelineConnectionPort, PipelineNodeIO, TaskNode, \
     CodeNode, FunctionNode, PipelineNodeType, PipelineNameSpace, DatasetNode
-from .pipeline_execution import PipelineExecution, PipelineExecutionNode, CycleRerunMethod
+from .pipeline_execution import PipelineExecution, PipelineExecutionNode, PipelineExecutionStatus, CycleRerunMethod
 from .feature import Feature
 from .feature_set import FeatureSet, FeatureEntityType
 from .organization import Organization, OrganizationsPlans, MemberOrgRole, CacheAction, PodType
@@ -75,3 +75,6 @@ from .app_module import AppModule
 from .resource_execution import ResourceExecution
 from .message import Message, NotificationEventContext
 from .prompt_item import Prompt, PromptItem, PromptType
+from .compute import ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources, \
+    NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute, \
+    ServiceDriver

dtlpy/entities/app.py

@@ -53,6 +53,7 @@ class App(entities.BaseEntity):
     _project = attr.ib(type=entities.Project, repr=False)
     _client_api = attr.ib(type=ApiClient, repr=False)
     _repositories = attr.ib(repr=False)
+    integrations = attr.ib(type=list, default=None)
 
     @_repositories.default
     def set_repositories(self):
@@ -186,6 +187,8 @@ class App(entities.BaseEntity):
             _json['status'] = self.status
         if self.settings != {}:
             _json['settings'] = self.settings
+        if self.integrations is not None:
+            _json['integrations'] = self.integrations
 
         return _json
 
@@ -210,7 +213,8 @@ class App(entities.BaseEntity):
             project=project,
             metadata=_json.get('metadata', None),
             status=_json.get('status', None),
-            settings=_json.get('settings', {})
+            settings=_json.get('settings', {}),
+            integrations=_json.get('integrations', None)
         )
         app.is_fetched = is_fetched
         return app

dtlpy/entities/compute.py

@@ -0,0 +1,374 @@
+from enum import Enum
+from typing import List, Optional, Dict
+from ..services.api_client import ApiClient
+from .. import repositories
+
+
+class ClusterProvider(str, Enum):
+    GCP = 'gcp'
+    AWS = 'aws'
+    AZURE = 'azure'
+    HPC = 'hpc'
+
+
+class ComputeType(str, Enum):
+    KUBERNETES = "kubernetes"
+
+
+class ComputeStatus(str, Enum):
+    READY = "ready"
+    INITIALIZING = "initializing"
+    PAUSE = "pause"
+
+
+class Toleration:
+    def __init__(self, name: str):
+        self.name = name
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            name=_json.get('name')
+        )
+
+    def to_json(self):
+        return {
+            'name': self.name
+        }
+
+
+class DeploymentResource:
+    def __init__(self, gpu: int, cpu: int, memory: str):
+        self.gpu = gpu
+        self.cpu = cpu
+        self.memory = memory
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            gpu=_json.get('gpu', None),
+            cpu=_json.get('cpu', None),
+            memory=_json.get('memory', None)
+        )
+
+    def to_json(self):
+        _json = {}
+        if self.gpu is not None:
+            _json['gpu'] = self.gpu
+        if self.cpu is not None:
+            _json['cpu'] = self.cpu
+        if self.memory is not None:
+            _json['memory'] = self.memory
+
+
+class DeploymentResources:
+    def __init__(self, request: DeploymentResource, limit: DeploymentResource):
+        self.request = request
+        self.limit = limit
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            request=DeploymentResource.from_json(_json.get('request', dict())),
+            limit=DeploymentResource.from_json(_json.get('limit', dict()))
+        )
+
+    def to_json(self):
+        return {
+            'request': self.request.to_json(),
+            'limit': self.limit.to_json()
+        }
+
+
+class NodePool:
+    def __init__(
+            self,
+            name: str,
+            is_dl_type_default: bool,
+            dl_type: Optional[str] = None,
+            tolerations: Optional[List[Toleration]] = None,
+            description: str = "",
+            node_selector: str = "",
+            preemtible: bool = False,
+            deployment_resources_request: DeploymentResources = None
+    ):
+        self.name = name
+        self.is_dl_type_default = is_dl_type_default
+        self.dl_type = dl_type
+        self.tolerations = tolerations if tolerations is not None else []
+        self.description = description
+        self.node_selector = node_selector
+        self.preemtible = preemtible
+        self.deployment_resources_request = deployment_resources_request
+
+    @classmethod
+    def from_json(cls, _json):
+        node_pool = cls(
+            name=_json.get('name'),
+            is_dl_type_default=_json.get('isDlTypeDefault'),
+            dl_type=_json.get('dlType'),
+            description=_json.get('description'),
+            node_selector=_json.get('nodeSelector'),
+            preemtible=_json.get('preemtible'),
+            deployment_resources_request=DeploymentResources.from_json(_json.get('deploymentResourcesRequest', dict())),
+            tolerations=[Toleration.from_json(t) for t in _json.get('tolerations', list())]
+        )
+
+        return node_pool
+
+    def to_json(self):
+        _json = {
+            'name': self.name,
+            'isDlTypeDefault': self.is_dl_type_default,
+            'description': self.description,
+            'nodeSelector': self.node_selector,
+            'preemtible': self.preemtible,
+            'deploymentResourcesRequest': self.deployment_resources_request.to_json(),
+            'tolerations': [t.to_json() for t in self.tolerations]
+        }
+
+        if self.dl_type is not None:
+            _json['dlType'] = self.dl_type
+
+        return _json
+
+
+class AuthenticationIntegration:
+    def __init__(self, id: str, type: str):
+        self.id = id
+        self.type = type
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            id=_json.get('id'),
+            type=_json.get('type')
+        )
+
+    def to_json(self):
+        return {
+            'id': self.id,
+            'type': self.type
+        }
+
+
+class Authentication:
+    def __init__(self, integration: AuthenticationIntegration):
+        self.integration = integration
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            integration=AuthenticationIntegration.from_json(_json.get('integration', dict()))
+        )
+
+    def to_json(self):
+        return {
+            'integration': self.integration.to_json()
+        }
+
+
+class ComputeCluster:
+    def __init__(
+            self,
+            name: str,
+            endpoint: str,
+            kubernetes_version: str,
+            provider: ClusterProvider,
+            node_pools: Optional[List[NodePool]] = None,
+            metadata: Optional[Dict] = None,
+            authentication: Optional[Authentication] = None,
+    ):
+        self.name = name
+        self.endpoint = endpoint
+        self.kubernetes_version = kubernetes_version
+        self.provider = provider
+        self.node_pools = node_pools if node_pools is not None else []
+        self.metadata = metadata if metadata is not None else {}
+        self.authentication = authentication if authentication is not None else Authentication(
+            AuthenticationIntegration("", ""))
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            name=_json.get('name'),
+            endpoint=_json.get('endpoint'),
+            kubernetes_version=_json.get('kubernetesVersion'),
+            provider=ClusterProvider(_json.get('provider')),
+            node_pools=[NodePool.from_json(np) for np in _json.get('nodePools', list())],
+            metadata=_json.get('metadata'),
+            authentication=Authentication.from_json(_json.get('authentication', dict()))
+        )
+
+    def to_json(self):
+        return {
+            'name': self.name,
+            'endpoint': self.endpoint,
+            'kubernetesVersion': self.kubernetes_version,
+            'provider': self.provider.value,
+            'nodePools': [np.to_json() for np in self.node_pools],
+            'metadata': self.metadata,
+            'authentication': self.authentication.to_json()
+        }
+
+
+class ComputeContext:
+    def __init__(self, labels: List[str], org: str, project: Optional[str] = None):
+        self.labels = labels
+        self.org = org
+        self.project = project
+
+    @classmethod
+    def from_json(cls, _json):
+        return cls(
+            labels=_json.get('labels', list()),
+            org=_json.get('org'),
+            project=_json.get('project')
+        )
+
+    def to_json(self):
+        return {
+            'labels': self.labels,
+            'org': self.org,
+            'project': self.project
+        }
+
+
+class Compute:
+    def __init__(
+            self,
+            id: str,
+            context: ComputeContext,
+            client_api: ApiClient,
+            shared_contexts: Optional[List[ComputeContext]] = None,
+            global_: Optional[bool] = None,
+            status: ComputeStatus = ComputeStatus.INITIALIZING,
+            type: ComputeType = ComputeType.KUBERNETES,
+            features: Optional[Dict] = None,
+            metadata: Optional[Dict] = None,
+    ):
+        self.id = id
+        self.context = context
+        self.shared_contexts = shared_contexts if shared_contexts is not None else []
+        self.global_ = global_
+        self.status = status
+        self.type = type
+        self.features = features if features is not None else dict()
+        self.metadata = metadata if metadata is not None else dict()
+        self._client_api = client_api
+        self._computes = None
+        self._serviceDrivers = None
+
+    @property
+    def computes(self):
+        if self._computes is None:
+            self._computes = repositories.Computes(client_api=self._client_api)
+        return self._computes
+
+    @property
+    def service_drivers(self):
+        if self._serviceDrivers is None:
+            self._serviceDrivers = repositories.ServiceDrivers(client_api=self._client_api)
+        return self._serviceDrivers
+
+    def delete(self):
+        return self._computes.delete(compute_id=self.id)
+
+    def update(self):
+        return self._computes.update(compute=self)
+
+    @classmethod
+    def from_json(cls, _json, client_api: ApiClient):
+        return cls(
+            id=_json.get('id'),
+            context=ComputeContext.from_json(_json.get('context', dict())),
+            shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
+            global_=_json.get('global'),
+            status=ComputeStatus(_json.get('status')),
+            type=ComputeType(_json.get('type')),
+            features=_json.get('features'),
+            client_api=client_api,
+            metadata=_json.get('metadata')
+        )
+
+    def to_json(self):
+        return {
+            'id': self.id,
+            'context': self.context.to_json(),
+            'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
+            'global': self.global_,
+            'status': self.status.value,
+            'type': self.type.value,
+            'features': self.features,
+            'metadata': self.metadata
+        }
+
+
+class KubernetesCompute(Compute):
+    def __init__(
+            self,
+            id: str,
+            context: ComputeContext,
+            cluster: ComputeCluster,
+            shared_contexts: Optional[List[ComputeContext]] = None,
+            global_: Optional[bool] = None,
+            status: ComputeStatus = ComputeStatus.INITIALIZING,
+            type: ComputeType = ComputeType.KUBERNETES,
+            features: Optional[Dict] = None,
+            metadata: Optional[Dict] = None,
+            client_api: ApiClient = None
+    ):
+        super().__init__(id=id, context=context, shared_contexts=shared_contexts, global_=global_, status=status,
+                         type=type, features=features, metadata=metadata, client_api=client_api)
+        self.cluster = cluster
+
+    @classmethod
+    def from_json(cls, _json, client_api: ApiClient):
+        return cls(
+            id=_json.get('id'),
+            context=ComputeContext.from_json(_json.get('context', dict())),
+            cluster=ComputeCluster.from_json(_json.get('cluster', dict())),
+            shared_contexts=[ComputeContext.from_json(sc) for sc in _json.get('sharedContexts', list())],
+            global_=_json.get('global'),
+            status=ComputeStatus(_json.get('status')),
+            type=ComputeType(_json.get('type')),
+            features=_json.get('features'),
+            metadata=_json.get('metadata'),
+            client_api=client_api
+        )
+
+    def to_json(self):
+        return {
+            'id': self.id,
+            'context': self.context.to_json(),
+            'cluster': self.cluster.to_json(),
+            'sharedContexts': [sc.to_json() for sc in self.shared_contexts],
+            'global': self.global_,
+            'status': self.status.value,
+            'type': self.type.value,
+            'features': self.features
+        }
+
+
+class ServiceDriver:
+    def __init__(self, name: str, context: ComputeContext, compute_id: str, client_api: ApiClient):
+        self.name = name
+        self.context = context
+        self.compute_id = compute_id
+        self.client_api = client_api
+
+    @classmethod
+    def from_json(cls, _json, client_api: ApiClient):
+        return cls(
+            name=_json.get('name'),
+            context=ComputeContext.from_json(_json.get('context', dict())),
+            compute_id=_json.get('computeId'),
+            client_api=client_api
+        )
+
+    def to_json(self):
+        return {
+            'name': self.name,
+            'context': self.context.to_json(),
+            'computeId': self.compute_id
+        }

dtlpy/entities/dpk.py

@@ -164,6 +164,7 @@ class Components(entities.DlEntity):
     compute_configs: List[DpkComputeConfig] = entities.DlProperty(location=['computeConfigs'], _kls='DpkComputeConfig')
     channels: List[DpkComponentChannel] = entities.DlProperty(location=['channels'], _kls='DpkComponentChannel')
     pipeline_templates: List[dict] = entities.DlProperty(location=['pipelineTemplates'])
+    integrations: List[dict] = entities.DlProperty(location=['integrations'])
 
     @panels.default
     def default_panels(self):

dtlpy/entities/filters.py

@@ -59,6 +59,7 @@ class FiltersOperations(str, Enum):
     LESS_THAN = "lt"
     EXISTS = "exists"
     MATCH = "match"
+    NIN = 'nin'
 
 
 class FiltersMethod(str, Enum):

dtlpy/entities/item.py

@@ -189,12 +189,11 @@ class Item(entities.BaseEntity):
         binary.write(json.dumps(_json).encode())
         binary.seek(0)
         binary.name = self.name
-        resp = requests.post(url=client_api.environment + f'/items/{self.id}/revisions',
-                             headers=client_api.auth,
-                             files={'file': (binary.name, binary)}
-                             )
-        if not resp.ok:
-            raise ValueError(resp.text)
+        success, resp = client_api.gen_request(req_type='post',
+                                      path=f'/items/{self.id}/revisions',
+                                      files={'file': (binary.name, binary)})
+        if not success:
+            raise exceptions.PlatformException(resp)
 
     @property
     def project(self):

dtlpy/entities/model.py

@@ -158,13 +158,13 @@ class Model(entities.BaseEntity):
         """
         if project is not None:
             if project.id != _json.get('context', {}).get('project', None):
-                logger.warning('Model has been fetched from a project that is not in it projects list')
+                logger.warning("Model's project is different then the input project")
                 project = None
 
         if package is not None:
             if package.id != _json.get('packageId', None):
-                logger.warning('Model has been fetched from a model that is not in it projects list')
-                model = None
+                logger.warning("Model's package is different then the input package")
+                package = None
 
         model_artifacts = [entities.Artifact.from_json(_json=artifact,
                                                        client_api=client_api,