dtlpy 1.116.6__py3-none-any.whl → 1.118.12__py3-none-any.whl

dtlpy/__init__.py

@@ -108,7 +108,7 @@ from .entities import (
     # compute
     ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources,
     NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute,
-    ServiceDriver, ExportType, OutputExportType
+    ServiceDriver, ExportType, OutputExportType, DynamicConcurrencyUpdateMethod
 )
 from .ml import BaseModelAdapter
 from .utilities import Converter, BaseServiceRunner, Progress, Context, AnnotationFormat

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.116.6'
+version = '1.118.12'

dtlpy/entities/__init__.py

@@ -44,7 +44,7 @@ from .package_slot import PackageSlot, SlotPostAction, SlotPostActionType, SlotD
 from .package_function import PackageFunction, FunctionIO, PackageInputType
 from .time_series import TimeSeries
 from .service import Service, KubernetesAutoscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRPSAutoscaler, \
-    InstanceCatalog, KubernetesRuntime, ServiceType, ServiceModeType
+    InstanceCatalog, KubernetesRuntime, ServiceType, ServiceModeType, DynamicConcurrencyUpdateMethod
 from .execution import Execution, ExecutionStatus
 from .command import Command, CommandsStatus
 from .assignment import Assignment, Workload, WorkloadUnit

dtlpy/entities/annotation.py

@@ -1827,7 +1827,7 @@ class FrameAnnotation(entities.BaseEntity):
         return frame
 
     @classmethod
-    def from_snapshot(cls, annotation, _json, fps):
+    def from_snapshot(cls, annotation, _json, fps=None):
         """
         new frame state to annotation
 

dtlpy/entities/app.py

@@ -93,7 +93,7 @@ class App(entities.BaseEntity):
         .. code-block:: python
             succeed = app.uninstall()
         """
-        return self.apps.uninstall(self.id)
+        return self.apps.uninstall(app=self)
 
     def update(self):
         """

dtlpy/entities/compute.py

@@ -13,6 +13,7 @@ class ClusterProvider(str, Enum):
     LOCAL = 'local'
     RANCHER_K3S = 'rancher-k3s'
     RANCHER_RKE = 'rancher-rke'
+    OPENSHIFT = 'openshift'
 
 
 class ComputeType(str, Enum):

dtlpy/entities/dataset.py

@@ -86,6 +86,9 @@ class Dataset(entities.BaseEntity):
     # api
     _client_api = attr.ib(type=ApiClient, repr=False)
 
+    # syncing status
+    is_syncing = attr.ib(default=False, repr=False)
+
     # entities
     _project = attr.ib(default=None, repr=False)
 
@@ -183,6 +186,7 @@ class Dataset(entities.BaseEntity):
                    expiration_options=expiration_options,
                    index_driver=_json.get('indexDriver', None),
                    enable_sync_with_cloned=_json.get('enableSyncWithCloned', None),
+                   is_syncing=_json.get('isSyncing', False),
                    src_dataset=_json.get('srcDataset', None))
         inst.is_fetched = is_fetched
         return inst
@@ -215,6 +219,7 @@ class Dataset(entities.BaseEntity):
                                                               attr.fields(Dataset).items_count,
                                                               attr.fields(Dataset).index_driver,
                                                               attr.fields(Dataset).enable_sync_with_cloned,
+                                                              attr.fields(Dataset).is_syncing,
                                                               attr.fields(Dataset).src_dataset,
                                                               ))
         _json.update({'items': self.items_url})
@@ -231,6 +236,7 @@ class Dataset(entities.BaseEntity):
             _json['expirationOptions'] = self.expiration_options.to_json()
         if self.enable_sync_with_cloned is not None:
             _json['enableSyncWithCloned'] = self.enable_sync_with_cloned
+        _json['isSyncing'] = self.is_syncing
         if self.src_dataset is not None:
             _json['srcDataset'] = self.src_dataset
         return _json
@@ -288,12 +294,15 @@ class Dataset(entities.BaseEntity):
     def set_repositories(self):
         reps = namedtuple('repositories',
                           field_names=['items', 'recipes', 'datasets', 'assignments', 'tasks', 'annotations',
-                                       'ontologies', 'features', 'settings', 'schema', 'collections'])
+                                       'ontologies', 'features', 'feature_sets', 'settings', 'schema', 'collections'])
+        _project_id = None
         if self._project is None:
             datasets = repositories.Datasets(client_api=self._client_api, project=self._project)
+            if self.projects is not None and len(self.projects) > 0:
+                _project_id = self.projects[0]
         else:
             datasets = self._project.datasets
-
+            _project_id = self._project.id
         return reps(
             items=repositories.Items(client_api=self._client_api, dataset=self, datasets=datasets),
             recipes=repositories.Recipes(client_api=self._client_api, dataset=self),
@@ -303,6 +312,7 @@ class Dataset(entities.BaseEntity):
             datasets=datasets,
             ontologies=repositories.Ontologies(client_api=self._client_api, dataset=self),
             features=repositories.Features(client_api=self._client_api, project=self._project, dataset=self),
+            feature_sets=repositories.FeatureSets(client_api=self._client_api, project=self._project, project_id=_project_id, dataset=self),
             settings=repositories.Settings(client_api=self._client_api, dataset=self),
             schema=repositories.Schema(client_api=self._client_api, dataset=self),
             collections=repositories.Collections(client_api=self._client_api, dataset=self)
@@ -353,6 +363,11 @@ class Dataset(entities.BaseEntity):
         assert isinstance(self._repositories.features, repositories.Features)
         return self._repositories.features
 
+    @property
+    def feature_sets(self):
+        assert isinstance(self._repositories.feature_sets, repositories.FeatureSets)
+        return self._repositories.feature_sets
+
     @property
     def collections(self):
         assert isinstance(self._repositories.collections, repositories.Collections)

dtlpy/entities/feature_set.py

@@ -135,6 +135,13 @@ class FeatureSet(entities.BaseEntity):
 
         return _json
 
+    def update(self):
+        """
+        Update feature set
+
+        :return: entities.FeatureSet
+        """
+        return self.feature_sets.update(self)
     def delete(self):
         """
         Delete the feature set

dtlpy/entities/item.py

@@ -80,6 +80,22 @@ class Item(entities.BaseEntity):
     def datasetId(self):
         return self.dataset_id
 
+    @property
+    def resolved_stream(self):
+        stream = self.metadata.get('system', dict()).get('shebang', dict()).get('linkInfo', dict()).get('ref', None)
+        if stream is None:
+            stream = self.stream
+            api_url = self._client_api.environment
+            if api_url != self._client_api.base_gate_url:
+                stream = stream.replace(api_url, self._client_api.base_gate_url)
+        else:
+            link_item_url_override = os.environ.get('LINK_ITEM_URL_OVERRIDE', None)
+            if link_item_url_override is not None:
+                src, target = link_item_url_override.split(',')
+                stream = stream.replace(src, target)
+
+        return stream
+
     @staticmethod
     def _protected_from_json(_json, client_api, dataset=None):
         """

dtlpy/entities/model.py

@@ -423,7 +423,7 @@ class Model(entities.BaseEntity):
         # default
         if 'id_to_label_map' not in self.configuration:
             if not (self.dataset_id == 'null' or self.dataset_id is None):
-                self.labels = [label.tag for label in self.dataset.labels]
+                self.labels = [flat_key for flat_key, _ in self.dataset.labels_flat_dict.items()]
             self.configuration['id_to_label_map'] = {int(idx): lbl for idx, lbl in enumerate(self.labels)}
         # use existing
         else:

dtlpy/entities/ontology.py

@@ -290,7 +290,7 @@ class Ontology(entities.BaseEntity):
 
     def update(self, system_metadata=False):
         """
-        Update items metadata
+        Update Ontology attribute
 
         :param bool system_metadata: bool - True, if you want to change metadata system
         :return: Ontology object

dtlpy/entities/paged_entities.py

@@ -9,7 +9,7 @@ from typing import Optional, List, Any
 import attr
 
 from .filters import FiltersOperations, FiltersOrderByDirection, FiltersResource
-from .. import miscellaneous
+from .. import miscellaneous, exceptions
 from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
@@ -243,8 +243,12 @@ class PagedEntities:
         :param page_offset: page offset (for offset-based)
         :param page_size: page size
         """
-        items = self.return_page(page_offset=page_offset, page_size=page_size)
-        self.items = items
+        try:
+            items = self.return_page(page_offset=page_offset, page_size=page_size)
+            self.items = items
+        except exceptions.BadRequest as e:
+            logger.warning(f"BadRequest error received: {str(e)}")
+            self.items = miscellaneous.List(list())
 
     def next_page(self) -> None:
         """

dtlpy/entities/service.py

@@ -11,6 +11,12 @@ from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
+class DynamicConcurrencyUpdateMethod(str, Enum):
+    """ The method of updating the dynamic concurrency.
+    """
+    RESTART = 'restart',
+    SYNC = 'sync'
+
 
 class ServiceType(str, Enum):
     """ The type of the service (SYSTEM).
@@ -136,6 +142,7 @@ class KubernetesRuntime(ServiceRuntime):
                  num_replicas=DEFAULT_NUM_REPLICAS,
                  concurrency=DEFAULT_CONCURRENCY,
                  dynamic_concurrency=None,
+                 dynamic_concurrency_config=None,
                  runner_image=None,
                  autoscaler=None,
                  **kwargs):
@@ -149,6 +156,7 @@ class KubernetesRuntime(ServiceRuntime):
         self.single_agent = kwargs.get('singleAgent', None)
         self.preemptible = kwargs.get('preemptible', None)
         self.dynamic_concurrency = kwargs.get('dynamicConcurrency', dynamic_concurrency)
+        self.dynamic_concurrency_config = kwargs.get('dynamicConcurrencyConfig', dynamic_concurrency_config)
 
         self.autoscaler = kwargs.get('autoscaler', autoscaler)
         if self.autoscaler is not None and isinstance(self.autoscaler, dict):
@@ -183,6 +191,9 @@ class KubernetesRuntime(ServiceRuntime):
         if self.dynamic_concurrency is not None:
             _json['dynamicConcurrency'] = self.dynamic_concurrency
 
+        if self.dynamic_concurrency_config is not None:
+            _json['dynamicConcurrencyConfig'] = self.dynamic_concurrency_config
+
         return _json
 
 

dtlpy/ml/base_model_adapter.py

@@ -368,14 +368,41 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                 annotation_filters.custom_filter['filter']['$and'].append({'metadata.system.model.name': {'$exists': False}})
         return annotation_filters
 
+    def __download_items(self, dataset, filters, local_path, annotation_options, annotation_filters=None):
+        """
+        Download items from dataset with optional annotation filters.
+
+        :param dataset: Dataset to download from
+        :param filters: Filters to apply
+        :param local_path: Local path to save files
+        :param annotation_options: Annotation download options
+        :param annotation_filters: Optional filters for annotations
+        :return: List of downloaded items
+        """
+        if annotation_options == entities.ViewAnnotationOptions.JSON:
+            downloader = repositories.Downloader(dataset.items)
+            return downloader._download_recursive(
+                local_path=local_path,
+                filters=filters,
+                annotation_filters=annotation_filters
+            )
+        else:
+            return dataset.items.download(
+                filters=filters,
+                local_path=local_path,
+                annotation_options=annotation_options,
+                annotation_filters=annotation_filters
+            )
+
     def __download_background_images(self, filters, data_subset_base_path, annotation_options):
         background_list = list()
         if self.configuration.get('include_background', False) is True:
             filters.custom_filter["filter"]["$and"].append({"annotated": False})
-            background_list = self.model_entity.dataset.items.download(
+            background_list = self.__download_items(
+                dataset=self.model_entity.dataset,
                 filters=filters,
                 local_path=data_subset_base_path,
-                annotation_options=annotation_options,
+                annotation_options=annotation_options
             )
         return background_list
 
@@ -434,49 +461,53 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
         if subsets is None:
             raise ValueError("Model (id: {}) must have subsets in metadata.system.subsets".format(self.model_entity.id))
         for subset, filters_dict in subsets.items():
+            _filters_dict = filters_dict.copy()
             data_subset_base_path = os.path.join(data_path, subset)
             if os.path.isdir(data_subset_base_path) and not overwrite:
                 # existing and dont overwrite
                 self.logger.debug("Subset {!r} already exists (and overwrite=False). Skipping.".format(subset))
                 continue
 
-            filters = entities.Filters(custom_filter=filters_dict)
+            filters = entities.Filters(custom_filter=_filters_dict)
             self.logger.debug("Downloading subset {!r} of {}".format(subset, self.model_entity.dataset.name))
 
             annotation_filters = None
-            if subset in annotations_subsets:
-                annotation_filters = entities.Filters(
-                    use_defaults=False,
-                    resource=entities.FiltersResource.ANNOTATION,
-                    custom_filter=annotations_subsets[subset],
-                )
-            # if user provided annotation_filters, skip the default filters
-            elif self.model_entity.output_type is not None and self.model_entity.output_type != "embedding":
-                annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION, use_defaults=False)
-                if self.model_entity.output_type in [
-                    entities.AnnotationType.SEGMENTATION,
-                    entities.AnnotationType.POLYGON,
-                ]:
-                    model_output_types = [entities.AnnotationType.SEGMENTATION, entities.AnnotationType.POLYGON]
-                else:
-                    model_output_types = [self.model_entity.output_type]
-
-                annotation_filters.add(
-                    field=entities.FiltersKnownFields.TYPE,
-                    values=model_output_types,
-                    operator=entities.FiltersOperations.IN,
-                )
+            if self.model_entity.output_type != "embedding":
+                if subset in annotations_subsets:
+                    annotation_filters = entities.Filters(
+                        use_defaults=False,
+                        resource=entities.FiltersResource.ANNOTATION,
+                        custom_filter=annotations_subsets[subset],
+                    )
+                # if user provided annotation_filters, skip the default filters
+                elif self.model_entity.output_type is not None:
+                    annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION, use_defaults=False)
+                    if self.model_entity.output_type in [
+                        entities.AnnotationType.SEGMENTATION,
+                        entities.AnnotationType.POLYGON,
+                    ]:
+                        model_output_types = [entities.AnnotationType.SEGMENTATION, entities.AnnotationType.POLYGON]
+                    else:
+                        model_output_types = [self.model_entity.output_type]
+
+                    annotation_filters.add(
+                        field=entities.FiltersKnownFields.TYPE,
+                        values=model_output_types,
+                        operator=entities.FiltersOperations.IN,
+                    )
 
-            annotation_filters = self.__include_model_annotations(annotation_filters)
-            annotations_subsets[subset] = annotation_filters.prepare()
+                annotation_filters = self.__include_model_annotations(annotation_filters)
+                annotations_subsets[subset] = annotation_filters.prepare()
 
-            ret_list = dataset.items.download(
+            ret_list = self.__download_items(
+                dataset=dataset,
                 filters=filters,
                 local_path=data_subset_base_path,
                 annotation_options=annotation_options,
-                annotation_filters=annotation_filters,
+                annotation_filters=annotation_filters
             )
-            filters = entities.Filters(custom_filter=subsets[subset])
+            _filters_dict = subsets[subset].copy()
+            filters = entities.Filters(custom_filter=_filters_dict)
             background_ret_list = self.__download_background_images(
                 filters=filters,
                 data_subset_base_path=data_subset_base_path,
@@ -679,7 +710,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             valid_vectors = []
             items_to_upload = []
             vectors_to_upload = []
-            
+
             for item, vector in zip(_items, vectors):
                 # Check if vector is valid
                 if vector is None or len(vector) != embeddings_size:
@@ -689,25 +720,25 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                 # Item and vector are valid
                 valid_items.append(item)
                 valid_vectors.append(vector)
-                
+
                 # Check if item should be skipped (prompt items)
                 _system_metadata = getattr(item, 'system', dict())
                 is_prompt = _system_metadata.get('shebang', dict()).get('dltype', '') == 'prompt'
                 if skip_default_items and is_prompt:
                     self.logger.debug(f"Skipping feature upload for prompt item {item.id}")
                     continue
-                
+
                 # Items were not skipped - should be uploaded
                 items_to_upload.append(item)
                 vectors_to_upload.append(vector)
-            
+
             # Update the original lists with valid items only
             _items[:] = valid_items
             vectors[:] = valid_vectors
-            
+
             if len(_items) != len(vectors):
                 raise ValueError(f"The number of items ({len(_items)}) is not equal to the number of vectors ({len(vectors)}).")
-            
+
             self.logger.debug(f"Uploading {len(items_to_upload)} items' feature vectors for model {self.model_entity.name}.")
             try:
                 start_time = time.time()
@@ -800,7 +831,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             logger.info("Received {s} for training".format(s=model.id))
             model = model.wait_for_model_ready()
             if model.status == 'failed':
-                raise ValueError("Model is in failed state, cannot train.")
+                logger.warning("Model failed. New training will attempt to resume from previous checkpoints.")
 
             ##############
             # Set status #

dtlpy/repositories/apps.py

@@ -272,15 +272,16 @@ class Apps:
 
         return app
 
-    def uninstall(self, app_id: str = None, app_name: str = None, wait: bool = True) -> bool:
+    def uninstall(self, app_id: str = None, app_name: str = None, wait: bool = True, app: entities.App = None) -> bool:
         """
         Delete an app entity.
 
         Note: You are required to add either app_id or app_name.
 
         :param str app_id: optional - the id of the app.
-        :param str app_name: optional - the name of the app.
+        :param str app_name: [DEPRECATED] - the name of the app.
         :param bool wait: optional - wait for the operation to finish.
+        :param entities.App app: optional - the app entity.
         :return whether we succeed uninstalling the specified app.
         :rtype bool
 
@@ -288,12 +289,12 @@ class Apps:
         .. code-block:: python
             # succeed = dl.apps.delete(app_id='app_id')
         """
-        if app_id is None and app_name is None:
+        if app is None and app_id is None:
             raise exceptions.PlatformException(
                 error='400',
                 message='You must provide an identifier in inputs')
-        if app_name is not None:
-            app = self.__get_by_name(app_name)
+
+        if app is not None:
             app_id = app.id
 
         success, response = self._client_api.gen_request(req_type='delete', path='/apps/{}'.format(app_id))
@@ -302,18 +303,16 @@ class Apps:
 
         try:
             app = self.get(app_id=app_id)
-        except Exception as e:
-            if e.status_code == '404':
-                return success
-            else:
-                raise e
-        if app.metadata:
+        except exceptions.NotFound:
+            return success
+
+        if wait and app.status == entities.CompositionStatus.TERMINATING and app.metadata is not None:
             command_id = app.metadata.get('system', {}).get('commands', {}).get('uninstall', None)
-            if wait and app.status == entities.CompositionStatus.TERMINATING and command_id is not None:
+            if command_id is not None:
                 command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
                 command.wait()
 
-        logger.debug(f"App deleted successfully (id: {app_id}, name: {app_name}")
+        logger.debug(f"App deleted successfully (id: {app.id}, name: {app.name}")
         return success
 
     def resume(self, app: entities.App = None, app_id: str = None) -> bool: