dtlpy 1.117.6__py3-none-any.whl → 1.118.12__py3-none-any.whl

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.117.6'
+version = '1.118.12'

dtlpy/entities/annotation.py

@@ -1827,7 +1827,7 @@ class FrameAnnotation(entities.BaseEntity):
         return frame
 
     @classmethod
-    def from_snapshot(cls, annotation, _json, fps):
+    def from_snapshot(cls, annotation, _json, fps=None):
         """
         new frame state to annotation
 

dtlpy/entities/app.py

@@ -93,7 +93,7 @@ class App(entities.BaseEntity):
         .. code-block:: python
             succeed = app.uninstall()
         """
-        return self.apps.uninstall(self.id)
+        return self.apps.uninstall(app=self)
 
     def update(self):
         """

dtlpy/entities/compute.py

@@ -13,6 +13,7 @@ class ClusterProvider(str, Enum):
     LOCAL = 'local'
     RANCHER_K3S = 'rancher-k3s'
     RANCHER_RKE = 'rancher-rke'
+    OPENSHIFT = 'openshift'
 
 
 class ComputeType(str, Enum):

dtlpy/entities/dataset.py

@@ -86,6 +86,9 @@ class Dataset(entities.BaseEntity):
     # api
     _client_api = attr.ib(type=ApiClient, repr=False)
 
+    # syncing status
+    is_syncing = attr.ib(default=False, repr=False)
+
     # entities
     _project = attr.ib(default=None, repr=False)
 
@@ -183,6 +186,7 @@ class Dataset(entities.BaseEntity):
                    expiration_options=expiration_options,
                    index_driver=_json.get('indexDriver', None),
                    enable_sync_with_cloned=_json.get('enableSyncWithCloned', None),
+                   is_syncing=_json.get('isSyncing', False),
                    src_dataset=_json.get('srcDataset', None))
         inst.is_fetched = is_fetched
         return inst
@@ -215,6 +219,7 @@ class Dataset(entities.BaseEntity):
                                                               attr.fields(Dataset).items_count,
                                                               attr.fields(Dataset).index_driver,
                                                               attr.fields(Dataset).enable_sync_with_cloned,
+                                                              attr.fields(Dataset).is_syncing,
                                                               attr.fields(Dataset).src_dataset,
                                                               ))
         _json.update({'items': self.items_url})
@@ -231,6 +236,7 @@ class Dataset(entities.BaseEntity):
             _json['expirationOptions'] = self.expiration_options.to_json()
         if self.enable_sync_with_cloned is not None:
             _json['enableSyncWithCloned'] = self.enable_sync_with_cloned
+        _json['isSyncing'] = self.is_syncing
         if self.src_dataset is not None:
             _json['srcDataset'] = self.src_dataset
         return _json
@@ -288,12 +294,15 @@ class Dataset(entities.BaseEntity):
     def set_repositories(self):
         reps = namedtuple('repositories',
                           field_names=['items', 'recipes', 'datasets', 'assignments', 'tasks', 'annotations',
-                                       'ontologies', 'features', 'settings', 'schema', 'collections'])
+                                       'ontologies', 'features', 'feature_sets', 'settings', 'schema', 'collections'])
+        _project_id = None
         if self._project is None:
             datasets = repositories.Datasets(client_api=self._client_api, project=self._project)
+            if self.projects is not None and len(self.projects) > 0:
+                _project_id = self.projects[0]
         else:
             datasets = self._project.datasets
-
+            _project_id = self._project.id
         return reps(
             items=repositories.Items(client_api=self._client_api, dataset=self, datasets=datasets),
             recipes=repositories.Recipes(client_api=self._client_api, dataset=self),
@@ -303,6 +312,7 @@ class Dataset(entities.BaseEntity):
             datasets=datasets,
             ontologies=repositories.Ontologies(client_api=self._client_api, dataset=self),
             features=repositories.Features(client_api=self._client_api, project=self._project, dataset=self),
+            feature_sets=repositories.FeatureSets(client_api=self._client_api, project=self._project, project_id=_project_id, dataset=self),
             settings=repositories.Settings(client_api=self._client_api, dataset=self),
             schema=repositories.Schema(client_api=self._client_api, dataset=self),
             collections=repositories.Collections(client_api=self._client_api, dataset=self)
@@ -353,6 +363,11 @@ class Dataset(entities.BaseEntity):
         assert isinstance(self._repositories.features, repositories.Features)
         return self._repositories.features
 
+    @property
+    def feature_sets(self):
+        assert isinstance(self._repositories.feature_sets, repositories.FeatureSets)
+        return self._repositories.feature_sets
+
     @property
     def collections(self):
         assert isinstance(self._repositories.collections, repositories.Collections)

dtlpy/entities/model.py

@@ -423,7 +423,7 @@ class Model(entities.BaseEntity):
         # default
         if 'id_to_label_map' not in self.configuration:
             if not (self.dataset_id == 'null' or self.dataset_id is None):
-                self.labels = [label.tag for label in self.dataset.labels]
+                self.labels = [flat_key for flat_key, _ in self.dataset.labels_flat_dict.items()]
             self.configuration['id_to_label_map'] = {int(idx): lbl for idx, lbl in enumerate(self.labels)}
         # use existing
         else:

dtlpy/entities/paged_entities.py

@@ -9,7 +9,7 @@ from typing import Optional, List, Any
 import attr
 
 from .filters import FiltersOperations, FiltersOrderByDirection, FiltersResource
-from .. import miscellaneous
+from .. import miscellaneous, exceptions
 from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
@@ -243,8 +243,12 @@ class PagedEntities:
         :param page_offset: page offset (for offset-based)
         :param page_size: page size
         """
-        items = self.return_page(page_offset=page_offset, page_size=page_size)
-        self.items = items
+        try:
+            items = self.return_page(page_offset=page_offset, page_size=page_size)
+            self.items = items
+        except exceptions.BadRequest as e:
+            logger.warning(f"BadRequest error received: {str(e)}")
+            self.items = miscellaneous.List(list())
 
     def next_page(self) -> None:
         """

dtlpy/entities/service.py

@@ -142,7 +142,7 @@ class KubernetesRuntime(ServiceRuntime):
                  num_replicas=DEFAULT_NUM_REPLICAS,
                  concurrency=DEFAULT_CONCURRENCY,
                  dynamic_concurrency=None,
-                 concurrency_update_method=None,
+                 dynamic_concurrency_config=None,
                  runner_image=None,
                  autoscaler=None,
                  **kwargs):
@@ -156,7 +156,7 @@ class KubernetesRuntime(ServiceRuntime):
         self.single_agent = kwargs.get('singleAgent', None)
         self.preemptible = kwargs.get('preemptible', None)
         self.dynamic_concurrency = kwargs.get('dynamicConcurrency', dynamic_concurrency)
-        self.concurrency_update_method = kwargs.get('concurrencyUpdateMethod', concurrency_update_method)
+        self.dynamic_concurrency_config = kwargs.get('dynamicConcurrencyConfig', dynamic_concurrency_config)
 
         self.autoscaler = kwargs.get('autoscaler', autoscaler)
         if self.autoscaler is not None and isinstance(self.autoscaler, dict):
@@ -191,8 +191,8 @@ class KubernetesRuntime(ServiceRuntime):
         if self.dynamic_concurrency is not None:
             _json['dynamicConcurrency'] = self.dynamic_concurrency
 
-        if self.concurrency_update_method is not None:
-            _json['concurrencyUpdateMethod'] = self.concurrency_update_method
+        if self.dynamic_concurrency_config is not None:
+            _json['dynamicConcurrencyConfig'] = self.dynamic_concurrency_config
 
         return _json
 

dtlpy/ml/base_model_adapter.py

@@ -472,31 +472,32 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             self.logger.debug("Downloading subset {!r} of {}".format(subset, self.model_entity.dataset.name))
 
             annotation_filters = None
-            if subset in annotations_subsets:
-                annotation_filters = entities.Filters(
-                    use_defaults=False,
-                    resource=entities.FiltersResource.ANNOTATION,
-                    custom_filter=annotations_subsets[subset],
-                )
-            # if user provided annotation_filters, skip the default filters
-            elif self.model_entity.output_type is not None and self.model_entity.output_type != "embedding":
-                annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION, use_defaults=False)
-                if self.model_entity.output_type in [
-                    entities.AnnotationType.SEGMENTATION,
-                    entities.AnnotationType.POLYGON,
-                ]:
-                    model_output_types = [entities.AnnotationType.SEGMENTATION, entities.AnnotationType.POLYGON]
-                else:
-                    model_output_types = [self.model_entity.output_type]
-
-                annotation_filters.add(
-                    field=entities.FiltersKnownFields.TYPE,
-                    values=model_output_types,
-                    operator=entities.FiltersOperations.IN,
-                )
+            if self.model_entity.output_type != "embedding":
+                if subset in annotations_subsets:
+                    annotation_filters = entities.Filters(
+                        use_defaults=False,
+                        resource=entities.FiltersResource.ANNOTATION,
+                        custom_filter=annotations_subsets[subset],
+                    )
+                # if user provided annotation_filters, skip the default filters
+                elif self.model_entity.output_type is not None:
+                    annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION, use_defaults=False)
+                    if self.model_entity.output_type in [
+                        entities.AnnotationType.SEGMENTATION,
+                        entities.AnnotationType.POLYGON,
+                    ]:
+                        model_output_types = [entities.AnnotationType.SEGMENTATION, entities.AnnotationType.POLYGON]
+                    else:
+                        model_output_types = [self.model_entity.output_type]
+
+                    annotation_filters.add(
+                        field=entities.FiltersKnownFields.TYPE,
+                        values=model_output_types,
+                        operator=entities.FiltersOperations.IN,
+                    )
 
-            annotation_filters = self.__include_model_annotations(annotation_filters)
-            annotations_subsets[subset] = annotation_filters.prepare()
+                annotation_filters = self.__include_model_annotations(annotation_filters)
+                annotations_subsets[subset] = annotation_filters.prepare()
 
             ret_list = self.__download_items(
                 dataset=dataset,
@@ -709,7 +710,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             valid_vectors = []
             items_to_upload = []
             vectors_to_upload = []
-            
+
             for item, vector in zip(_items, vectors):
                 # Check if vector is valid
                 if vector is None or len(vector) != embeddings_size:
@@ -719,25 +720,25 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                 # Item and vector are valid
                 valid_items.append(item)
                 valid_vectors.append(vector)
-                
+
                 # Check if item should be skipped (prompt items)
                 _system_metadata = getattr(item, 'system', dict())
                 is_prompt = _system_metadata.get('shebang', dict()).get('dltype', '') == 'prompt'
                 if skip_default_items and is_prompt:
                     self.logger.debug(f"Skipping feature upload for prompt item {item.id}")
                     continue
-                
+
                 # Items were not skipped - should be uploaded
                 items_to_upload.append(item)
                 vectors_to_upload.append(vector)
-            
+
             # Update the original lists with valid items only
             _items[:] = valid_items
             vectors[:] = valid_vectors
-            
+
             if len(_items) != len(vectors):
                 raise ValueError(f"The number of items ({len(_items)}) is not equal to the number of vectors ({len(vectors)}).")
-            
+
             self.logger.debug(f"Uploading {len(items_to_upload)} items' feature vectors for model {self.model_entity.name}.")
             try:
                 start_time = time.time()
@@ -830,7 +831,7 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             logger.info("Received {s} for training".format(s=model.id))
             model = model.wait_for_model_ready()
             if model.status == 'failed':
-                raise ValueError("Model is in failed state, cannot train.")
+                logger.warning("Model failed. New training will attempt to resume from previous checkpoints.")
 
             ##############
             # Set status #

dtlpy/repositories/apps.py

@@ -272,15 +272,16 @@ class Apps:
 
         return app
 
-    def uninstall(self, app_id: str = None, app_name: str = None, wait: bool = True) -> bool:
+    def uninstall(self, app_id: str = None, app_name: str = None, wait: bool = True, app: entities.App = None) -> bool:
         """
         Delete an app entity.
 
         Note: You are required to add either app_id or app_name.
 
         :param str app_id: optional - the id of the app.
-        :param str app_name: optional - the name of the app.
+        :param str app_name: [DEPRECATED] - the name of the app.
         :param bool wait: optional - wait for the operation to finish.
+        :param entities.App app: optional - the app entity.
         :return whether we succeed uninstalling the specified app.
         :rtype bool
 
@@ -288,12 +289,12 @@ class Apps:
         .. code-block:: python
             # succeed = dl.apps.delete(app_id='app_id')
         """
-        if app_id is None and app_name is None:
+        if app is None and app_id is None:
             raise exceptions.PlatformException(
                 error='400',
                 message='You must provide an identifier in inputs')
-        if app_name is not None:
-            app = self.__get_by_name(app_name)
+
+        if app is not None:
             app_id = app.id
 
         success, response = self._client_api.gen_request(req_type='delete', path='/apps/{}'.format(app_id))
@@ -302,18 +303,16 @@ class Apps:
 
         try:
             app = self.get(app_id=app_id)
-        except Exception as e:
-            if e.status_code == '404':
-                return success
-            else:
-                raise e
-        if app.metadata:
+        except exceptions.NotFound:
+            return success
+
+        if wait and app.status == entities.CompositionStatus.TERMINATING and app.metadata is not None:
             command_id = app.metadata.get('system', {}).get('commands', {}).get('uninstall', None)
-            if wait and app.status == entities.CompositionStatus.TERMINATING and command_id is not None:
+            if command_id is not None:
                 command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
                 command.wait()
 
-        logger.debug(f"App deleted successfully (id: {app_id}, name: {app_name}")
+        logger.debug(f"App deleted successfully (id: {app.id}, name: {app.name}")
         return success
 
     def resume(self, app: entities.App = None, app_id: str = None) -> bool: