dtlpy 1.85.25__py3-none-any.whl → 1.87.18__py3-none-any.whl

dtlpy/entities/service.py

@@ -80,8 +80,6 @@ class InstanceCatalog(str, Enum):
          - regular pod with medium size
        * - REGULAR_L
          - regular pod with large size
-       * - REGULAR_XL
-         - regular pod with extra large size
        * - HIGHMEM_XS
          - highmem pod with extra small size
        * - HIGHMEM_S
@@ -90,25 +88,27 @@ class InstanceCatalog(str, Enum):
          - highmem pod with medium size
        * - HIGHMEM_L
          - highmem pod with large size
-       * - HIGHMEM_XL
-         - highmem pod with extra large size
        * - GPU_K80_S
-         - GPU pod with small size
+         - GPU NVIDIA K80 pod with small size
        * - GPU_K80_M
-         - GPU pod with medium size
+         - GPU NVIDIA K80 pod with medium size
+       * - GPU_T4_S
+         - GPU NVIDIA T4 pod with regular memory
+       * - GPU_T4_M
+         - GPU NVIDIA T4 pod with highmem
     """
     REGULAR_XS = "regular-xs"
     REGULAR_S = "regular-s"
     REGULAR_M = "regular-m"
     REGULAR_L = "regular-l"
-    REGULAR_XL = "regular-xl"
     HIGHMEM_XS = "highmem-xs"
     HIGHMEM_S = "highmem-s"
     HIGHMEM_M = "highmem-m"
     HIGHMEM_L = "highmem-l"
-    HIGHMEM_XL = "highmem-xl"
     GPU_K80_S = "gpu-k80-s"
     GPU_K80_M = "gpu-k80-m"
+    GPU_T4_S = "gpu-t4"
+    GPU_T4_M = "gpu-t4-m"
 
 
 class RuntimeType(str, Enum):
@@ -150,7 +150,7 @@ class KubernetesRuntime(ServiceRuntime):
         self.concurrency = kwargs.get('concurrency', concurrency)
         self.runner_image = kwargs.get('runnerImage', runner_image)
         self._proxy_image = kwargs.get('proxyImage', None)
-        self.single_agent = kwargs.get('singleAgent', False)
+        self.single_agent = kwargs.get('singleAgent', None)
         self.preemptible = kwargs.get('preemptible', None)
 
         self.autoscaler = kwargs.get('autoscaler', autoscaler)
@@ -166,10 +166,12 @@ class KubernetesRuntime(ServiceRuntime):
             'podType': self.pod_type,
             'numReplicas': self.num_replicas,
             'concurrency': self.concurrency,
-            'singleAgent': self.single_agent,
             'autoscaler': None if self.autoscaler is None else self.autoscaler.to_json()
         }
 
+        if self.single_agent is not None:
+            _json['singleAgent'] = self.single_agent
+
         if self.runner_image is not None:
             _json['runnerImage'] = self.runner_image
 
@@ -221,6 +223,7 @@ class Service(entities.BaseEntity):
     max_attempts = attr.ib()
     mode = attr.ib(repr=False)
     metadata = attr.ib()
+    archive = attr.ib(repr=False)
 
     # SDK
     _package = attr.ib(repr=False)
@@ -229,6 +232,7 @@ class Service(entities.BaseEntity):
     # repositories
     _project = attr.ib(default=None, repr=False)
     _repositories = attr.ib(repr=False)
+    updated_by = attr.ib(default=None)
 
     @property
     def createdAt(self):
@@ -263,7 +267,7 @@ class Service(entities.BaseEntity):
         return status, service
 
     @classmethod
-    def from_json(cls, _json: dict, client_api: ApiClient=None, package=None, project=None, is_fetched=True):
+    def from_json(cls, _json: dict, client_api: ApiClient = None, package=None, project=None, is_fetched=True):
         """
         Build a service entity object from a json
 
@@ -323,7 +327,9 @@ class Service(entities.BaseEntity):
             secrets=_json.get("secrets", None),
             type=_json.get("type", None),
             mode=_json.get('mode', dict()),
-            metadata=_json.get('metadata', None)
+            metadata=_json.get('metadata', None),
+            archive=_json.get('archive', None),
+            updated_by=_json.get('updatedBy', None)
         )
         inst.is_fetched = is_fetched
         return inst
@@ -352,9 +358,13 @@ class Service(entities.BaseEntity):
     @property
     def package(self):
         if self._package is None:
-            self._package = repositories.Packages(client_api=self._client_api).get(package_id=self.package_id,
-                                                                                   fetch=None)
-        assert isinstance(self._package, entities.Package)
+            try:
+                self._package = repositories.Packages(client_api=self._client_api).get(package_id=self.package_id,
+                                                                                       fetch=None)
+                assert isinstance(self._package, entities.Package)
+            except:
+                self._package = repositories.Dpks(client_api=self._client_api).get(dpk_id=self.package_id)
+                assert isinstance(self._package, entities.Dpk)
         return self._package
 
     @property
@@ -443,7 +453,9 @@ class Service(entities.BaseEntity):
                 attr.fields(Service).secrets,
                 attr.fields(Service)._type,
                 attr.fields(Service).mode,
-                attr.fields(Service).metadata
+                attr.fields(Service).metadata,
+                attr.fields(Service).archive,
+                attr.fields(Service).updated_by,
             )
         )
 
@@ -463,6 +475,9 @@ class Service(entities.BaseEntity):
         _json['createdAt'] = self.created_at
         _json['updatedAt'] = self.updated_at
 
+        if self.updated_by is not None:
+            _json['updatedBy'] = self.updated_by
+
         if self.max_attempts is not None:
             _json['maxAttempts'] = self.max_attempts
 
@@ -487,6 +502,8 @@ class Service(entities.BaseEntity):
         if self.metadata:
             _json['metadata'] = self.metadata
 
+        if self.archive:
+            _json['archive'] = self.archive
         return _json
 
     def update(self, force=False):

dtlpy/entities/task.py

@@ -92,6 +92,7 @@ class Task:
     available_actions = attr.ib()
     total_items = attr.ib()
     priority = attr.ib()
+    _description = attr.ib()
 
     # sdk
     _client_api = attr.ib(repr=False)
@@ -102,6 +103,18 @@ class Task:
     _tasks = attr.ib(default=None, repr=False)
     _settings = attr.ib(default=None, repr=False)
 
+    @property
+    def description(self):
+        return self._description
+
+    @description.setter
+    def description(self, description):
+        if not isinstance(description, str):
+            raise ValueError('description should be a string')
+        if self._description is None:
+            self._description = {}
+        self._description['content'] = description
+
     @staticmethod
     def _protected_from_json(_json, client_api, project, dataset):
         """
@@ -176,7 +189,8 @@ class Task:
             created_at=_json.get('createdAt', None),
             available_actions=actions,
             total_items=_json.get('totalItems', None),
-            priority=_json.get('priority', None)
+            priority=_json.get('priority', None),
+            description=_json.get('description', None)
         )
 
     def to_json(self):
@@ -208,6 +222,7 @@ class Task:
                 attr.fields(Task).created_at,
                 attr.fields(Task).total_items,
                 attr.fields(Task)._settings,
+                attr.fields(Task)._description
             )
         )
         _json['projectId'] = self.project_id
@@ -217,6 +232,7 @@ class Task:
         _json['dueDate'] = self.due_date
         _json['totalItems'] = self.total_items
         _json['forReview'] = self.for_review
+        _json['description'] = self.description
 
         if self.available_actions is not None:
             _json['availableActions'] = [action.to_json() for action in self.available_actions]
@@ -472,3 +488,4 @@ class Task:
         :rtype: bool
         """
         return self.tasks.set_status(status=status, operation=operation, item_ids=item_ids, task_id=self.id)
+    

dtlpy/entities/trigger.py

@@ -84,6 +84,8 @@ class BaseTrigger(entities.BaseEntity):
     _op_type = attr.ib(default='service')
     _repositories = attr.ib(repr=False)
 
+    updated_by = attr.ib(default=None)
+
     @staticmethod
     def _get_operation(operation):
         op_type = operation.get('type', None)
@@ -243,6 +245,7 @@ class BaseTrigger(entities.BaseEntity):
                                                               attr.fields(BaseTrigger).created_at,
                                                               attr.fields(BaseTrigger).updated_at,
                                                               attr.fields(BaseTrigger).operation,
+                                                              attr.fields(BaseTrigger).updated_by,
                                                               ))
 
         # rename
@@ -251,6 +254,8 @@ class BaseTrigger(entities.BaseEntity):
         _json['updatedAt'] = self.updated_at
         if self.is_global is not None:
             _json['global'] = self.is_global
+        if self.updated_by is not None:
+            _json['updatedBy'] = self.updated_by
         return _json
 
     def delete(self):
@@ -342,7 +347,8 @@ class Trigger(BaseTrigger):
             op_type=operation.get('type', None),
             spec=spec,
             pipeline_id=pipeline_id,
-            operation=operation
+            operation=operation,
+            updated_by=_json.get('updatedBy', None),
         )
 
 

dtlpy/ml/base_model_adapter.py

@@ -1,10 +1,10 @@
-import copy
 import tempfile
 import datetime
 import logging
 import shutil
 import base64
 import tqdm
+import sys
 import io
 import os
 from PIL import Image
@@ -12,7 +12,7 @@ from functools import partial
 import numpy as np
 from concurrent.futures import ThreadPoolExecutor
 import attr
-from .. import entities, utilities, repositories
+from .. import entities, utilities, repositories, exceptions
 from ..services import service_defaults
 from ..services.api_client import ApiClient
 
@@ -36,6 +36,8 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                                       'image': self._item_to_image}
         if model_entity is not None:
             self.load_from_model(model_entity=model_entity)
+        logger.warning(
+            "in case of a mismatch between 'model.name' and 'model_info.name' in the model adapter, model_info.name will be updated to align with 'model.name'.")
 
     ##################
     # Configurations #
@@ -307,8 +309,6 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
         if cleanup:
             shutil.rmtree(path=local_path, ignore_errors=True)
             self.logger.info("Clean-up. deleting {}".format(local_path))
-        self.model_entity.status = 'trained'
-        self.model_entity = self.model_entity.update()
 
     # ===============
     # SERVICE METHODS
@@ -337,10 +337,15 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
         pool = ThreadPoolExecutor(max_workers=16)
 
         annotations = list()
-        for i_batch in tqdm.tqdm(range(0, len(items), batch_size), desc='predicting', unit='bt', leave=None):
+        for i_batch in tqdm.tqdm(range(0, len(items), batch_size), desc='predicting', unit='bt', leave=None, file=sys.stdout):
             batch_items = items[i_batch: i_batch + batch_size]
             batch = list(pool.map(self.prepare_item_func, batch_items))
             batch_collections = self.predict(batch, **kwargs)
+            _futures = list(pool.map(partial(self._update_predictions_metadata),
+                                     batch_items,
+                                     batch_collections))
+            # Loop over the futures to make sure they are all done to avoid race conditions
+            _ = [_f for _f in _futures]
             if upload_annotations is True:
                 self.logger.debug(
                     "Uploading items' annotation for model {!r}.".format(self.model_entity.name))
@@ -408,12 +413,11 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                     cleanup=False,
                     progress: utilities.Progress = None,
                     context: utilities.Context = None):
-        # FROM PARENT
         """
-            Train on existing model.
-            data will be taken from dl.Model.datasetId
-            configuration is as defined in dl.Model.configuration
-            upload the output the model's bucket (model.bucket)
+        Train on existing model.
+        data will be taken from dl.Model.datasetId
+        configuration is as defined in dl.Model.configuration
+        upload the output the model's bucket (model.bucket)
         """
         if isinstance(model, dict):
             model = repositories.Models(client_api=self._client_api).get(model_id=model['id'])
@@ -462,7 +466,8 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                                 progress=99)
 
             self.save_to_model(local_path=output_path, replace=True)
-
+            model.status = 'trained'
+            model.update()
             ###########
             # cleanup #
             ###########
@@ -597,6 +602,46 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
         image = np.asarray(Image.open(io.BytesIO(binary)))
         return image
 
+    def _update_predictions_metadata(self, item: entities.Item, predictions: entities.AnnotationCollection):
+        """
+        add model_name and model_id to the metadata of the annotations.
+        add model_info to the metadata of the system metadata of the annotation.
+        Add item id to all the annotations in the AnnotationCollection
+
+        :param item: Entity.Item
+        :param predictions: item's AnnotationCollection
+        :return:
+        """
+        for prediction in predictions:
+            if prediction.type == entities.AnnotationType.SEGMENTATION:
+                try:
+                    color = self.model_entity.dataset._get_ontology().color_map.get(prediction.label)
+                except (exceptions.BadRequest, exceptions.NotFound):
+                    color = None
+                    logger.warning("Can't get annotation color from item's dataset, using model's dataset.")
+                if color is None:
+                    try:
+                        color = self.model_entity.dataset._get_ontology().color_map.get(prediction.label)
+                    except (exceptions.BadRequest, exceptions.NotFound):
+                        logger.warning("Can't get annotation color from model's dataset, using default.")
+                        color = prediction.color
+                prediction.color = color
+
+            prediction.item_id = item.id
+            if 'user' in prediction.metadata and 'model' in prediction.metadata['user']:
+                prediction.metadata['user']['model']['model_id'] = self.model_entity.id
+                prediction.metadata['user']['model']['name'] = self.model_entity.name
+            if 'system' not in prediction.metadata:
+                prediction.metadata['system'] = dict()
+            if 'model' not in prediction.metadata['system']:
+                prediction.metadata['system']['model'] = dict()
+            confidence = prediction.metadata.get('user', dict()).get('model', dict()).get('confidence', None)
+            prediction.metadata['system']['model'] = {
+                'model_id': self.model_entity.id,
+                'name': self.model_entity.name,
+                'confidence': confidence
+            }
+
     ##############################
     # Callback Factory functions #
     ##############################

dtlpy/ml/train_utils.py

@@ -58,4 +58,3 @@ def prepare_dataset(dataset: entities.Dataset,
         cloned_dataset.metadata['system']['clone_info'].update({'filters': json.dumps(filters.prepare())})
     cloned_dataset.update(system_metadata=True)
     return cloned_dataset
-    # cloned_dataset.set_readonly(True)

dtlpy/new_instance.py

@@ -166,14 +166,16 @@ class Dtlpy:
         REGULAR_S = 'regular-s'
         REGULAR_M = 'regular-m'
         REGULAR_L = 'regular-l'
-        REGULAR_XL = 'regular-xl'
         HIGHMEM_MICRO = 'highmem-micro'
         HIGHMEM_XS = 'highmem-xs'
         HIGHMEM_S = 'highmem-s'
         HIGHMEM_M = 'highmem-m'
         HIGHMEM_L = 'highmem-l'
-        HIGHMEM_XL = 'highmem-xl'
-        GPU_K80_S = 'gpu-k80-s'
+        GPU_K80_S = "gpu-k80-s"
+        GPU_K80_M = "gpu-k80-m"
+        GPU_T4_S = "gpu-t4-s"
+        GPU_T4_M = "gpu-t4-m"
+
 
     class LoggingLevel:
         DEBUG = 'debug'

dtlpy/repositories/artifacts.py

@@ -22,7 +22,7 @@ class Artifacts:
                  project_id: str = None,
                  model: entities.Model = None,
                  package: entities.Package = None,
-                 dataset_name='Binaries'):
+                 dataset_name=None):
         self._client_api = client_api
         self._project = project
         self._dataset = dataset
@@ -40,21 +40,15 @@ class Artifacts:
         if self._dataset is None:
             # get dataset from project
             try:
-                self._dataset = self.project.datasets.get(dataset_name=self.dataset_name)
+                if self.dataset_name is None:
+                    self.dataset_name = 'Binaries'
+                    self._dataset = self.project.datasets._get_binaries_dataset()
+                else:
+                    self._dataset = self.project.datasets.get(dataset_name=self.dataset_name)
             except exceptions.NotFound:
-                self._dataset = None
-            if self._dataset is None:
-                logger.debug(
-                    'Dataset for artifacts was not found. Creating... dataset name: {ds!r}. project_id={id}'.format(
-                        ds=self.dataset_name, id=self.project.id))
-                self._dataset = self.project.datasets.create(dataset_name=self.dataset_name)
-                # add system to metadata
-                if 'metadata' not in self._dataset.to_json():
-                    self._dataset.metadata = dict()
-                if 'system' not in self._dataset.metadata:
-                    self._dataset.metadata['system'] = dict()
-                self._dataset.metadata['system']['scope'] = 'system'
-                self.project.datasets.update(dataset=self._dataset, system_metadata=True)
+                raise ValueError(
+                    f'Missing "{self.dataset_name}" dataset in the project. Please contact support for help')
+
         return self._dataset
 
     @property

dtlpy/repositories/codebases.py

@@ -65,21 +65,9 @@ class Codebases:
         if self._dataset is None:
             # get dataset from project
             try:
-                self._dataset = self.project.datasets.get(dataset_name='Binaries')
+                self._dataset = self.project.datasets._get_binaries_dataset()
             except exceptions.NotFound:
-                self._dataset = None
-            if self._dataset is None:
-                logger.debug(
-                    'Dataset for codebases was not found. Creating... dataset name: "Binaries". project_id={}'.format(
-                        self.project.id))
-                self._dataset = self.project.datasets.create(dataset_name='Binaries')
-                # add system to metadata
-                if 'metadata' not in self._dataset.to_json():
-                    self._dataset.metadata = dict()
-                if 'system' not in self._dataset.metadata:
-                    self._dataset.metadata['system'] = dict()
-                self._dataset.metadata['system']['scope'] = 'system'
-                self.project.datasets.update(dataset=self._dataset, system_metadata=True)
+                raise ValueError('Missing "Binaries" dataset in the project. Please contact support for help')
         assert isinstance(self._dataset, entities.Dataset)
         return self._dataset
 

dtlpy/repositories/commands.py

@@ -1,5 +1,4 @@
 import numpy as np
-import warnings
 import logging
 import time
 import tqdm
@@ -10,7 +9,7 @@ from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
-MAX_SLEEP_TIME = 8
+MAX_SLEEP_TIME = 30
 
 
 class Commands:
@@ -71,15 +70,15 @@ class Commands:
         return entities.Command.from_json(client_api=self._client_api,
                                           _json=response.json())
 
-    def wait(self, command_id, timeout=0, step=None, url=None, backoff_factor=0.1):
+    def wait(self, command_id, timeout=0, step=None, url=None, backoff_factor=1):
         """
         Wait for command to finish
 
         backoff_factor: A backoff factor to apply between attempts after the second try
         {backoff factor} * (2 ** ({number of total retries} - 1))
-        seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
-        for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
-        than 8 sec
+        seconds. If the backoff_factor is 1, then :func:`.sleep` will sleep
+        for [0s, 2s, 4s, ...] between retries. It will never be longer
+        than 30 sec
 
         :param str command_id: Command id to wait to
         :param int timeout: int, seconds to wait until TimeoutError is raised. if 0 - wait until done
@@ -103,7 +102,7 @@ class Commands:
             if not command.in_progress():
                 break
             elapsed = time.time() - start
-            sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** (num_tries - 1)), MAX_SLEEP_TIME])
+            sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
             num_tries += 1
             logger.debug("Command {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(command.id,
                                                                                                          elapsed,

dtlpy/repositories/datasets.py

@@ -96,6 +96,21 @@ class Datasets:
         filters.add(field='dir', values=folder_path + '*')
         return filters
 
+    def _get_binaries_dataset(self):
+        filters = entities.Filters(resource=entities.FiltersResource.DATASET)
+        filters.add(field='name', values='Binaries')
+        filters.system_space = True
+        datasets = self.list(filters=filters)
+        if len(datasets) == 0:
+            # empty list
+            raise exceptions.PlatformException('404', 'Dataset not found. Name: "Binaries"')
+            # dataset = None
+        elif len(datasets) > 1:
+            raise exceptions.PlatformException('400', 'More than one dataset with same name.')
+        else:
+            dataset = datasets[0]
+        return dataset
+
     @property
     def platform_url(self):
         return self._client_api._get_resource_url("projects/{}/datasets".format(self.project.id))
@@ -165,58 +180,78 @@ class Datasets:
         self._client_api.state_io.put('dataset', dataset.to_json())
         logger.info('Checked out to dataset {}'.format(dataset.name))
 
-    @_api_reference.add(path='/datasets', method='get')
-    def list(self, name=None, creator=None) -> miscellaneous.List[entities.Dataset]:
+    @_api_reference.add(path='/datasets/query', method='post')
+    def list(self, name=None, creator=None, filters: entities.Filters = None) -> miscellaneous.List[entities.Dataset]:
         """
         List all datasets.
 
         **Prerequisites**: You must be an *owner* or *developer* to use this method.
 
         :param str name: list by name
-        :param str creator: list by creator
+        :param str creator: list by
+        :param dtlpy.entities.filters.Filters filters: Filters entity containing filters parameters
         :return: List of datasets
         :rtype: list
 
         **Example**:
 
         .. code-block:: python
-
-            datasets = project.datasets.list(name='name')
+            filters = dl.Filters(resource='datasets')
+            filters.add(field='readonly', values=False)
+            datasets = project.datasets.list(filters=filters)
         """
-        url = '/datasets'
+        if filters is None:
+            filters = entities.Filters(resource=entities.FiltersResource.DATASET)
+        # assert type filters
+        elif not isinstance(filters, entities.Filters):
+            raise exceptions.PlatformException(error='400',
+                                               message='Unknown filters type: {!r}'.format(type(filters)))
+        if filters.resource != entities.FiltersResource.DATASET:
+            raise exceptions.PlatformException(
+                error='400',
+                message='Filters resource must to be FiltersResource.DATASET. Got: {!r}'.format(filters.resource))
 
-        query_params = {
-            'name': name,
-            'creator': creator
-        }
+        url = '/datasets/query'
 
+        if name is not None:
+            filters.add(field='name', values=name)
+        if creator is not None:
+            filters.add(field='creator', values=creator)
         if self._project is not None:
-            query_params['projects'] = self.project.id
-
-        url += '?{}'.format(urlencode({key: val for key, val in query_params.items() if val is not None}, doseq=True))
-
-        success, response = self._client_api.gen_request(req_type='get',
-                                                         path=url)
-        if success:
-            pool = self._client_api.thread_pools('entity.create')
-            datasets_json = response.json()
-            jobs = [None for _ in range(len(datasets_json))]
-            # return triggers list
-            for i_dataset, dataset in enumerate(datasets_json):
-                jobs[i_dataset] = pool.submit(entities.Dataset._protected_from_json,
-                                              **{'client_api': self._client_api,
-                                                 '_json': dataset,
-                                                 'datasets': self,
-                                                 'project': self.project})
-
-            # get all results
-            results = [j.result() for j in jobs]
-            # log errors
-            _ = [logger.warning(r[1]) for r in results if r[0] is False]
-            # return good jobs
-            datasets = miscellaneous.List([r[1] for r in results if r[0] is True])
-        else:
-            raise exceptions.PlatformException(response)
+            filters.context = {"projects": [self._project.id]}
+        filters.page_size = 1000
+        filters.page = 0
+        datasets = list()
+        while True:
+            success, response = self._client_api.gen_request(req_type='POST',
+                                                             json_req=filters.prepare(),
+                                                             path=url,
+                                                             headers={'user_query': filters._user_query})
+            if success:
+                pool = self._client_api.thread_pools('entity.create')
+                datasets_json = response.json()['items']
+                jobs = [None for _ in range(len(datasets_json))]
+                # return triggers list
+                for i_dataset, dataset in enumerate(datasets_json):
+                    jobs[i_dataset] = pool.submit(entities.Dataset._protected_from_json,
+                                                  **{'client_api': self._client_api,
+                                                     '_json': dataset,
+                                                     'datasets': self,
+                                                     'project': self.project})
+
+                # get all results
+                results = [j.result() for j in jobs]
+                # log errors
+                _ = [logger.warning(r[1]) for r in results if r[0] is False]
+                # return good jobs
+                datasets.extend([r[1] for r in results if r[0] is True])
+                if response.json()['hasNextPage'] is True:
+                    filters.page += 1
+                else:
+                    break
+            else:
+                raise exceptions.PlatformException(response)
+        datasets = miscellaneous.List(datasets)
         return datasets
 
     @_api_reference.add(path='/datasets/{id}', method='get')
@@ -953,10 +988,5 @@ class Datasets:
 
             project.datasets.set_readonly(dataset='dataset_entity', state=True)
         """
-        if dataset.readonly != state:
-            patch = {'readonly': state}
-            self.update(dataset=dataset,
-                        patch=patch)
-            dataset._readonly = state
-        else:
-            logger.warning('Dataset is already "readonly={}". Nothing was done'.format(state))
+        import warnings
+        warnings.warn("`readonly` flag on dataset is deprecated, doing nothing.", DeprecationWarning)

dtlpy/repositories/downloader.py

@@ -816,7 +816,7 @@ class Downloader:
                 total=3,
                 read=3,
                 connect=3,
-                backoff_factor=0.3,
+                backoff_factor=1,
             )
             adapter = HTTPAdapter(max_retries=retry)
             s.mount('http://', adapter)