dtlpy 1.98.8__py3-none-any.whl → 1.99.11__py3-none-any.whl

dtlpy/__init__.py

@@ -239,6 +239,7 @@ def checkout_state():
 
 
 def use_attributes_2(state: bool = True):
+    warnings.warn("Function 'use_attributes_2()' is deprecated as of version 1.99.11 and has been non-functional since version 1.90.39.  To work with attributes 2.0, simply use 'update_attributes()'.", DeprecationWarning)
     client_api.attributes_mode.use_attributes_2 = state
 
 

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.98.8'
+version = '1.99.11'

dtlpy/entities/dpk.py

@@ -59,6 +59,7 @@ class Toolbar(entities.DlEntity):
 
 class Panel(entities.DlEntity):
     name = entities.DlProperty(location=['name'], _type=str)
+    path = entities.DlProperty(location=['path'], _type=str, default=None)
     min_role = entities.DlProperty(location=['minRole'], _type=list)
     supported_slots = entities.DlProperty(location=['supportedSlots'], _type=list)
 

dtlpy/entities/model.py

@@ -307,7 +307,7 @@ class Model(entities.BaseEntity):
     @_repositories.default
     def set_repositories(self):
         reps = namedtuple('repositories',
-                          field_names=['projects', 'datasets', 'packages', 'models', 'ontologies', 'artifacts',
+                          field_names=['projects', 'datasets', 'models', 'packages', 'ontologies', 'artifacts',
                                        'metrics', 'dpks', 'services'])
 
         r = reps(projects=repositories.Projects(client_api=self._client_api),
@@ -552,14 +552,34 @@ class Model(entities.BaseEntity):
                                     filters=filters,
                                     service_config=service_config)
 
-    def predict(self, item_ids):
+    def predict(self, item_ids=None, dataset_id=None):
         """
         Run model prediction with items
 
         :param item_ids: a list of item id to run the prediction.
+        :param dataset_id: dataset id to run the prediction on
         :return:
         """
-        return self.models.predict(model=self, item_ids=item_ids)
+        return self.models.predict(model=self, item_ids=item_ids, dataset_id=dataset_id)
+
+    def embed(self, item_ids):
+        """
+        Run model embed with items
+
+        :param item_ids: a list of item id to run the embed.
+        :return:
+        """
+        return self.models.embed(model=self, item_ids=item_ids)
+
+    def embed_datasets(self, dataset_ids, attach_trigger=False):
+        """
+        Run model embed with datasets
+
+        :param dataset_ids: a list of dataset id to run the embed.
+        :param attach_trigger: bool - True, if you want to activate the trigger
+        :return:
+        """
+        return self.models.embed_datasets(model=self, dataset_ids=dataset_ids, attach_trigger=attach_trigger)
 
     def deploy(self, service_config=None) -> entities.Service:
         """

dtlpy/entities/prompt_item.py

@@ -347,7 +347,6 @@ class PromptItem:
     def add(self,
             message: dict,
             prompt_key: str = None,
-            stream: bool = True,
             model_info: dict = None):
         """
         add a prompt to the prompt item
@@ -356,29 +355,28 @@ class PromptItem:
 
         :param message:
         :param prompt_key:
-        :param stream:
         :param model_info:
         :return:
         """
-        if prompt_key is None:
-            prompt_key = len(self.prompts) + 1
         role = message.get('role', 'user')
         content = message.get('content', list())
 
         if self.role_mapping.get(role, 'item') == 'item':
+            if prompt_key is None:
+                prompt_key = str(len(self.prompts) + 1)
             # for new prompt we need a new key
-            prompt = Prompt(key=str(prompt_key), role=role)
+            prompt = Prompt(key=prompt_key, role=role)
             for element in content:
                 prompt.add_element(value=element.get('value', ''),
                                    mimetype=element.get('mimetype', PromptType.TEXT))
 
             # create new prompt and add to prompts
             self.prompts.append(prompt)
-            if self._item is not None and stream is True:
+            if self._item is not None:
                 self._item._Item__update_item_binary(_json=self.to_json())
         else:
-            # for response - we need to assign to previous key
-            prompt_key = str(prompt_key - 1)
+            if prompt_key is None:
+                prompt_key = str(len(self.prompts))
             assistant_message = content[0]
             assistant_mimetype = assistant_message.get('mimetype', PromptType.TEXT)
             uploaded_annotation = None
@@ -421,21 +419,19 @@ class PromptItem:
                 prompt.add_element(mimetype=PromptType.METADATA,
                                    value={"model_info": model_info})
 
-                if stream:
-                    existing_annotation = entities.Annotation.new(item=self._item,
-                                                                  metadata=metadata,
-                                                                  annotation_definition=annotation_definition)
-                    uploaded_annotation = existing_annotation.upload()
-                    prompt.add_element(mimetype=PromptType.METADATA,
-                                       value={"id": uploaded_annotation.id})
+                existing_annotation = entities.Annotation.new(item=self._item,
+                                                              metadata=metadata,
+                                                              annotation_definition=annotation_definition)
+                uploaded_annotation = existing_annotation.upload()
+                prompt.add_element(mimetype=PromptType.METADATA,
+                                   value={"id": uploaded_annotation.id})
                 existing_prompt = prompt
                 self.assistant_prompts.append(prompt)
 
-            # TODO Shadi fix
             existing_prompt_element = [element for element in existing_prompt.elements if
                                        element['mimetype'] != PromptType.METADATA][-1]
             existing_prompt_element['value'] = assistant_message.get('value')
-            if stream is True and uploaded_annotation is None:
+            if uploaded_annotation is None:
                 # Creating annotation with old dict to match platform dict
                 annotation_definition = entities.FreeText(text='')
                 metadata = {'system': {'promptId': prompt_key},
@@ -451,7 +447,3 @@ class PromptItem:
                 # update the annotation with the new text
                 annotation.annotation_definition.text = existing_prompt_element['value']
                 self._item.annotations.update(annotation)
-
-    def update(self):
-        if self._item is not None:
-            self._item._Item__update_item_binary(_json=self.to_json())

dtlpy/repositories/downloader.py

@@ -602,12 +602,20 @@ class Downloader:
         else:
             return item, '', False
 
-    def __video_validation(self, item, downloaded_file):
+    def __file_validation(self, item, downloaded_file):
         res = False
-        size_diff = os.stat(downloaded_file).st_size - item.metadata['system']['size']
+        resume = True
+        if isinstance(downloaded_file, io.BytesIO):
+            file_size = downloaded_file.getbuffer().nbytes
+        else:
+            file_size = os.stat(downloaded_file).st_size
+        expected_size = item.metadata['system']['size']
+        size_diff = file_size - expected_size
         if size_diff == 0:
             res = True
-        return res
+        if size_diff > 0:
+            resume = False
+        return res, file_size, resume
 
     def __thread_download(self,
                           item,
@@ -650,132 +658,165 @@ class Downloader:
 
         item, url, is_url = self.__get_link_source(item=item)
 
+        # save as byte stream
+        data = io.BytesIO()
         if need_to_download:
-            if not is_url:
-                headers = {'x-dl-sanitize': '0'}
-                result, response = self.items_repository._client_api.gen_request(req_type="get",
-                                                                                 headers=headers,
-                                                                                 path="/items/{}/stream".format(
-                                                                                     item.id),
-                                                                                 stream=True,
-                                                                                 dataset_id=item.dataset_id)
-                if not result:
-                    raise PlatformException(response)
-            else:
-                _, ext = os.path.splitext(item.metadata['system']['shebang']['linkInfo']['ref'].split('?')[0])
-                local_filepath += ext
-                response = self.get_url_stream(url=url)
-
-            if save_locally:
-                # save to file
-                if not os.path.exists(os.path.dirname(local_filepath)):
-                    # create folder if not exists
-                    os.makedirs(os.path.dirname(local_filepath), exist_ok=True)
-
-                # decide if create progress bar for item
-                total_length = response.headers.get("content-length")
-                one_file_pbar = None
-                try:
-                    one_file_progress_bar = total_length is not None and int(
-                        total_length) > 10e6  # size larger than 10 MB
-                    if one_file_progress_bar:
-                        one_file_pbar = tqdm.tqdm(total=int(total_length),
-                                                  unit='B',
-                                                  unit_scale=True,
-                                                  unit_divisor=1024,
-                                                  position=1,
-                                                  file=sys.stdout,
-                                                  disable=self.items_repository._client_api.verbose.disable_progress_bar,
-                                                  desc='Download Item')
-                except Exception as err:
-                    one_file_progress_bar = False
-                    logger.debug('Cant decide downloaded file length, bar will not be presented: {}'.format(err))
-
-                # start download
-                if self.items_repository._client_api.sdk_cache.use_cache and \
-                        self.items_repository._client_api.cache is not None:
-                    response_output = os.path.normpath(response.content)
-                    if isinstance(response_output, bytes):
-                        response_output = response_output.decode('utf-8')[1:-1]
-
-                    if os.path.isfile(os.path.normpath(response_output)):
-                        if response_output != local_filepath:
-                            source_path = os.path.normpath(response_output)
-                            shutil.copyfile(source_path, local_filepath)
+            chunk_resume = {0: 0}
+            start_point = 0
+            download_done = False
+            while chunk_resume.get(start_point, '') != 3 and not download_done:
+                if not is_url:
+                    headers = {'x-dl-sanitize': '0', 'Range': 'bytes={}-'.format(start_point)}
+                    result, response = self.items_repository._client_api.gen_request(req_type="get",
+                                                                                     headers=headers,
+                                                                                     path="/items/{}/stream".format(
+                                                                                         item.id),
+                                                                                     stream=True,
+                                                                                     dataset_id=item.dataset_id)
+                    if not result:
+                        raise PlatformException(response)
                 else:
+                    _, ext = os.path.splitext(item.metadata['system']['shebang']['linkInfo']['ref'].split('?')[0])
+                    local_filepath += ext
+                    response = self.get_url_stream(url=url)
+
+                if save_locally:
+                    # save to file
+                    if not os.path.exists(os.path.dirname(local_filepath)):
+                        # create folder if not exists
+                        os.makedirs(os.path.dirname(local_filepath), exist_ok=True)
+
+                    # decide if create progress bar for item
+                    total_length = response.headers.get("content-length")
+                    one_file_pbar = None
                     try:
-                        temp_file_path = local_filepath + '.download'
-                        with open(temp_file_path, "wb") as f:
+                        one_file_progress_bar = total_length is not None and int(
+                            total_length) > 10e6  # size larger than 10 MB
+                        if one_file_progress_bar:
+                            one_file_pbar = tqdm.tqdm(total=int(total_length),
+                                                      unit='B',
+                                                      unit_scale=True,
+                                                      unit_divisor=1024,
+                                                      position=1,
+                                                      file=sys.stdout,
+                                                      disable=self.items_repository._client_api.verbose.disable_progress_bar,
+                                                      desc='Download Item')
+                    except Exception as err:
+                        one_file_progress_bar = False
+                        logger.debug('Cant decide downloaded file length, bar will not be presented: {}'.format(err))
+
+                    # start download
+                    if self.items_repository._client_api.sdk_cache.use_cache and \
+                            self.items_repository._client_api.cache is not None:
+                        response_output = os.path.normpath(response.content)
+                        if isinstance(response_output, bytes):
+                            response_output = response_output.decode('utf-8')[1:-1]
+
+                        if os.path.isfile(os.path.normpath(response_output)):
+                            if response_output != local_filepath:
+                                source_path = os.path.normpath(response_output)
+                                shutil.copyfile(source_path, local_filepath)
+                    else:
+                        try:
+                            temp_file_path = local_filepath + '.download'
+                            with open(temp_file_path, "ab") as f:
+                                try:
+                                    for chunk in response.iter_content(chunk_size=chunk_size):
+                                        if chunk:  # filter out keep-alive new chunks
+                                            f.write(chunk)
+                                            if one_file_progress_bar:
+                                                one_file_pbar.update(len(chunk))
+                                except Exception as err:
+                                    pass
+                            file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
+                                                                                              download_progress=temp_file_path,
+                                                                                              chunk_resume=chunk_resume)
+                            if file_validation:
+                                shutil.move(temp_file_path, local_filepath)
+                                download_done = True
+                            else:
+                                if not is_url:
+                                    continue
+                                else:
+                                    raise PlatformException(
+                                        error="400",
+                                        message='Downloaded file is corrupted. Please try again. If the issue repeats please contact support.')
+                        except Exception as err:
+                            if os.path.isfile(temp_file_path):
+                                os.remove(temp_file_path)
+                            raise err
+                    if one_file_progress_bar:
+                        one_file_pbar.close()
+                    # save to output variable
+                    data = local_filepath
+                    # if image - can download annotation mask
+                    if item.annotated and annotation_options:
+                        self._download_img_annotations(item=item,
+                                                       img_filepath=local_filepath,
+                                                       annotation_options=annotation_options,
+                                                       annotation_filters=annotation_filters,
+                                                       local_path=local_path,
+                                                       overwrite=overwrite,
+                                                       thickness=thickness,
+                                                       alpha=alpha,
+                                                       with_text=with_text,
+                                                       export_version=export_version
+                                                       )
+                else:
+                    if self.items_repository._client_api.sdk_cache.use_cache and \
+                            self.items_repository._client_api.cache is not None:
+                        response_output = os.path.normpath(response.content)
+                        if isinstance(response_output, bytes):
+                            response_output = response_output.decode('utf-8')[1:-1]
+
+                        if os.path.isfile(response_output):
+                            source_file = response_output
+                            with open(source_file, 'wb') as f:
+                                data = f.read()
+                    else:
+                        try:
                             for chunk in response.iter_content(chunk_size=chunk_size):
                                 if chunk:  # filter out keep-alive new chunks
-                                    f.write(chunk)
-                                    if one_file_progress_bar:
-                                        one_file_pbar.update(len(chunk))
-                        # TODO remove this after the BE fix
-                        if self.__video_validation(item=item,
-                                                   downloaded_file=temp_file_path) or is_url:
-                            shutil.move(temp_file_path, local_filepath)
-                        else:
-                            os.remove(temp_file_path)
+                                    data.write(chunk)
+                            file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
+                                                                                              download_progress=data,
+                                                                                              chunk_resume=chunk_resume)
+                            if file_validation:
+                                download_done = True
+                            else:
+                                continue
+                        except Exception as err:
+                            raise err
+                    # go back to the beginning of the stream
+                    data.seek(0)
+                    data.name = item.name
+                    if not save_locally and to_array:
+                        if 'image' not in item.mimetype:
                             raise PlatformException(
-                                error=500,
-                                message='The downloaded file is corrupted. Please try again. If the issue repeats please contact support.')
-                    except Exception as err:
-                        if os.path.isfile(temp_file_path):
-                            os.remove(temp_file_path)
-                        raise err
-                if one_file_progress_bar:
-                    one_file_pbar.close()
-                # save to output variable
-                data = local_filepath
-                # if image - can download annotation mask
-                if item.annotated and annotation_options:
-                    self._download_img_annotations(item=item,
-                                                   img_filepath=local_filepath,
-                                                   annotation_options=annotation_options,
-                                                   annotation_filters=annotation_filters,
-                                                   local_path=local_path,
-                                                   overwrite=overwrite,
-                                                   thickness=thickness,
-                                                   alpha=alpha,
-                                                   with_text=with_text,
-                                                   export_version=export_version
-                                                   )
-            else:
-                # save as byte stream
-                data = io.BytesIO()
-                if self.items_repository._client_api.sdk_cache.use_cache and \
-                        self.items_repository._client_api.cache is not None:
-                    response_output = os.path.normpath(response.content)
-                    if isinstance(response_output, bytes):
-                        response_output = response_output.decode('utf-8')[1:-1]
-
-                    if os.path.isfile(response_output):
-                        source_file = response_output
-                        with open(source_file, 'wb') as f:
-                            data = f.read()
-                else:
-                    try:
-                        for chunk in response.iter_content(chunk_size=chunk_size):
-                            if chunk:  # filter out keep-alive new chunks
-                                data.write(chunk)
-                    except Exception as err:
-                        raise err
-                # go back to the beginning of the stream
-                data.seek(0)
-                data.name = item.name
-                if not save_locally and to_array:
-                    if 'image' not in item.mimetype:
-                        raise PlatformException(
-                            error="400",
-                            message='Download element type numpy.ndarray support for image only. '
-                                    'Item Id: {} is {} type'.format(item.id, item.mimetype))
+                                error="400",
+                                message='Download element type numpy.ndarray support for image only. '
+                                        'Item Id: {} is {} type'.format(item.id, item.mimetype))
 
-                    data = np.array(Image.open(data))
+                        data = np.array(Image.open(data))
         else:
             data = local_filepath
         return data
 
+    def __get_next_chunk(self, item, download_progress, chunk_resume):
+        size_validation, file_size, resume = self.__file_validation(item=item,
+                                                                    downloaded_file=download_progress)
+        start_point = file_size
+        if not size_validation:
+            if chunk_resume.get(start_point, None) is None:
+                chunk_resume = {start_point: 1}
+            else:
+                chunk_resume[start_point] += 1
+            if chunk_resume[start_point] == 3 or not resume:
+                raise PlatformException(
+                    error=500,
+                    message='The downloaded file is corrupted. Please try again. If the issue repeats please contact support.')
+        return size_validation, start_point, chunk_resume
+
     def __default_local_path(self):
 
         # create default local path

dtlpy/repositories/models.py

@@ -660,18 +660,21 @@ class Models:
                                             client_api=self._client_api,
                                             project=self._project)
 
-    def predict(self, model, item_ids):
+    def predict(self, model, item_ids, dataset_id=None):
         """
         Run model prediction with items
 
         :param model: dl.Model entity to run the prediction.
         :param item_ids: a list of item id to run the prediction.
+        :param dataset_id: a dataset id to run the prediction.
         :return:
         """
         if len(model.metadata['system'].get('deploy', {}).get('services', [])) == 0:
             # no services for model
             raise ValueError("Model doesnt have any associated services. Need to deploy before predicting")
-        payload = {'input': {'itemIds': item_ids},
+        if item_ids is None and dataset_id is None:
+            raise ValueError("Need to provide either item_ids or dataset_id")
+        payload = {'input': {'itemIds': item_ids, 'datasetId': dataset_id},
                    'config': {'serviceId': model.metadata['system']['deploy']['services'][0]}}
 
         success, response = self._client_api.gen_request(req_type="post",
@@ -683,6 +686,62 @@ class Models:
                                             client_api=self._client_api,
                                             project=self._project)
 
+    def embed(self, model, item_ids=None, dataset_id=None):
+        """
+        Run model embed with items
+
+        :param model: dl.Model entity to run the prediction.
+        :param item_ids: a list of item id to run the embed.
+        :param dataset_id: a dataset id to run the embed.
+        :return: Execution
+        :rtype: dtlpy.entities.execution.Execution
+        """
+        if len(model.metadata['system'].get('deploy', {}).get('services', [])) == 0:
+            # no services for model
+            raise ValueError("Model doesnt have any associated services. Need to deploy before predicting")
+        if item_ids is None and dataset_id is None:
+            raise ValueError("Need to provide either item_ids or dataset_id")
+        payload = {'input': {'itemIds': item_ids, 'datasetId': dataset_id},
+                   'config': {'serviceId': model.metadata['system']['deploy']['services'][0]}}
+
+        success, response = self._client_api.gen_request(req_type="post",
+                                                         path=f"/ml/models/{model.id}/embed",
+                                                         json_req=payload)
+        if not success:
+            raise exceptions.PlatformException(response)
+        return entities.Execution.from_json(_json=response.json(),
+                                            client_api=self._client_api,
+                                            project=self._project)
+
+    def embed_datasets(self, model, dataset_ids, attach_trigger=False):
+        """
+        Run model embed with datasets
+
+        :param model: dl.Model entity to run the prediction.
+        :param dataset_ids: a list of dataset id to run the embed.
+        :param attach_trigger: bool, if True will activate the trigger
+        :return:
+        """
+        if len(model.metadata['system'].get('deploy', {}).get('services', [])) == 0:
+            # no services for model
+            raise ValueError("Model doesnt have any associated services. Need to deploy before predicting")
+        if dataset_ids is None:
+            raise ValueError("Need to provide either dataset_id")
+        payload = {'datasetIds': dataset_ids,
+                   'config': {'serviceId': model.metadata['system']['deploy']['services'][0]},
+                   'attachTrigger': attach_trigger
+                   }
+
+        success, response = self._client_api.gen_request(req_type="post",
+                                                         path=f"/ml/models/{model.id}/embed/datasets",
+                                                         json_req=payload)
+        if not success:
+            raise exceptions.PlatformException(response)
+        command = entities.Command.from_json(_json=response.json(),
+                                             client_api=self._client_api)
+        command = command.wait()
+        return command
+
     def deploy(self, model_id: str, service_config=None) -> entities.Service:
         """
         Deploy a trained model. This will create a service that will execute predictions

dtlpy/repositories/upload_element.py

@@ -24,6 +24,7 @@ class BaseUploadElement:
         self.filename = all_upload_elements['filename']
         self.export_version = all_upload_elements['export_version']
         self.item_description = all_upload_elements['item_description']
+        self.driver_path = all_upload_elements['driver_path']
 
 
 class BinaryUploadElement(BaseUploadElement):
@@ -101,6 +102,16 @@ class ExternalItemUploadElement(BaseUploadElement):
         else:
             annotations_filepath = None
         # append to list
+        if self.remote_path is None:
+            split_dir = os.path.dirname(filepath).split('//')
+            if len(split_dir) > 1:
+                split_dir = split_dir[1] + '/'
+            if self.driver_path == '/':
+                self.driver_path = None
+            if self.driver_path and split_dir.startswith(self.driver_path):
+                self.remote_path = split_dir.replace(self.driver_path, '')
+            else:
+                self.remote_path = split_dir
         remote_filepath = self.remote_path + self.remote_name
         self.type = 'external_file'
         self.buffer = filepath

dtlpy/repositories/uploader.py

@@ -135,10 +135,13 @@ class Uploader:
                                     item_description=None):
         # fix remote path
         if remote_path is None:
-            remote_path = "/"
-        if not remote_path.startswith('/'):
+            if isinstance(local_path, str) and local_path.startswith('external://'):
+                remote_path = None
+            else:
+                remote_path = "/"
+        if remote_path and not remote_path.startswith('/'):
             remote_path = f"/{remote_path}"
-        if not remote_path.endswith("/"):
+        if remote_path and not remote_path.endswith("/"):
             remote_path = f"{remote_path}/"
         if file_types is not None and not isinstance(file_types, list):
             msg = '"file_types" should be a list of file extension. e.g [".jpg", ".png"]'
@@ -198,6 +201,12 @@ class Uploader:
         if remote_name is None:
             remote_name_list = [None] * len(local_path_list)
 
+        try:
+            driver_path = self.items_repository.dataset.project.drivers.get(
+                driver_id=self.items_repository.dataset.driver).path
+        except Exception:
+            driver_path = None
+
         futures = deque()
         total_size = 0
         for upload_item_element, remote_name, upload_annotations_element in zip(local_path_list,
@@ -238,7 +247,8 @@ class Uploader:
                 'filename': None,
                 'root': None,
                 'export_version': export_version,
-                'item_description': item_description
+                'item_description': item_description,
+                'driver_path': driver_path
             }
             if isinstance(upload_item_element, str):
                 with_head_folder = True
@@ -342,16 +352,9 @@ class Uploader:
         return futures
 
     async def __single_external_sync(self, element):
-        remote_path = element.remote_path
-        if len(remote_path) <= 1:
-            split_dir = os.path.dirname(element.buffer).split('//')
-            if len(split_dir) > 1:
-                remote_path = split_dir[1] + '/'
-
-        filename = remote_path + element.remote_name
         storage_id = element.buffer.split('//')[1]
         req_json = dict()
-        req_json['filename'] = filename
+        req_json['filename'] = element.remote_filepath
         req_json['storageId'] = storage_id
         success, response = self.items_repository._client_api.gen_request(req_type='post',
                                                                           path='/datasets/{}/imports'.format(
@@ -530,9 +533,10 @@ class Uploader:
                                    ref=item.id)
                 if pbar is not None:
                     pbar.update()
-                    self.items_repository._client_api.callbacks.run_on_event(event=self.items_repository._client_api.callbacks.CallbackEvent.ITEMS_UPLOAD,
-                                                                             context={'item_id': item.id, 'dataset_id': item.dataset_id},
-                                                                             progress=round(pbar.n / pbar.total * 100, 0))
+                    self.items_repository._client_api.callbacks.run_on_event(
+                        event=self.items_repository._client_api.callbacks.CallbackEvent.ITEMS_UPLOAD,
+                        context={'item_id': item.id, 'dataset_id': item.dataset_id},
+                        progress=round(pbar.n / pbar.total * 100, 0))
             else:
                 if isinstance(element.buffer, str):
                     ref = element.buffer

dtlpy/services/api_client.py

@@ -1170,12 +1170,7 @@ class ApiClient:
                 def callback(bytes_read):
                     pass
 
-        timeout = aiohttp.ClientTimeout(
-            total=None,  # Disable overall timeout
-            connect=2 * 60,  # Set connect timeout (in seconds)
-            sock_read=10 * 60,  # Set read timeout for socket read operations
-            sock_connect=2 * 60  # Set timeout for connection setup
-        )
+        timeout = aiohttp.ClientTimeout(total=0)
         async with aiohttp.ClientSession(headers=headers, timeout=timeout) as session:
             try:
                 form = aiohttp.FormData({})
@@ -1187,7 +1182,8 @@ class ApiClient:
                     form.add_field('description', item_description)
                 form.add_field('file', AsyncUploadStream(buffer=to_upload,
                                                          callback=callback,
-                                                         name=uploaded_filename))
+                                                         name=uploaded_filename,
+                                                         chunk_timeout=2 * 60))
                 url = '{}?mode={}'.format(self.base_gate_url + remote_url, mode)
 
                 # use SSL context

dtlpy/services/async_utils.py

@@ -36,6 +36,8 @@ class AsyncThreadEventLoop(threading.Thread):
     def semaphore(self, name, n=None):
         if n is None:
             n = self.n
+        else:
+            n = min(n, self.n)
         if name not in self._semaphores:
             self._semaphores[name] = asyncio.BoundedSemaphore(n)
         return self._semaphores[name]
@@ -94,17 +96,32 @@ class AsyncResponseError(AsyncResponse):
 
 
 class AsyncUploadStream(io.IOBase):
-    def __init__(self, buffer, callback=None, name=''):
+    def __init__(self, buffer, callback=None, name='', chunk_timeout=10, max_retries=3):
         self.buffer = buffer
         self.buffer.seek(0)
         self.callback = callback
         self._name = name
+        self.chunk_timeout = chunk_timeout
+        self.max_retries = max_retries
 
     @property
     def name(self):
         return self._name
 
+    async def async_read(self, size):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                data = await asyncio.wait_for(asyncio.to_thread(self.buffer.read, size), timeout=self.chunk_timeout)
+                if self.callback is not None:
+                    self.callback(size)
+                return data
+            except asyncio.TimeoutError:
+                retries += 1
+                logger.warning(
+                    f"Chunk read timed out after {self.chunk_timeout} seconds. Retrying {retries}/{self.max_retries}...")
+
+        raise Exception(f"Chunk read failed after {self.max_retries} retries due to timeouts")
+
     def read(self, size):
-        if self.callback is not None:
-            self.callback(size)
-        return self.buffer.read(size)
+        return asyncio.run(self.async_read(size))

{dtlpy-1.98.8.dist-info → dtlpy-1.99.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dtlpy
-Version: 1.98.8
+Version: 1.99.11
 Summary: SDK and CLI for Dataloop platform
 Home-page: https://github.com/dataloop-ai/dtlpy
 Author: Dataloop Team

{dtlpy-1.98.8.dist-info → dtlpy-1.99.11.dist-info}/RECORD

@@ -1,5 +1,5 @@
-dtlpy/__init__.py,sha256=fZYNrXe_suwThe-iSg5z45dkhTqiHe7btaxjhITesXE,20677
-dtlpy/__version__.py,sha256=VyHfNvXyywstm8UJU4FD18Pn8gxwDCQgKYhzxKXGUUY,19
+dtlpy/__init__.py,sha256=XcjyX8cwvOrrIHDS00P2iRvp11Kfysj1-mPlltnIe1s,20899
+dtlpy/__version__.py,sha256=pz-xlegF77txuTofrPEqyKxub0sp_icRq1r7TlkHMbM,20
 dtlpy/exceptions.py,sha256=EQCKs3pwhwZhgMByQN3D3LpWpdxwcKPEEt-bIaDwURM,2871
 dtlpy/new_instance.py,sha256=u_c6JtgqsKCr7TU24-g7_CaST9ghqamMhM4Z0Zxt50w,10121
 dtlpy/assets/__init__.py,sha256=D_hAa6NM8Zoy32sF_9b7m0b7I-BQEyBFg8-9Tg2WOeo,976
@@ -59,7 +59,7 @@ dtlpy/entities/command.py,sha256=ARu8ttk-C7_Ice7chRyTtyOtakBTF09FC04mEk73SO8,501
 dtlpy/entities/compute.py,sha256=4FEpahPFFGHxye_fLh_p_kP6iEQ3QJK7S5hAdd6Afos,12744
 dtlpy/entities/dataset.py,sha256=tNCl7nNCx-DrZ3z96APhRdvllfQA1-9y8DpL6Ma2l0I,47516
 dtlpy/entities/directory_tree.py,sha256=Rni6pLSWytR6yeUPgEdCCRfTg_cqLOdUc9uCqz9KT-Q,1186
-dtlpy/entities/dpk.py,sha256=a5C1UG_cvDnXSee650WHH43QflxbJCo_g0V17-GRb24,17639
+dtlpy/entities/dpk.py,sha256=fsJnKXyWTLI_sVkHt7j2stz_bdvXZ8ouNTiWSsiuUcA,17714
 dtlpy/entities/driver.py,sha256=O_QdK1EaLjQyQkmvKsmkNgmvmMb1mPjKnJGxK43KrOA,7197
 dtlpy/entities/execution.py,sha256=WBiAws-6wZnQQ3y9wyvOeexA3OjxfaRdwDu5dSFYL1g,13420
 dtlpy/entities/feature.py,sha256=9fFjD0W57anOVSAVU55ypxN_WTCsWTG03Wkc3cAAj78,3732
@@ -71,7 +71,7 @@ dtlpy/entities/item.py,sha256=G6VVcVCudqeShWigZmNIuKD4OkvTRJ05CeXFXNe3Jk8,29691
 dtlpy/entities/label.py,sha256=ycDYavIgKhz806plIX-64c07_TeHpDa-V7LnfFVe4Rg,3869
 dtlpy/entities/links.py,sha256=FAmEwHtsrqKet3c0UHH9u_gHgG6_OwF1-rl4xK7guME,2516
 dtlpy/entities/message.py,sha256=ApJuaKEqxATpXjNYUjGdYPu3ibQzEMo8-LtJ_4xAcPI,5865
-dtlpy/entities/model.py,sha256=UKtai_V8ckTNPlhzflmJNHXJvH6BH9UYOwCMWXNZueU,24822
+dtlpy/entities/model.py,sha256=ufrWny7eqpu4J3XKy8SXMLDBnIRuPvcNUcmbtM1pdxc,25567
 dtlpy/entities/node.py,sha256=yPPYDLtNMc6vZbbf4FIffY86y7tkaTvYm42Jb7k3Ofk,39617
 dtlpy/entities/ontology.py,sha256=ok4p3sLBc_SS5hs2gZr5-gbblrveM7qSIX4z67QSKeQ,31967
 dtlpy/entities/organization.py,sha256=AMkx8hNIIIjnu5pYlNjckMRuKt6H3lnOAqtEynkr7wg,9893
@@ -84,7 +84,7 @@ dtlpy/entities/paged_entities.py,sha256=6y44H3FSclQvhB1KLI4zuIs317hWOhdHUynldRrU
 dtlpy/entities/pipeline.py,sha256=X9238WbMGfZcXdQVEtkw8twZwl0O4EZB4TxbTSEyPeI,20788
 dtlpy/entities/pipeline_execution.py,sha256=XCXlBAHFYVL2HajE71hK-bPxI4gTwZvg5SKri4BgyRA,9928
 dtlpy/entities/project.py,sha256=ZUx8zA3mr6N145M62R3UDPCCzO1vxfyWO6vjES-bO-g,14653
-dtlpy/entities/prompt_item.py,sha256=Kmvguz3f0sGtkKZS9OEA_-Yi4aQRCgdg1GBkaLQyyTg,19592
+dtlpy/entities/prompt_item.py,sha256=d4rqP961PYlJvJJDRXZPI7Z6NdwRXlx_Q0_N0xtZ_B8,19276
 dtlpy/entities/recipe.py,sha256=Q1HtYgind3bEe-vnDZWhw6H-rcIAGhkGHPRWtLIkPSE,11917
 dtlpy/entities/reflect_dict.py,sha256=2NaSAL-CO0T0FYRYFQlaSpbsoLT2Q18AqdHgQSLX5Y4,3273
 dtlpy/entities/resource_execution.py,sha256=1HuVV__U4jAUOtOkWlWImnM3Yts8qxMSAkMA9sBhArY,5033
@@ -165,7 +165,7 @@ dtlpy/repositories/commands.py,sha256=kXhmyBpLZNs-6vKBo4iXaommpjcGBDXs287IICUnQM
 dtlpy/repositories/compositions.py,sha256=H417BvlQAiWr5NH2eANFke6CfEO5o7DSvapYpf7v5Hk,2150
 dtlpy/repositories/computes.py,sha256=EtfE_3JhTdNlSYDPkKXBFkq-DBl4sgQqIm50ajvFdWM,9976
 dtlpy/repositories/datasets.py,sha256=rDpJXNyxOlJwDQB-wNkM-JIqOGH10q9nujnAl6y8_xU,52077
-dtlpy/repositories/downloader.py,sha256=pNwL7Nid8xmOyYNiv4DB_WY4RoKlxQ-U9nG2V99Gyr8,41342
+dtlpy/repositories/downloader.py,sha256=h5Gs_hVXIOobzdwTHgLfkJYWiwtbRn3my-QMWnWJccw,44082
 dtlpy/repositories/dpks.py,sha256=mj3QPvfzj_jZAscwIgpKUfa7fLxptc3OJQ_RrSfgYxo,17487
 dtlpy/repositories/drivers.py,sha256=fF0UuHCyBzop8pHfryex23mf0kVFAkqzNdOmwBbaWxY,10204
 dtlpy/repositories/executions.py,sha256=4UoU6bnB3kl5cMuF1eJvDecfZCaB06gKWxPfv6_g1_k,32598
@@ -174,7 +174,7 @@ dtlpy/repositories/features.py,sha256=7xA2ihEuNgZD7HBQMMGLWpsS2V_3PgieKW2YAk1OeU
 dtlpy/repositories/integrations.py,sha256=Wi-CpT2PH36GFu3znWP5Uf2CmkqWBUYyOdwvatGD_eM,11798
 dtlpy/repositories/items.py,sha256=90Z8-thLWBd49fmmnP-P6pZxhHX1k4Wv6Qfxq-Ovcz4,38092
 dtlpy/repositories/messages.py,sha256=QU0Psckg6CA_Tlw9AVxqa-Ay1fRM4n269sSIJkH9o7E,3066
-dtlpy/repositories/models.py,sha256=GdVWHJ6kOIxM01wH7RVQ3CVaR4OmGurWJdQVHZezLDM,34789
+dtlpy/repositories/models.py,sha256=HGDyV0kUNeH96QgjOcjigg7KV0-NubFLLapmiCVeNik,37889
 dtlpy/repositories/nodes.py,sha256=xXJm_YA0vDUn0dVvaGeq6ORM0vI3YXvfjuylvGRtkxo,3061
 dtlpy/repositories/ontologies.py,sha256=unnMhD2isR9DVE5S8Fg6fSDf1ZZ5Xemxxufx4LEUT3w,19577
 dtlpy/repositories/organizations.py,sha256=6ijUDFbsogfRul1g_vUB5AZOb41MRmV5NhNU7WLHt3A,22825
@@ -190,14 +190,14 @@ dtlpy/repositories/settings.py,sha256=pvqNse0ANCdU3NSLJEzHco-PZq__OIsPSPVJveB9E4
 dtlpy/repositories/tasks.py,sha256=v09S2pYGkKx_vBG7SWigJeuMhp0GsefKo3Td7ImrWb0,49039
 dtlpy/repositories/times_series.py,sha256=m-bKFEgiZ13yQNelDjBfeXMUy_HgsPD_JAHj1GVx9fU,11420
 dtlpy/repositories/triggers.py,sha256=izdNyCN1gDc5uo7AXntso0HSMTDIzGFUp-dSEz8cn_U,21990
-dtlpy/repositories/upload_element.py,sha256=4CDZRKLubanOP0ZyGwxAHTtl6GLzwAyRAIm-PLYt0ck,10140
-dtlpy/repositories/uploader.py,sha256=SW3mJHFgd5JgYUhwTwm63xXFQ8DB97-bIzc3Fk9BYMU,31219
+dtlpy/repositories/upload_element.py,sha256=R2KWIXmkp_dMAIr81tu3Y_VRfldj0ju8__V28ombkcg,10677
+dtlpy/repositories/uploader.py,sha256=c-YPOGd3ONCHrPJKGDNe_DiFkUiIAdeWY5Yv1Neg_rM,31306
 dtlpy/repositories/webhooks.py,sha256=IIpxOJ-7KeQp1TY9aJZz-FuycSjAoYx0TDk8z86KAK8,9033
 dtlpy/services/__init__.py,sha256=VfVJy2otIrDra6i7Sepjyez2ujiE6171ChQZp-YgxsM,904
 dtlpy/services/aihttp_retry.py,sha256=tgntZsAY0dW9v08rkjX1T5BLNDdDd8svtgn7nH8DSGU,5022
-dtlpy/services/api_client.py,sha256=DRGSi2gTbgLh_LR0vhwWh3f-tYuJql6VKL58Ov1Iqug,69478
+dtlpy/services/api_client.py,sha256=_LpnqLO-9eFAMQkOk5qP3wMVcdwKyOQ-iNvC0jdvATg,69294
 dtlpy/services/api_reference.py,sha256=cW-B3eoi9Xs3AwI87_Kr6GV_E6HPoC73aETFaGz3A-0,1515
-dtlpy/services/async_utils.py,sha256=bVz7PLCpnldyQXMKTPahnQqAudxNW1-c71nfMjcI41Q,2858
+dtlpy/services/async_utils.py,sha256=6Dm82Yj8FWPSnxHy-zs6C7uvepiIsPiWtMf7sM2ChMg,3629
 dtlpy/services/calls_counter.py,sha256=gr0io5rIsO5-7Cgc8neA1vK8kUtYhgFPmDQ2jXtiZZs,1036
 dtlpy/services/check_sdk.py,sha256=tnFWCzkJa8w2jLtw-guwuqpOtXGyiVU7ZCDFiUZUqzY,3593
 dtlpy/services/cookie.py,sha256=sSZR1QV4ienCcZ8lEK_Y4nZYBgAxO3kHrcBXFKGcmwQ,3694
@@ -223,9 +223,9 @@ dtlpy/utilities/reports/report.py,sha256=3nEsNnIWmdPEsd21nN8vMMgaZVcPKn9iawKTTeO
 dtlpy/utilities/videos/__init__.py,sha256=SV3w51vfPuGBxaMeNemx6qEMHw_C4lLpWNGXMvdsKSY,734
 dtlpy/utilities/videos/video_player.py,sha256=LCxg0EZ_DeuwcT7U_r7MRC6Q19s0xdFb7x5Gk39PRms,24072
 dtlpy/utilities/videos/videos.py,sha256=Dj916B4TQRIhI7HZVevl3foFrCsPp0eeWwvGbgX3-_A,21875
-dtlpy-1.98.8.data/scripts/dlp,sha256=-F0vSCWuSOOtgERAtsPMPyMmzitjhB7Yeftg_PDlDjw,10
-dtlpy-1.98.8.data/scripts/dlp.bat,sha256=QOvx8Dlx5dUbCTMpwbhOcAIXL1IWmgVRSboQqDhIn3A,37
-dtlpy-1.98.8.data/scripts/dlp.py,sha256=tEokRaDINISXnq8yNx_CBw1qM5uwjYiZoJOYGqWB3RU,4267
+dtlpy-1.99.11.data/scripts/dlp,sha256=-F0vSCWuSOOtgERAtsPMPyMmzitjhB7Yeftg_PDlDjw,10
+dtlpy-1.99.11.data/scripts/dlp.bat,sha256=QOvx8Dlx5dUbCTMpwbhOcAIXL1IWmgVRSboQqDhIn3A,37
+dtlpy-1.99.11.data/scripts/dlp.py,sha256=tEokRaDINISXnq8yNx_CBw1qM5uwjYiZoJOYGqWB3RU,4267
 tests/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/assets/models_flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/assets/models_flow/failedmain.py,sha256=n8F4eu_u7JPrJ1zedbJPvv9e3lHb3ihoErqrBIcseEc,1847
@@ -233,9 +233,9 @@ tests/assets/models_flow/main.py,sha256=xotAjdHpFnIic3Wb-4f7GSg2igtuXZjvRPiYdCTa
 tests/assets/models_flow/main_model.py,sha256=Hl_tv7Q6KaRL3yLkpUoLMRqu5-ab1QsUYPL6RPEoamw,2042
 tests/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/features/environment.py,sha256=V23cUx_p4VpNk9kc2I0BDZJHO_xcJBFJq8m3JlYCooc,16736
-dtlpy-1.98.8.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-dtlpy-1.98.8.dist-info/METADATA,sha256=21j22oI6cRAaGPEiDzFoctdPwBRHrm3wGwQlrVCiB90,3002
-dtlpy-1.98.8.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-dtlpy-1.98.8.dist-info/entry_points.txt,sha256=C4PyKthCs_no88HU39eioO68oei64STYXC2ooGZTc4Y,43
-dtlpy-1.98.8.dist-info/top_level.txt,sha256=ZWuLmQGUOtWAdgTf4Fbx884w1o0vBYq9dEc1zLv9Mig,12
-dtlpy-1.98.8.dist-info/RECORD,,
+dtlpy-1.99.11.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+dtlpy-1.99.11.dist-info/METADATA,sha256=w7iA4gBhm97WutAg8hbHGg468RcdO1CzLc-2xrR4kM4,3003
+dtlpy-1.99.11.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+dtlpy-1.99.11.dist-info/entry_points.txt,sha256=C4PyKthCs_no88HU39eioO68oei64STYXC2ooGZTc4Y,43
+dtlpy-1.99.11.dist-info/top_level.txt,sha256=ZWuLmQGUOtWAdgTf4Fbx884w1o0vBYq9dEc1zLv9Mig,12
+dtlpy-1.99.11.dist-info/RECORD,,