dtlpy 1.116.6__py3-none-any.whl → 1.117.6__py3-none-any.whl

dtlpy/__init__.py

@@ -108,7 +108,7 @@ from .entities import (
     # compute
     ClusterProvider, ComputeType, ComputeStatus, Toleration, DeploymentResource, DeploymentResources,
     NodePool, AuthenticationIntegration, Authentication, ComputeCluster, ComputeContext, Compute, KubernetesCompute,
-    ServiceDriver, ExportType, OutputExportType
+    ServiceDriver, ExportType, OutputExportType, DynamicConcurrencyUpdateMethod
 )
 from .ml import BaseModelAdapter
 from .utilities import Converter, BaseServiceRunner, Progress, Context, AnnotationFormat

dtlpy/__version__.py

@@ -1 +1 @@
-version = '1.116.6'
+version = '1.117.6'

dtlpy/entities/__init__.py

@@ -44,7 +44,7 @@ from .package_slot import PackageSlot, SlotPostAction, SlotPostActionType, SlotD
 from .package_function import PackageFunction, FunctionIO, PackageInputType
 from .time_series import TimeSeries
 from .service import Service, KubernetesAutoscalerType, KubernetesRabbitmqAutoscaler, KubernetesAutoscaler, KubernetesRPSAutoscaler, \
-    InstanceCatalog, KubernetesRuntime, ServiceType, ServiceModeType
+    InstanceCatalog, KubernetesRuntime, ServiceType, ServiceModeType, DynamicConcurrencyUpdateMethod
 from .execution import Execution, ExecutionStatus
 from .command import Command, CommandsStatus
 from .assignment import Assignment, Workload, WorkloadUnit

dtlpy/entities/feature_set.py

@@ -135,6 +135,13 @@ class FeatureSet(entities.BaseEntity):
 
         return _json
 
+    def update(self):
+        """
+        Update feature set
+
+        :return: entities.FeatureSet
+        """
+        return self.feature_sets.update(self)
     def delete(self):
         """
         Delete the feature set

dtlpy/entities/item.py

@@ -80,6 +80,22 @@ class Item(entities.BaseEntity):
     def datasetId(self):
         return self.dataset_id
 
+    @property
+    def resolved_stream(self):
+        stream = self.metadata.get('system', dict()).get('shebang', dict()).get('linkInfo', dict()).get('ref', None)
+        if stream is None:
+            stream = self.stream
+            api_url = self._client_api.environment
+            if api_url != self._client_api.base_gate_url:
+                stream = stream.replace(api_url, self._client_api.base_gate_url)
+        else:
+            link_item_url_override = os.environ.get('LINK_ITEM_URL_OVERRIDE', None)
+            if link_item_url_override is not None:
+                src, target = link_item_url_override.split(',')
+                stream = stream.replace(src, target)
+
+        return stream
+
     @staticmethod
     def _protected_from_json(_json, client_api, dataset=None):
         """

dtlpy/entities/ontology.py

@@ -290,7 +290,7 @@ class Ontology(entities.BaseEntity):
 
     def update(self, system_metadata=False):
         """
-        Update items metadata
+        Update Ontology attribute
 
         :param bool system_metadata: bool - True, if you want to change metadata system
         :return: Ontology object

dtlpy/entities/service.py

@@ -11,6 +11,12 @@ from ..services.api_client import ApiClient
 
 logger = logging.getLogger(name='dtlpy')
 
+class DynamicConcurrencyUpdateMethod(str, Enum):
+    """ The method of updating the dynamic concurrency.
+    """
+    RESTART = 'restart',
+    SYNC = 'sync'
+
 
 class ServiceType(str, Enum):
     """ The type of the service (SYSTEM).
@@ -136,6 +142,7 @@ class KubernetesRuntime(ServiceRuntime):
                  num_replicas=DEFAULT_NUM_REPLICAS,
                  concurrency=DEFAULT_CONCURRENCY,
                  dynamic_concurrency=None,
+                 concurrency_update_method=None,
                  runner_image=None,
                  autoscaler=None,
                  **kwargs):
@@ -149,6 +156,7 @@ class KubernetesRuntime(ServiceRuntime):
         self.single_agent = kwargs.get('singleAgent', None)
         self.preemptible = kwargs.get('preemptible', None)
         self.dynamic_concurrency = kwargs.get('dynamicConcurrency', dynamic_concurrency)
+        self.concurrency_update_method = kwargs.get('concurrencyUpdateMethod', concurrency_update_method)
 
         self.autoscaler = kwargs.get('autoscaler', autoscaler)
         if self.autoscaler is not None and isinstance(self.autoscaler, dict):
@@ -183,6 +191,9 @@ class KubernetesRuntime(ServiceRuntime):
         if self.dynamic_concurrency is not None:
             _json['dynamicConcurrency'] = self.dynamic_concurrency
 
+        if self.concurrency_update_method is not None:
+            _json['concurrencyUpdateMethod'] = self.concurrency_update_method
+
         return _json
 
 

dtlpy/ml/base_model_adapter.py

@@ -368,14 +368,41 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
                 annotation_filters.custom_filter['filter']['$and'].append({'metadata.system.model.name': {'$exists': False}})
         return annotation_filters
 
+    def __download_items(self, dataset, filters, local_path, annotation_options, annotation_filters=None):
+        """
+        Download items from dataset with optional annotation filters.
+
+        :param dataset: Dataset to download from
+        :param filters: Filters to apply
+        :param local_path: Local path to save files
+        :param annotation_options: Annotation download options
+        :param annotation_filters: Optional filters for annotations
+        :return: List of downloaded items
+        """
+        if annotation_options == entities.ViewAnnotationOptions.JSON:
+            downloader = repositories.Downloader(dataset.items)
+            return downloader._download_recursive(
+                local_path=local_path,
+                filters=filters,
+                annotation_filters=annotation_filters
+            )
+        else:
+            return dataset.items.download(
+                filters=filters,
+                local_path=local_path,
+                annotation_options=annotation_options,
+                annotation_filters=annotation_filters
+            )
+
     def __download_background_images(self, filters, data_subset_base_path, annotation_options):
         background_list = list()
         if self.configuration.get('include_background', False) is True:
             filters.custom_filter["filter"]["$and"].append({"annotated": False})
-            background_list = self.model_entity.dataset.items.download(
+            background_list = self.__download_items(
+                dataset=self.model_entity.dataset,
                 filters=filters,
                 local_path=data_subset_base_path,
-                annotation_options=annotation_options,
+                annotation_options=annotation_options
             )
         return background_list
 
@@ -434,13 +461,14 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
         if subsets is None:
             raise ValueError("Model (id: {}) must have subsets in metadata.system.subsets".format(self.model_entity.id))
         for subset, filters_dict in subsets.items():
+            _filters_dict = filters_dict.copy()
             data_subset_base_path = os.path.join(data_path, subset)
             if os.path.isdir(data_subset_base_path) and not overwrite:
                 # existing and dont overwrite
                 self.logger.debug("Subset {!r} already exists (and overwrite=False). Skipping.".format(subset))
                 continue
 
-            filters = entities.Filters(custom_filter=filters_dict)
+            filters = entities.Filters(custom_filter=_filters_dict)
             self.logger.debug("Downloading subset {!r} of {}".format(subset, self.model_entity.dataset.name))
 
             annotation_filters = None
@@ -470,13 +498,15 @@ class BaseModelAdapter(utilities.BaseServiceRunner):
             annotation_filters = self.__include_model_annotations(annotation_filters)
             annotations_subsets[subset] = annotation_filters.prepare()
 
-            ret_list = dataset.items.download(
+            ret_list = self.__download_items(
+                dataset=dataset,
                 filters=filters,
                 local_path=data_subset_base_path,
                 annotation_options=annotation_options,
-                annotation_filters=annotation_filters,
+                annotation_filters=annotation_filters
             )
-            filters = entities.Filters(custom_filter=subsets[subset])
+            _filters_dict = subsets[subset].copy()
+            filters = entities.Filters(custom_filter=_filters_dict)
             background_ret_list = self.__download_background_images(
                 filters=filters,
                 data_subset_base_path=data_subset_base_path,

dtlpy/repositories/datasets.py

@@ -2,23 +2,28 @@
 Datasets Repository
 """
 
+import copy
+import json
+import logging
 import os
 import sys
+import tempfile
 import time
-import copy
-import tqdm
-import logging
 import zipfile
-import json
-from typing import Union, Generator, Optional
+from pathlib import Path
+from typing import Generator, Optional, Union
+
+import tqdm
 
-from .. import entities, repositories, miscellaneous, exceptions, services, PlatformException, _api_reference
+from .. import _api_reference, entities, exceptions, miscellaneous, PlatformException, repositories, services
+from ..entities.dataset import ExportType, OutputExportType
+from ..services import service_defaults
 from ..services.api_client import ApiClient
-from ..entities.dataset import OutputExportType, ExportType
 
 logger = logging.getLogger(name='dtlpy')
 
 MAX_ITEMS_PER_SUBSET = 50000
+DOWNLOAD_ANNOTATIONS_MAX_ITEMS_PER_SUBSET = 1000
 
 class Datasets:
     """
@@ -129,6 +134,54 @@ class Datasets:
             dataset_id = dataset.id
         return dataset_id
 
+    @staticmethod
+    def _save_item_json_file(item_data, base_path: Path, export_version=None):
+        """
+        Save a single item's JSON data to a file, creating the directory structure as needed.
+        
+        :param dict item_data: The item data dictionary (must have 'filename' key)
+        :param Path base_path: Base directory path where JSON files should be saved
+        :param entities.ExportVersion export_version: Optional export version (V1 or V2) affecting filename handling
+        :return: Path to the saved JSON file
+        :rtype: Path
+        """
+        # Get filename and remove leading slash
+        filename = item_data.get('filename', '')
+        if not filename:
+            raise ValueError("item_data must have a 'filename' key")
+        filename = filename.lstrip('/')
+        
+        # Determine relative JSON path based on export version
+        if export_version == entities.ExportVersion.V1:
+            # V1: Replace extension with .json (e.g., "file.jpg" -> "file.json")
+            rel_json_path = str(Path(filename).with_suffix('.json'))
+        elif export_version == entities.ExportVersion.V2:
+            # V2: Append .json (e.g., "file.jpg" -> "file.jpg.json")
+            rel_json_path = filename + '.json'
+        else:
+            # Default/None: Replace extension with .json (backward compatible with section 1)
+            rel_json_path = os.path.splitext(filename)[0] + '.json'
+        
+        # Remove leading slash if present
+        if rel_json_path.startswith('/'):
+            rel_json_path = rel_json_path[1:]
+        
+        # Build output path
+        out_path = base_path / rel_json_path
+        
+        # Create parent directories
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # Write JSON file
+        try:
+            with open(out_path, 'w') as outf:
+                json.dump(item_data, outf, indent=2)
+        except Exception:
+            logger.exception(f'Failed writing export item JSON to {out_path}')
+            raise
+        
+        return out_path
+
     @staticmethod
     def _build_payload(filters, include_feature_vectors, include_annotations,
                        export_type, annotation_filters, feature_vector_filters, dataset_lock, lock_timeout_sec, export_summary):
@@ -902,17 +955,7 @@ class Datasets:
         logger.debug("start building per-item JSON files under local_path mirroring remote structure")
         # Build per-item JSON files under local_path mirroring remote structure
         for item in all_items:
-            rel_json_path = os.path.splitext(item.get('filename'))[0] + '.json'
-            # Remove leading slash to make it a relative path
-            if rel_json_path.startswith('/'):
-                rel_json_path = rel_json_path[1:]
-            out_path = os.path.join(base_dir, rel_json_path)
-            os.makedirs(os.path.dirname(out_path), exist_ok=True)
-            try:
-                with open(out_path, 'w') as outf:
-                    json.dump(item, outf)
-            except Exception:
-                logger.exception(f'Failed writing export item JSON to {out_path}')
+            self._save_item_json_file(item_data=item, base_path=Path(base_dir), export_version=None)
         logger.debug("end building per-item JSON files under local_path mirroring remote structure")
         return base_dir
 
@@ -1159,7 +1202,7 @@ class Datasets:
                              include_annotations_in_output: bool = True,
                              export_png_files: bool = False,
                              filter_output_annotations: bool = False,
-                             alpha: float = None,
+                             alpha: float = 1,
                              export_version=entities.ExportVersion.V1,
                              dataset_lock: bool = False, 
                              lock_timeout_sec: int = None,
@@ -1216,33 +1259,26 @@ class Datasets:
         elif not isinstance(annotation_options, list):
             annotation_options = [annotation_options]
         for ann_option in annotation_options:
-            if not isinstance(ann_option, entities.ViewAnnotationOptions):
-                if ann_option not in list(entities.ViewAnnotationOptions):
-                    raise PlatformException(
-                        error='400',
-                        message='Unknown annotation download option: {}, please choose from: {}'.format(
-                            ann_option, list(entities.ViewAnnotationOptions)))
-
+            if ann_option not in entities.ViewAnnotationOptions:
+                raise PlatformException(
+                    error='400',
+                    message=f'Unknown annotation download option: {ann_option}, please choose from: {list(entities.ViewAnnotationOptions)}',
+                )
         if remote_path is not None:
-            logger.warning(
-                '"remote_path" is ignored. Use "filters=dl.Filters(field="dir, values={!r}"'.format(remote_path))
+            logger.warning(f'"remote_path" is ignored. Use "filters=dl.Filters(field="dir, values={remote_path!r}"')
+        if filter_output_annotations is True:
+            logger.warning("'filter_output_annotations' is ignored but kept for legacy support")
+        if include_annotations_in_output is False:
+            logger.warning("include_annotations_in_output was False, but was set to True since this function downloads annotations.")
+            include_annotations_in_output = True
+
         if local_path is None:
             if dataset.project is None:
                 # by dataset name
-                local_path = os.path.join(
-                    services.service_defaults.DATALOOP_PATH,
-                    "datasets",
-                    "{}_{}".format(dataset.name, dataset.id),
-                )
+                local_path = str(Path(service_defaults.DATALOOP_PATH) / "datasets" / f"{dataset.name}_{dataset.id}")
             else:
                 # by dataset and project name
-                local_path = os.path.join(
-                    services.service_defaults.DATALOOP_PATH,
-                    "projects",
-                    dataset.project.name,
-                    "datasets",
-                    dataset.name,
-                )
+                local_path = str(Path(service_defaults.DATALOOP_PATH) / "projects" / dataset.project.name / "datasets" / dataset.name)
 
         if filters is None:
             filters = entities.Filters()
@@ -1260,53 +1296,98 @@ class Datasets:
                                  method=entities.FiltersMethod.OR)
 
         downloader = repositories.Downloader(items_repository=dataset.items)
-        downloader.download_annotations(dataset=dataset,
-                                        filters=filters,
-                                        annotation_filters=annotation_filters,
-                                        local_path=local_path,
-                                        overwrite=overwrite,
-                                        include_annotations_in_output=include_annotations_in_output,
-                                        export_png_files=export_png_files,
-                                        filter_output_annotations=filter_output_annotations,
-                                        export_version=export_version,
-                                        dataset_lock=dataset_lock,
-                                        lock_timeout_sec=lock_timeout_sec,
-                                        export_summary=export_summary                                      
-                                        )
-        if annotation_options:
-            pages = dataset.items.list(filters=filters)
-            if not isinstance(annotation_options, list):
-                annotation_options = [annotation_options]
-            # convert all annotations to annotation_options
+
+        # Setup for incremental processing
+        if len(annotation_options) == 0 :
+            pool = None
+            progress = None
+            jobs = []
+        else:             
+            # Get total count for progress bar
+            filter_copy = copy.deepcopy(filters)
+            filter_copy.page_size = 0
+            pages = dataset.items.list(filters=filter_copy)
+            total_items = pages.items_count
+
+            # Setup thread pool and progress bar
             pool = dataset._client_api.thread_pools(pool_name='dataset.download')
-            jobs = [None for _ in range(pages.items_count)]
-            progress = tqdm.tqdm(total=pages.items_count,
-                                 disable=dataset._client_api.verbose.disable_progress_bar_download_annotations,
-                                 file=sys.stdout, desc='Download Annotations')
-            i_item = 0
-            for page in pages:
-                for item in page:
-                    jobs[i_item] = pool.submit(
-                        Datasets._convert_single,
-                        **{
-                            'downloader': downloader,
-                            'item': item,
-                            'img_filepath': None,
-                            'local_path': local_path,
-                            'overwrite': overwrite,
-                            'annotation_options': annotation_options,
-                            'annotation_filters': annotation_filters,
-                            'thickness': thickness,
-                            'with_text': with_text,
-                            'progress': progress,
-                            'alpha': alpha,
-                            'export_version': export_version
-                        }
-                    )
-                    i_item += 1
-            # get all results
+            progress = tqdm.tqdm(
+                total=total_items,
+                disable=dataset._client_api.verbose.disable_progress_bar_download_annotations,
+                file=sys.stdout,
+                desc='Download Annotations'
+            )
+            jobs = []
+            
+
+        # Call _export_recursive as generator
+        export_generator = dataset.project.datasets._export_recursive(
+            dataset=dataset,
+            local_path=tempfile.mkdtemp(prefix='annotations_jsons_'),
+            filters=filters,
+            annotation_filters=annotation_filters,
+            include_annotations=True,
+            export_type=ExportType.JSON,
+            dataset_lock=dataset_lock,
+            lock_timeout_sec=lock_timeout_sec,
+            export_summary=export_summary,
+            timeout=0,
+            max_items_per_subset=DOWNLOAD_ANNOTATIONS_MAX_ITEMS_PER_SUBSET
+        )
+
+        # Process each subset JSON file incrementally
+        for subset_json_file in export_generator:
+            if subset_json_file is None or not Path(subset_json_file).is_file():
+                continue
+
+            try:
+                # Open and load the items array
+                with open(subset_json_file, 'r') as f:
+                    items_data = json.load(f)
+
+                # Process each item immediately
+                for item_data in items_data:
+                    # Split and save individual JSON file
+                    Datasets._save_item_json_file(item_data=item_data, base_path=Path(local_path) / 'json', export_version=export_version)
+
+                    # If annotation_options are provided, submit to thread pool immediately
+                    if annotation_options:
+                        # Create Item entity from item_data
+                        item = entities.Item.from_json(
+                            _json=item_data,
+                            client_api=dataset._client_api,
+                            dataset=dataset
+                        )
+
+                        job = pool.submit(
+                            Datasets._convert_single,
+                            **{
+                                'downloader': downloader,
+                                'item': item,
+                                'img_filepath': None,
+                                'local_path': local_path,
+                                'overwrite': overwrite,
+                                'annotation_options': annotation_options,
+                                'annotation_filters': annotation_filters,
+                                'thickness': thickness,
+                                'with_text': with_text,
+                                'progress': progress,
+                                'alpha': alpha,
+                                'export_version': export_version
+                            }
+                        )
+                        jobs.append(job)
+
+                # Clean up temporary subset JSON file
+                os.remove(subset_json_file)
+            except Exception as e:
+                logger.exception(f'Failed processing subset JSON file {subset_json_file}: {e}')
+
+        # Wait for all thread pool jobs to complete
+        if annotation_options:
             _ = [j.result() for j in jobs]
             progress.close()
+
         return local_path
 
     def _upload_single_item_annotation(self, item, file, pbar):

dtlpy/repositories/downloader.py

@@ -1,18 +1,22 @@
+import copy
+import io
+import json
+import logging
+import multiprocessing
+import os
+import shutil
+import sys
+import tempfile
+import traceback
 from pathlib import Path
-from requests.adapters import HTTPAdapter
-from urllib3.util import Retry
-from PIL import Image
+from urllib.parse import unquote, urlparse
+
 import numpy as np
-import traceback
-from urllib.parse import urlparse, unquote
 import requests
-import logging
-import shutil
-import json
 import tqdm
-import sys
-import os
-import io
+from PIL import Image
+from requests.adapters import HTTPAdapter
+from urllib3.util import Retry
 
 from .. import entities, repositories, miscellaneous, PlatformException, exceptions
 from ..services import Reporter
@@ -20,12 +24,257 @@ from ..services import Reporter
 logger = logging.getLogger(name='dtlpy')
 
 NUM_TRIES = 3  # try to download 3 time before fail on item
-
+DOWNLOAD_MAX_ITEMS_PER_SUBSET = 1000
 
 class Downloader:
     def __init__(self, items_repository):
         self.items_repository = items_repository
 
+    def _process_download_results(self, reporter, raise_on_error=False):
+        """
+        Process download results and generate summary report.
+
+        :param reporter: Reporter instance containing download results
+        :param raise_on_error: If True, raise exception on download errors
+        :return: Output from reporter
+        """
+        # reporting
+        n_download = reporter.status_count(status='download')
+        n_exist = reporter.status_count(status='exist')
+        n_error = reporter.status_count(status='error')
+        logger.info(f"Number of files downloaded:{n_download}")
+        logger.info(f"Number of files exists: {n_exist}")
+        logger.info(f"Total number of files: {n_download + n_exist}")
+
+        # log error
+        if n_error > 0:
+            log_filepath = reporter.generate_log_files()
+            # Get up to 5 error examples for the exception message
+            error_text = ""
+            error_counter = 0
+            if reporter._errors:
+                for _id, error in reporter._errors.items():
+                    error_counter += 1
+                    error_text += f"Item ID: {_id}, Error: {error} | "
+                    if error_counter >= 5:
+                        break
+            error_message = f"Errors in {n_error} files. Errors: {error_text}"
+            if log_filepath is not None:
+                error_message += f", see {log_filepath} for full log"
+            if raise_on_error is True:
+                raise PlatformException(
+                    error="400", message=error_message
+                )
+            else:
+                logger.warning(error_message)
+        
+        if int(n_download) <= 1 and int(n_exist) <= 1:
+            try:
+                return next(reporter.output)
+            except StopIteration:
+                return None
+        return reporter.output
+
+    def _process_item_json(self, local_path, item_json, reporter, pbar, overwrite=False):
+        """
+        Process a single item JSON for download, saving both the item file and metadata.
+
+        :param local_path: Local path to save files
+        :param item_json: Item JSON metadata
+        :param reporter: Reporter instance for tracking progress
+        :param pbar: Progress bar instance
+        :param overwrite: Whether to overwrite existing files
+        :return: Error message, traceback, and downloaded filepath
+        """
+        err = None
+        trace = None
+        downloaded_filepath = None
+        item_id = item_json['id']
+        filename = item_json['filename'].lstrip('/')
+        
+        for i_try in range(NUM_TRIES):
+            try:
+                # Download the image
+                image_path = Path(local_path) / 'items' / filename
+                # Ensure the directory for the image file exists (in case filename has subdirectories)
+                image_path.parent.mkdir(parents=True, exist_ok=True)
+                item = entities.Item.from_json(_json = item_json, client_api=self.items_repository._client_api, is_fetched=False)
+                downloaded_data = self.__thread_download(
+                    item=item,
+                    local_path=str(image_path.parent),
+                    local_filepath=str(image_path),
+                    save_locally=True,
+                    to_array=False,
+                    overwrite=overwrite,
+                    annotation_options=[],
+                    annotation_filters=None,
+                )
+
+                if downloaded_data is None:
+                    err = 'Failed to download image'
+                    trace = ''
+                else:
+                    # Save the item JSON directly
+                    json_filename = Path(filename).stem + '.json'
+                    json_path = Path(local_path) / 'json' / Path(filename).parent / json_filename
+
+                    # Ensure the directory for the JSON file exists (in case filename has subdirectories)
+                    json_path.parent.mkdir(parents=True, exist_ok=True)
+
+                    # Save the original item_json directly
+                    with open(json_path, 'w', encoding='utf-8') as f:
+                        json.dump(item_json, f, indent=2, ensure_ascii=False)
+
+                    downloaded_filepath = str(image_path)
+                
+                if downloaded_filepath is not None:
+                    break
+
+            except Exception as e:
+                logger.debug(f"Download item: {filename}. Try {i_try + 1}/{NUM_TRIES}. Fail.")
+                err = e
+                trace = traceback.format_exc()
+
+        pbar.update()
+        if downloaded_filepath is None:
+            if err is None:
+                err = self.items_repository._client_api.platform_exception
+            reporter.set_index(status="error", ref=item_id, success=False, error=f"{err}\n{trace}")
+        else:
+            reporter.set_index(ref=item_id, status="download", output=downloaded_filepath, success=True)
+
+    def _download_recursive(
+        self,
+        local_path=None,
+        filters: entities.Filters = None,
+        annotation_filters: entities.Filters = None,
+        file_types=None,
+        overwrite=False,
+        raise_on_error=False,
+        dataset_lock=False,
+        lock_timeout_sec=None,
+    ):
+        """
+        Download items recursively from a dataset.
+        
+        :param local_path: Local path to save downloaded items
+        :param filters: Filters entity to filter items
+        :param annotation_filters: Filters entity to filter annotations
+        :param file_types: List of file types to download
+        :param overwrite: Whether to overwrite existing files
+        :param raise_on_error: Raise error if download fails
+        :param dataset_lock: Lock dataset during download
+        :param lock_timeout_sec: Lock timeout in seconds
+        """
+        filters, annotation_filters = self._prepare_filters(filters=filters,annotation_filters=annotation_filters,file_types=file_types)
+        filter_copy = copy.deepcopy(filters)
+        filter_copy.page_size = 0
+        num_items = self.items_repository.list(filters=filter_copy).items_count
+        if num_items == 0:
+            return list()
+        client_api = self.items_repository._client_api
+        reporter = Reporter(
+            num_workers=num_items,
+            resource=Reporter.ITEMS_DOWNLOAD,
+            print_error_logs=client_api.verbose.print_error_logs,
+            client_api=client_api,
+        )
+
+        # Create directories once using pathlib
+        local_path_obj = Path(local_path)
+        items_dir = local_path_obj / 'items'
+        jsons_dir = local_path_obj / 'json'
+        items_dir.mkdir(parents=True, exist_ok=True)
+        jsons_dir.mkdir(parents=True, exist_ok=True)
+
+        jobs = [None for _ in range(num_items)]
+        # crrently keep the thread count to default.
+        # client_api._thread_pools_names['item.download'] = 5 * multiprocessing.cpu_count()
+        pool = client_api.thread_pools(pool_name='item.download')
+        pbar = tqdm.tqdm(
+            total=num_items,
+            disable=client_api.verbose.disable_progress_bar_download_dataset,
+            file=sys.stdout,
+            desc='Download Items',
+        )
+        try:
+            i_item = 0
+            import time
+            start_time = time.time()
+            for json_file in self.items_repository.dataset.project.datasets._export_recursive(
+                dataset=self.items_repository.dataset,
+                local_path=tempfile.mkdtemp(prefix='download_recursive_jsons_'),
+                max_items_per_subset=DOWNLOAD_MAX_ITEMS_PER_SUBSET,
+                include_annotations=True,
+                filters=filters,
+                annotation_filters=annotation_filters,
+                dataset_lock=dataset_lock,
+                lock_timeout_sec=lock_timeout_sec,
+            ):
+                end_time = time.time()
+                with open(json_file, 'r') as f:
+                    data = json.load(f)
+                for item_json in data:
+                    jobs[i_item] = pool.submit(
+                        self._process_item_json,
+                        **{
+                            "local_path": local_path,
+                            "item_json": item_json,
+                            "reporter": reporter,
+                            "pbar": pbar,
+                            "overwrite": overwrite,
+                        },
+                    )
+                    i_item += 1
+        finally:
+            _ = [j.result() for j in jobs if j is not None]
+            pbar.close()
+        return self._process_download_results(reporter=reporter, raise_on_error=raise_on_error)
+
+    @staticmethod
+    def _prepare_filters(filters: entities.Filters = None,
+                        annotation_filters: entities.Filters = None,
+                        file_types=None):
+        """
+        Prepare and merge filters with annotation filters.
+
+        :param filters: Filters entity or None
+        :param annotation_filters: Annotation filters to merge with item filters
+        :param file_types: List of file types to filter
+        :return: Prepared filters entity
+        """
+        # filters
+        if filters is None:
+            filters = entities.Filters()
+            filters._user_query = 'false'
+        # file types
+        if file_types is not None:
+            filters.add(field='metadata.system.mimetype', values=file_types, operator=entities.FiltersOperations.IN)
+        if annotation_filters is not None:
+            if len(annotation_filters.and_filter_list) > 0 or len(annotation_filters.or_filter_list) > 0:
+                for annotation_filter_and in annotation_filters.and_filter_list:
+                    filters.add_join(field=annotation_filter_and.field,
+                                    values=annotation_filter_and.values,
+                                    operator=annotation_filter_and.operator,
+                                    method=entities.FiltersMethod.AND)
+                for annotation_filter_or in annotation_filters.or_filter_list:
+                    filters.add_join(field=annotation_filter_or.field,
+                                    values=annotation_filter_or.values,
+                                    operator=annotation_filter_or.operator,
+                                    method=entities.FiltersMethod.OR)
+            elif annotation_filters.custom_filter is not None:
+                annotation_query_dict = annotation_filters.prepare()
+                items_query_dict = filters.prepare()
+                items_query_dict["join"] = annotation_query_dict
+                filters.reset()
+                filters.custom_filter = items_query_dict
+
+        else:
+            annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
+            filters._user_query = 'false'
+
+        return filters, annotation_filters
+
     def download(self,
                  # filter options
                  filters: entities.Filters = None,
@@ -131,35 +380,12 @@ class Downloader:
             items_to_download = [items]
             num_items = len(items)
         else:
-            # filters
-            if filters is None:
-                filters = entities.Filters()
-                filters._user_query = 'false'
-            # file types
-            if file_types is not None:
-                filters.add(field='metadata.system.mimetype', values=file_types, operator=entities.FiltersOperations.IN)
-            if annotation_filters is not None:
-                if len(annotation_filters.and_filter_list) > 0 or len(annotation_filters.or_filter_list) > 0:
-                    for annotation_filter_and in annotation_filters.and_filter_list:
-                        filters.add_join(field=annotation_filter_and.field,
-                                        values=annotation_filter_and.values,
-                                        operator=annotation_filter_and.operator,
-                                        method=entities.FiltersMethod.AND)
-                    for annotation_filter_or in annotation_filters.or_filter_list:
-                        filters.add_join(field=annotation_filter_or.field,
-                                        values=annotation_filter_or.values,
-                                        operator=annotation_filter_or.operator,
-                                        method=entities.FiltersMethod.OR)
-                elif annotation_filters.custom_filter is not None:
-                    annotation_query_dict = annotation_filters.prepare()
-                    items_query_dict = filters.prepare()
-                    items_query_dict["join"] = annotation_query_dict
-                    filters.reset()
-                    filters.custom_filter = items_query_dict
-
-            else:
-                annotation_filters = entities.Filters(resource=entities.FiltersResource.ANNOTATION)
-                filters._user_query = 'false'
+            # Prepare and merge filters
+            filters, annotation_filters = self._prepare_filters(
+                filters=filters,
+                annotation_filters=annotation_filters,
+                file_types=file_types
+            )
 
             items_to_download = self.items_repository.list(filters=filters)
             num_items = items_to_download.items_count
@@ -234,7 +460,8 @@ class Downloader:
         # pool
         pool = client_api.thread_pools(pool_name='item.download')
         # download
-        pbar = tqdm.tqdm(total=num_items, disable=client_api.verbose.disable_progress_bar_download_dataset, file=sys.stdout,
+        pbar = tqdm.tqdm(total=num_items, disable=client_api.verbose.disable_progress_bar_download_dataset,
+                         file=sys.stdout,
                          desc='Download Items')
         try:
             i_item = 0
@@ -305,41 +532,8 @@ class Downloader:
         finally:
             _ = [j.result() for j in jobs if j is not None]
             pbar.close()
-        # reporting
-        n_download = reporter.status_count(status='download')
-        n_exist = reporter.status_count(status='exist')
-        n_error = reporter.status_count(status='error')
-        logger.info("Number of files downloaded:{}".format(n_download))
-        logger.info("Number of files exists: {}".format(n_exist))
-        logger.info("Total number of files: {}".format(n_download + n_exist))
 
-        # log error
-        if n_error > 0:
-            log_filepath = reporter.generate_log_files()
-            # Get up to 5 error examples for the exception message
-            error_text = ""
-            error_counter = 0
-            if reporter._errors:
-                for _id, error in reporter._errors.items():
-                    error_counter += 1
-                    error_text += f"Item ID: {_id}, Error: {error} | "
-                    if error_counter >= 5:
-                        break
-            error_message = f"Errors in {n_error} files. Errors: {error_text}"
-            if log_filepath is not None:
-                error_message += f", see {log_filepath} for full log"
-            if raise_on_error is True:
-                raise PlatformException(
-                    error="400", message=error_message
-                )
-            else:
-                logger.warning(error_message)
-        if int(n_download) <= 1 and int(n_exist) <= 1:
-            try:
-                return next(reporter.output)
-            except StopIteration:
-                return None
-        return reporter.output
+        return self._process_download_results(reporter=reporter, raise_on_error=raise_on_error)
 
     def __thread_download_wrapper(self, i_item,
                                   # item params
@@ -403,7 +597,7 @@ class Downloader:
                              export_version=entities.ExportVersion.V1,
                              dataset_lock=False,
                              lock_timeout_sec=None,
-                             export_summary=False                         
+                             export_summary=False
                              ):
         """
         Download annotations json for entire dataset
@@ -633,27 +827,12 @@ class Downloader:
     @staticmethod
     def __get_link_source(item):
         assert isinstance(item, entities.Item)
-        if not item.is_fetched:
-            return item, '', False
-
-        if not item.filename.endswith('.json') or \
-                item.metadata.get('system', {}).get('shebang', {}).get('dltype', '') != 'link':
-            return item, '', False
-
-        # recursively get next id link item
-        while item.filename.endswith('.json') and \
-                item.metadata.get('system', {}).get('shebang', {}).get('dltype', '') == 'link' and \
-                item.metadata.get('system', {}).get('shebang', {}).get('linkInfo', {}).get('type', '') == 'id':
-            item = item.dataset.items.get(item_id=item.metadata['system']['shebang']['linkInfo']['ref'])
-
-        # check if link
-        if item.filename.endswith('.json') and \
-                item.metadata.get('system', {}).get('shebang', {}).get('dltype', '') == 'link' and \
-                item.metadata.get('system', {}).get('shebang', {}).get('linkInfo', {}).get('type', '') == 'url':
-            url = item.metadata['system']['shebang']['linkInfo']['ref']
-            return item, url, True
-        else:
-            return item, '', False
+        is_url = False
+        url = item.resolved_stream
+        if item.metadata.get('system', {}).get('shebang', {}).get('linkInfo', {}).get('type', '') == 'url':
+            is_url = True
+
+        return item, url, is_url, url.startswith('file://')
 
     def __file_validation(self, item, downloaded_file):
         res = False
@@ -688,7 +867,7 @@ class Downloader:
         """
         Get a single item's binary data
         Calling this method will returns the item body itself , an image for example with the proper mimetype.
-
+  
         :param item: Item entity to download
         :param save_locally: bool. save to file or return buffer
         :param local_path: item local folder to save to.
@@ -709,8 +888,7 @@ class Downloader:
         if save_locally and os.path.isfile(local_filepath):
             need_to_download = overwrite
 
-        item, url, is_url = self.__get_link_source(item=item)
-        is_local_link = isinstance(url, str) and url.startswith('file://')
+        item, url, is_url, is_local_link = self.__get_link_source(item=item)
 
         # save as byte stream
         data = io.BytesIO()
@@ -804,9 +982,11 @@ class Downloader:
 
                             file_validation = True
                             if not is_url:
-                                file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
-                                                                                                  download_progress=temp_file_path,
-                                                                                                  chunk_resume=chunk_resume)
+                                file_validation, start_point, chunk_resume = self.__get_next_chunk(
+                                    item=item,
+                                    download_progress=temp_file_path,
+                                    chunk_resume=chunk_resume
+                                )
                             if file_validation:
                                 shutil.move(temp_file_path, local_filepath)
                                 download_done = True
@@ -933,6 +1113,7 @@ class Downloader:
         """
         :param url:
         """
+        response = None
 
         if url.startswith('file://'):
             parsed = urlparse(url)
@@ -953,24 +1134,24 @@ class Downloader:
                 )
 
             try:
-                return io.BufferedReader(io.FileIO(path, 'rb'))
+                response = io.BufferedReader(io.FileIO(path, 'rb'))
             except PermissionError as e:
                 raise PlatformException(
                     error='403',
                     message=f'Permission denied accessing file: {url}'
                 ) from e
-
-        prepared_request = requests.Request(method='GET', url=url).prepare()
-        with requests.Session() as s:
-            retry = Retry(
-                total=3,
-                read=3,
-                connect=3,
-                backoff_factor=1,
-            )
-            adapter = HTTPAdapter(max_retries=retry)
-            s.mount('http://', adapter)
-            s.mount('https://', adapter)
-            response = s.send(request=prepared_request, stream=True)
+        else:
+            prepared_request = requests.Request(method='GET', url=url).prepare()
+            with requests.Session() as s:
+                retry = Retry(
+                    total=3,
+                    read=3,
+                    connect=3,
+                    backoff_factor=1,
+                )
+                adapter = HTTPAdapter(max_retries=retry)
+                s.mount('http://', adapter)
+                s.mount('https://', adapter)
+                response = s.send(request=prepared_request, stream=True)
 
         return response

dtlpy/repositories/feature_sets.py

@@ -200,7 +200,7 @@ class FeatureSets:
         """
 
         success, response = self._client_api.gen_request(req_type="delete",
-                                                         path="{}/{}".format(self.URL, feature_set_id))
+                                                         path=f"{self.URL}/{feature_set_id}")
 
         # check response
         if success:
@@ -209,6 +209,36 @@ class FeatureSets:
         else:
             raise exceptions.PlatformException(response)
 
+    @_api_reference.add(path='/features/set/{id}', method='patch')
+    def update(self, feature_set: entities.FeatureSet) -> entities.FeatureSet:
+        """
+        Update a Feature Set
+
+        **Prerequisites**: You must be in the role of an *owner* or *developer*.
+
+       :param dtlpy.entities.FeatureSet feature_set: FeatureSet object
+       :return: FeatureSet
+       :rtype: dtlpy.entities.FeatureSet
+
+       **Example**:
+
+        .. code-block:: python
+
+            dl.feature_sets.update(feature_set='feature_set')
+       """
+        success, response = self._client_api.gen_request(req_type="patch",
+                                                         path=f"{self.URL}/{feature_set.id}",
+                                                         json_req=feature_set.to_json())
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        logger.debug("feature_set updated successfully")
+        # update dataset labels
+        feature_set = entities.FeatureSet.from_json(_json=response.json(),
+                                                    client_api=self._client_api,
+                                                    is_fetched=True)
+        return feature_set
+
     def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Item]:
         pool = self._client_api.thread_pools(pool_name='entity.create')
         jobs = [None for _ in range(len(response_items))]

dtlpy/repositories/recipes.py

@@ -42,10 +42,6 @@ class Recipes:
     @property
     def project(self) -> entities.Project:
         if self._project is None:
-            project = self._client_api.state_io.get('project')
-            if project is not None:
-                self._project = entities.Project.from_json(_json=project, client_api=self._client_api)
-                self._project_id = self._project.id
             if self._project_id is None:
                 if self._dataset is None:
                     raise exceptions.PlatformException(
@@ -122,7 +118,21 @@ class Recipes:
         if attributes is None:
             attributes = list()
         if project_ids is None:
-            project_ids = [self.project.id]
+            if self._dataset is not None:
+                project_ids = [self._dataset.project.id]
+            else:
+                # get from cache
+                project = self._client_api.state_io.get('project')
+                if project is not None:
+                    # build entity from json
+                    p = entities.Project.from_json(_json=project, client_api=self._client_api)
+                    project_ids = [p.id]
+                else:
+                    # get from self.project property
+                    try:
+                        project_ids = [self.project.id]
+                    except exceptions.PlatformException:
+                        raise exceptions.PlatformException('Must provide project_ids')
         if ontology_ids is None:
             ontolgies = repositories.Ontologies(client_api=self._client_api,
                                                 recipe=None)

dtlpy/services/api_client.py

@@ -830,10 +830,11 @@ class ApiClient:
         environments = self.environments
         if environment in environments:
             logger.warning('Environment exists. Overwriting. env: {}'.format(environment))
-        if token is None:
-            token = None
-        if alias is None:
-            alias = None
+        if alias is not None:
+            keys_to_remove = [e for e, v in environments.items() if v.get('alias') == alias]
+            for e in keys_to_remove:
+                logger.warning('Alias exists. Overwriting. alias: {}'.format(alias))
+                environments.pop(e)
         environments[environment] = {'audience': audience,
                                      'client_id': client_id,
                                      'auth0_url': auth0_url,

{dtlpy-1.116.6.dist-info → dtlpy-1.117.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dtlpy
-Version: 1.116.6
+Version: 1.117.6
 Summary: SDK and CLI for Dataloop platform
 Home-page: https://github.com/dataloop-ai/dtlpy
 Author: Dataloop Team

{dtlpy-1.116.6.dist-info → dtlpy-1.117.6.dist-info}/RECORD

@@ -1,5 +1,5 @@
-dtlpy/__init__.py,sha256=59ZkyPzyOQJaA9O8ZpyyFAWvof8OxQO2IrmoEN4QA2Y,20593
-dtlpy/__version__.py,sha256=qhqfKmFRYTm7tEnTLeiyuBcjOerPrhpw7PACo1jXzlE,20
+dtlpy/__init__.py,sha256=p9qDLEQ01cTj0Xx8FkxJPHSmP7LMHMv8b5-ZWW1FPrU,20625
+dtlpy/__version__.py,sha256=EAXt30NxCM3wPLWcW6lyXBUATt56UfSsHHucdFvuHBo,20
 dtlpy/exceptions.py,sha256=EQCKs3pwhwZhgMByQN3D3LpWpdxwcKPEEt-bIaDwURM,2871
 dtlpy/new_instance.py,sha256=XegQav2hzPrPAUzuRFvUIGSPidoK-rbb02Q43NPsIpo,9982
 dtlpy/assets/__init__.py,sha256=D_hAa6NM8Zoy32sF_9b7m0b7I-BQEyBFg8-9Tg2WOeo,976
@@ -43,7 +43,7 @@ dtlpy/dlp/dlp,sha256=-F0vSCWuSOOtgERAtsPMPyMmzitjhB7Yeftg_PDlDjw,10
 dtlpy/dlp/dlp.bat,sha256=QOvx8Dlx5dUbCTMpwbhOcAIXL1IWmgVRSboQqDhIn3A,37
 dtlpy/dlp/dlp.py,sha256=Zv9yoXwNAx4gkED-JiayN-ZkX2dPn4FB0SDx9qc7muo,4404
 dtlpy/dlp/parser.py,sha256=p-TFaiAU2c3QkI97TXzL2LDR3Eq0hGDFrTc9J2jWLh4,30551
-dtlpy/entities/__init__.py,sha256=rAA5GbNpbwa37vdKIe9FtpvIOI1rEHYwj0Da_Mv4Xn4,5018
+dtlpy/entities/__init__.py,sha256=tHndgUHZOaiXJdOY-NTvGoHcJ29EkGzz389sWQPWyaI,5050
 dtlpy/entities/analytic.py,sha256=6bMG4FD7VIDr3ca8hJ25yd9IyAuzPdaN1lsV4VD7z2Q,13739
 dtlpy/entities/annotation.py,sha256=0bF-N3ApbUaTWa_cIPNHMGxaWGG0q3lQos6fMDX5mCc,66661
 dtlpy/entities/annotation_collection.py,sha256=CEYSBHhhDkC0VJdHsBSrA6TgdKGMcKeI3tFM40UJwS8,29838
@@ -63,17 +63,17 @@ dtlpy/entities/dpk.py,sha256=XrK8X8p4Ag6LMjDrDpMstY-h_yTll_sMmKTZT6bLbWE,17923
 dtlpy/entities/driver.py,sha256=usyWqlM8KxQ-L9uCTSAmPXCu2dU4fordAN8uL0I2wRM,7720
 dtlpy/entities/execution.py,sha256=uQe535w9OcAoDiNWf96KcpFzUDEUU-DYsUalv5VziyM,13673
 dtlpy/entities/feature.py,sha256=9fFjD0W57anOVSAVU55ypxN_WTCsWTG03Wkc3cAAj78,3732
-dtlpy/entities/feature_set.py,sha256=niw4MkmrDbD_LWQu1X30uE6U4DCzmFhPTaYeZ6VZDB0,4443
+dtlpy/entities/feature_set.py,sha256=fCBEJVyo91SWz9bw8HczduRfgPJkBsAgeT6PPrYfy5k,4600
 dtlpy/entities/filters.py,sha256=6CK66kkh5LbLjnzKoGe9jsLP8KBfdBRvNumkj-4EtfM,29399
 dtlpy/entities/gis_item.py,sha256=Uk-wMBxwcHsImjz4qOjP-EyZAohbRzN43kMpCaVjCXU,3982
 dtlpy/entities/integration.py,sha256=XraOApW9jbT6EdZraRX2In6sMbfNgEGf2V5Um2RCRqA,6001
-dtlpy/entities/item.py,sha256=iR8qzk-8h5YuQDR4P9KYIEhKOaYZSVgnVMxS32L-3Hg,34884
+dtlpy/entities/item.py,sha256=JoF3vTlsBUXwsC7b5KCkvDOeGP7Iby--Ww4PJQ5_UF4,35586
 dtlpy/entities/label.py,sha256=ycDYavIgKhz806plIX-64c07_TeHpDa-V7LnfFVe4Rg,3869
 dtlpy/entities/links.py,sha256=FAmEwHtsrqKet3c0UHH9u_gHgG6_OwF1-rl4xK7guME,2516
 dtlpy/entities/message.py,sha256=ApJuaKEqxATpXjNYUjGdYPu3ibQzEMo8-LtJ_4xAcPI,5865
 dtlpy/entities/model.py,sha256=gFZ_I6Th4KqkWS5X57aVfEoMAGtAhcPlCOXofo9ARUM,27792
 dtlpy/entities/node.py,sha256=RiCqG659Pb1PZNMewR-F7eNbU5tt713fiZY9xW6-Pes,39199
-dtlpy/entities/ontology.py,sha256=R29baDbx_TKdaUJZKVJPee1_sJ-DLwwlQ2AEjExP2Ko,32530
+dtlpy/entities/ontology.py,sha256=qA8XOhHPiZ7fUK2QPBola2xA8SGEdzsgCy5oozgARwc,32534
 dtlpy/entities/organization.py,sha256=Zm-tTHV82PvYyTNetRRXqvmvzBCbXEwS-gAENf7Zny4,9874
 dtlpy/entities/package.py,sha256=QSDePHlp4ik19aUE3dAUC7edh0oUUVjzSmMG867avc4,26363
 dtlpy/entities/package_defaults.py,sha256=wTD7Z7rGYjVy8AcUxTFEnkOkviiJaLVZYvduiUBKNZo,211
@@ -88,7 +88,7 @@ dtlpy/entities/prompt_item.py,sha256=S_cgekiUsAK0OLP_XXbfvpNv7Zr5XT86jCB2w1yyyjQ
 dtlpy/entities/recipe.py,sha256=SX0T7gw-_9Cs2FZyC_htIxQd7CwDwb2zA3SqB37vymM,11917
 dtlpy/entities/reflect_dict.py,sha256=2NaSAL-CO0T0FYRYFQlaSpbsoLT2Q18AqdHgQSLX5Y4,3273
 dtlpy/entities/resource_execution.py,sha256=1HuVV__U4jAUOtOkWlWImnM3Yts8qxMSAkMA9sBhArY,5033
-dtlpy/entities/service.py,sha256=xG4Rj-kyhJNreesJgq7_Php7vTYztCvFVXifWbCog7s,33479
+dtlpy/entities/service.py,sha256=sIWKx-b9JbxHGmZmgeBv5eArQrI9HB00C1Zmj7ejTc0,33925
 dtlpy/entities/service_driver.py,sha256=N3fL_xTPLu75UBFQZO5Plxx2kpED-UIILxTYbD58rzQ,3917
 dtlpy/entities/setting.py,sha256=uXagJHtcCR3nJYClR_AUGZjz_kx3TejPcUZ8ginHFIA,8561
 dtlpy/entities/task.py,sha256=ajVIkB-3Aqm9Udn87ITvIsGwduyCTtGeqV-FjSYtZKg,19605
@@ -149,7 +149,7 @@ dtlpy/miscellaneous/list_print.py,sha256=fBGTMXFUwDG8DD4W6HyR8BTGtbTckLf4W09quNR
 dtlpy/miscellaneous/zipping.py,sha256=JplTc8UDFvO8WaD5vKuumVLN0lU_-GtHoE0doWKtmKg,5383
 dtlpy/ml/__init__.py,sha256=vPkyXpc9kcWWZ_PxyPEOsjKBJdEbowLkZr8FZIb_OBM,799
 dtlpy/ml/base_feature_extractor_adapter.py,sha256=iiEGYAx0Rdn4K46H_FlKrAv3ebTXHSxNVAmio0BxhaI,1178
-dtlpy/ml/base_model_adapter.py,sha256=YQrO2mG2FiC3WobRUnaLqlq-GJEHvmBSyB3nuQGaU9o,62822
+dtlpy/ml/base_model_adapter.py,sha256=DS5J0vdWlk2W46sDIlFq1xDapgoz8riNhcWm0itJ7QY,64077
 dtlpy/ml/metrics.py,sha256=BG2E-1Mvjv2e2No9mIJKVmvzqBvLqytKcw3hA7wVUNc,20037
 dtlpy/ml/predictions_utils.py,sha256=He_84U14oS2Ss7T_-Zj5GDiBZwS-GjMPURUh7u7DjF8,12484
 dtlpy/ml/summary_writer.py,sha256=dehDi8zmGC1sAGyy_3cpSWGXoGQSiQd7bL_Thoo8yIs,2784
@@ -166,12 +166,12 @@ dtlpy/repositories/collections.py,sha256=lA4T7irQ_dtI_bcVxrHHikClcglp8ltsoPcLwHy
 dtlpy/repositories/commands.py,sha256=aismmQOkZbpo9vN1fVYTHXrsJoQNrN7wiANUCiUWUmY,5717
 dtlpy/repositories/compositions.py,sha256=H417BvlQAiWr5NH2eANFke6CfEO5o7DSvapYpf7v5Hk,2150
 dtlpy/repositories/computes.py,sha256=K1a28vJn3tB-h_Xvl8blr-lOytfz3MvyzFzauCja8UA,16936
-dtlpy/repositories/datasets.py,sha256=W-nuH5oX0PGu3lj3-Z0EJpZy10wt-eEMKp0EZ3N4gYo,69846
-dtlpy/repositories/downloader.py,sha256=_GCYBtGR7Rkd0MlLSt7OFOAIck92kPuXmFJfKo_QCWw,49103
+dtlpy/repositories/datasets.py,sha256=1-ud7Ulc8z5IwQezjrW4C6V24xVUhR0U_dgix76zDYw,72778
+dtlpy/repositories/downloader.py,sha256=EkCscU8QHa6H0-t17laFZn49Up-wNHbFd2DxVIqX8Fc,56209
 dtlpy/repositories/dpks.py,sha256=Cu8pqFZr6MlrdidjysQT_X1hyKaL5YNUn81puOM5FX0,18508
 dtlpy/repositories/drivers.py,sha256=z9qu4I2lwa0aujkKxj0bvn71Zzs8U8byqSx8S9DAIQw,19553
 dtlpy/repositories/executions.py,sha256=BuFv7J6U2Q7om4OlC0q6tnk-1Vcw5m0adfR9a5Mj98Y,32361
-dtlpy/repositories/feature_sets.py,sha256=UowMDAl_CRefRB5oZzubnsjU_OFgiPPdQXn8q2j4Kuw,9666
+dtlpy/repositories/feature_sets.py,sha256=JIn1tFn8I50OPNRjKvEpgdQNcYKIGYIPjm-j-PR7uaQ,10874
 dtlpy/repositories/features.py,sha256=SNmECqKSfHlNgjjG_RlX-GAU3udYN9_ZYOe8mFNy010,10671
 dtlpy/repositories/integrations.py,sha256=Y5c37fQCaIkw1p5jPEbAqytgRVXuqe771eHC1hNDE7A,19491
 dtlpy/repositories/items.py,sha256=u2Vg0jOTZ9EhV1sPJdAeIUyfqBRv63Gl-IXaMlon8PM,40028
@@ -184,7 +184,7 @@ dtlpy/repositories/packages.py,sha256=QhkXMZkpseCt0pDropJuqoHJL0RMa5plk8AN0V3w6N
 dtlpy/repositories/pipeline_executions.py,sha256=sEC03bu5DsHc554z3xDbMCP529rhfADcktXgWkswEwI,17281
 dtlpy/repositories/pipelines.py,sha256=5qosyxLFgNbcmL7uoTr9klAj1VM-7mWvsOvngbUU1Qk,24320
 dtlpy/repositories/projects.py,sha256=TKLCuL7Inlv4GwgcQcuXkPQtgacfrXYjsTQng8nPC7Y,21623
-dtlpy/repositories/recipes.py,sha256=ZtH9s3BBLCpzlx59onqvrw3PvKYCWEM7payQbAY9tgA,16677
+dtlpy/repositories/recipes.py,sha256=q1FMk4kBPzBS-QIbkxeSsMcAJmYuS7gpYL8t3XIBWII,17117
 dtlpy/repositories/resource_executions.py,sha256=PyzsbdJxz6jf17Gx13GZmqdu6tZo3TTVv-DypnJ_sY0,5374
 dtlpy/repositories/schema.py,sha256=kTKDrbwm7BfQnBAK81LpAl9ChNFdyUweSLNazlJJhjk,3953
 dtlpy/repositories/service_drivers.py,sha256=rxbhLUtT7TCbkVxH4HJtFT9ywcikByPFdX7_4kiTiK0,6766
@@ -198,7 +198,7 @@ dtlpy/repositories/uploader.py,sha256=I9mP-Ikj0zUOdMTf-7FN_huHWXYeWc8gzVRpfUPAXj
 dtlpy/repositories/webhooks.py,sha256=IIpxOJ-7KeQp1TY9aJZz-FuycSjAoYx0TDk8z86KAK8,9033
 dtlpy/services/__init__.py,sha256=VfVJy2otIrDra6i7Sepjyez2ujiE6171ChQZp-YgxsM,904
 dtlpy/services/aihttp_retry.py,sha256=tgntZsAY0dW9v08rkjX1T5BLNDdDd8svtgn7nH8DSGU,5022
-dtlpy/services/api_client.py,sha256=vTysmEqXMEyEwAKaQK9JGILkHfrQNJFtZ60WEqQG8QQ,71728
+dtlpy/services/api_client.py,sha256=MbpTsGVgFonv04pWCfYkOP0shM3LcfCOrucVEXvNn4M,71907
 dtlpy/services/api_reference.py,sha256=cW-B3eoi9Xs3AwI87_Kr6GV_E6HPoC73aETFaGz3A-0,1515
 dtlpy/services/async_utils.py,sha256=kaYHTPw0Lg8PeJJq8whPyzrBYkzD7offs5hsKRZXJm8,3960
 dtlpy/services/calls_counter.py,sha256=gr0io5rIsO5-7Cgc8neA1vK8kUtYhgFPmDQ2jXtiZZs,1036
@@ -226,14 +226,14 @@ dtlpy/utilities/reports/report.py,sha256=3nEsNnIWmdPEsd21nN8vMMgaZVcPKn9iawKTTeO
 dtlpy/utilities/videos/__init__.py,sha256=SV3w51vfPuGBxaMeNemx6qEMHw_C4lLpWNGXMvdsKSY,734
 dtlpy/utilities/videos/video_player.py,sha256=LCxg0EZ_DeuwcT7U_r7MRC6Q19s0xdFb7x5Gk39PRms,24072
 dtlpy/utilities/videos/videos.py,sha256=Dj916B4TQRIhI7HZVevl3foFrCsPp0eeWwvGbgX3-_A,21875
-dtlpy-1.116.6.data/scripts/dlp,sha256=-F0vSCWuSOOtgERAtsPMPyMmzitjhB7Yeftg_PDlDjw,10
-dtlpy-1.116.6.data/scripts/dlp.bat,sha256=QOvx8Dlx5dUbCTMpwbhOcAIXL1IWmgVRSboQqDhIn3A,37
-dtlpy-1.116.6.data/scripts/dlp.py,sha256=ZpfJvYE1_OTSorEYBphqTOutnHSb5TqOXh0y_mUCTJs,4393
-dtlpy-1.116.6.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+dtlpy-1.117.6.data/scripts/dlp,sha256=-F0vSCWuSOOtgERAtsPMPyMmzitjhB7Yeftg_PDlDjw,10
+dtlpy-1.117.6.data/scripts/dlp.bat,sha256=QOvx8Dlx5dUbCTMpwbhOcAIXL1IWmgVRSboQqDhIn3A,37
+dtlpy-1.117.6.data/scripts/dlp.py,sha256=ZpfJvYE1_OTSorEYBphqTOutnHSb5TqOXh0y_mUCTJs,4393
+dtlpy-1.117.6.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
 tests/features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/features/environment.py,sha256=FjOLwKvQx44TNvIidvoQaTXxvoEaSFsXQ9xkqkskkdo,18933
-dtlpy-1.116.6.dist-info/METADATA,sha256=a4Z5q1gGBk5e-PnmYqRC6FfPNynDl1N8LqbMm0Tmawo,5835
-dtlpy-1.116.6.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-dtlpy-1.116.6.dist-info/entry_points.txt,sha256=C4PyKthCs_no88HU39eioO68oei64STYXC2ooGZTc4Y,43
-dtlpy-1.116.6.dist-info/top_level.txt,sha256=ZWuLmQGUOtWAdgTf4Fbx884w1o0vBYq9dEc1zLv9Mig,12
-dtlpy-1.116.6.dist-info/RECORD,,
+dtlpy-1.117.6.dist-info/METADATA,sha256=d7p2fvvyIiNWCqPQP7AYTQEBw78AgqxXGr67f7G5o68,5835
+dtlpy-1.117.6.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+dtlpy-1.117.6.dist-info/entry_points.txt,sha256=C4PyKthCs_no88HU39eioO68oei64STYXC2ooGZTc4Y,43
+dtlpy-1.117.6.dist-info/top_level.txt,sha256=ZWuLmQGUOtWAdgTf4Fbx884w1o0vBYq9dEc1zLv9Mig,12
+dtlpy-1.117.6.dist-info/RECORD,,