dtlpy 1.108.7__py3-none-any.whl → 1.109.19__py3-none-any.whl

dtlpy/repositories/computes.py

@@ -2,11 +2,16 @@ import base64
 import datetime
 import json
 
+from dtlpy import miscellaneous
+
 from ..services.api_client import ApiClient
 from .. import exceptions, entities, repositories
 from typing import List, Optional, Dict
-from ..entities import ComputeCluster, ComputeContext, ComputeType, Project
+from ..entities import ComputeCluster, ComputeContext, ComputeType
 from ..entities.integration import IntegrationType
+import logging
+
+logger = logging.getLogger(name='dtlpy')
 
 
 class Computes:
@@ -17,6 +22,7 @@ class Computes:
         self._commands = None
         self._projects = None
         self._organizations = None
+        self.log_cache = dict()
 
     @property
     def commands(self) -> repositories.Commands:
@@ -63,6 +69,7 @@ class Computes:
         :param status: Compute status
         :param settings: Compute settings
         :return: Compute
+        :rtype: dl.entities.compute.Compute
         """
 
         shared_contexts_json = []
@@ -91,26 +98,69 @@ class Computes:
         if not success:
             raise exceptions.PlatformException(response)
 
-        compute = entities.Compute.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
+        compute = self._build_compute_by_type(response.json())
 
         if wait:
             command_id = compute.metadata.get('system', {}).get('commands', {}).get('create', None)
             if command_id is not None:
                 command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
-                command.wait()
+                try:
+                    command.wait(iteration_callback=self.__get_log_compute_progress_callback(compute.id))
+                except Exception as e:
+                    self.log_cache.pop(compute.id, None)
+                    raise e
                 compute = self.get(compute_id=compute.id)
 
         return compute
 
+    def _build_compute_by_type(self, _json):
+        if _json.get('type') == 'kubernetes':
+            compute = entities.KubernetesCompute.from_json(
+                _json=_json,
+                client_api=self._client_api
+            )
+        else:
+            compute = entities.Compute.from_json(
+                _json=_json,
+                client_api=self._client_api
+            )
+        return compute
+
+    def __get_log_compute_progress_callback(self, compute_id: str):
+        def func():
+            compute = self.get(compute_id=compute_id)
+            bootstrap_progress = compute.metadata.get('system', {}).get('bootstrapProcess', {}).get('progress', None)
+            bootstrap_logs = compute.metadata.get('system', {}).get('bootstrapProcess', {}).get('logs', None)
+            validation_progress = compute.metadata.get('system', {}).get('validation', {}).get('progress', None)
+            validation_logs = compute.metadata.get('system', {}).get('validation', {}).get('logs', None)
+            if bootstrap_progress not in [None, 100]:
+                logger.info(f"Bootstrap in progress: {bootstrap_progress}%")
+                last_index = len(self.log_cache.get(compute_id, {}).get('bootstrap', []))
+                new_logs = bootstrap_logs[last_index:]
+                if new_logs:
+                    logger.info("Bootstrap Logs: {}".format('\n'.join(new_logs)))
+                    if compute_id not in self.log_cache:
+                        self.log_cache[compute_id] = {}
+                    self.log_cache[compute_id]['bootstrap'] = bootstrap_logs
+            if validation_progress not in [None, 100]:
+                logger.info(f"Validating created compute. Progress: {validation_progress}%")
+                last_index = len(self.log_cache.get(compute_id, {}).get('validation', []))
+                new_logs = validation_logs[last_index:]
+                if new_logs:
+                    logger.info("Validation Logs: {}".format('\n'.join(new_logs)))
+                    if compute_id not in self.log_cache:
+                        self.log_cache[compute_id] = {}
+                    self.log_cache[compute_id]['validation'] = validation_logs
+        return func
+
+
     def get(self, compute_id: str):
         """
         Get a compute
 
         :param compute_id: Compute ID
         :return: Compute
+        :rtype: dl.entities.compute.Compute
         """
 
         # request
@@ -122,10 +172,7 @@ class Computes:
         if not success:
             raise exceptions.PlatformException(response)
 
-        compute = entities.Compute.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
+        compute = self._build_compute_by_type(response.json())
 
         return compute
 
@@ -135,6 +182,7 @@ class Computes:
 
         :param compute: Compute
         :return: Compute
+        :rtype: dl.entities.compute.Compute
         """
 
         # request
@@ -147,10 +195,7 @@ class Computes:
         if not success:
             raise exceptions.PlatformException(response)
 
-        compute = entities.Compute.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
+        compute = self._build_compute_by_type(response.json())
 
         return compute
 
@@ -172,6 +217,60 @@ class Computes:
 
         return True
 
+    def validate(self, compute_id: str, wait: bool = True):
+        """
+        Validate a compute
+
+        :param str compute_id: Compute ID
+        :param bool wait: Wait for validation
+        :return: Compute
+        :rtype: dl.entities.compute.Compute
+        """
+
+        # request
+        success, response = self._client_api.gen_request(
+            req_type='post',
+            path=self._base_url + '/{}/validate'.format(compute_id)
+        )
+
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        compute = self._build_compute_by_type(response.json())
+
+        if wait:
+            command_id = compute.metadata.get('system', {}).get('commands', {}).get('validate', None)
+            if command_id is not None:
+                command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
+                try:
+                    command.wait(iteration_callback=self.__get_log_compute_progress_callback(compute.id))
+                except Exception as e:
+                    self.log_cache.pop(compute.id, None)
+                    raise e
+                compute = self.get(compute_id=compute.id)
+
+        return compute
+
+    def list_global(self):
+        """
+        List computes
+
+        :return: List of computes
+        :rtype: list[str]
+        """
+
+        # request
+        success, response = self._client_api.gen_request(
+            req_type='get',
+            path=self._base_url + '/globals',
+        )
+
+        if not success:
+            raise exceptions.PlatformException(response)
+
+
+        return response.json()
+
     @staticmethod
     def read_file(file_path):
         try:
@@ -230,121 +329,59 @@ class Computes:
         return compute
 
 
-class ServiceDrivers:
-
-    def __init__(self, client_api: ApiClient):
-        self._client_api = client_api
-        self._base_url = '/serviceDrivers'
-
-    def create(
-            self,
-            name: str,
-            compute_id: str,
-            context: entities.ComputeContext
-    ):
-        """
-        Create a new service driver
-
-        :param name: Service driver name
-        :param compute_id: Compute ID
-        :param context: Compute context
-        :return: Service driver
-
-        """
-
-        payload = {
-            'name': name,
-            'computeId': compute_id,
-            'context': context.to_json()
-        }
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='post',
-            path=self._base_url,
-            json_req=payload
-        )
-
-        if not success:
-            raise exceptions.PlatformException(response)
-
-        service_driver = entities.ServiceDriver.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
-
-        return service_driver
-
-    def get(self, service_driver_id: str):
-        """
-        Get a service driver
-
-        :param service_driver_id: Service driver ID
-        :return: Service driver
-        """
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='get',
-            path=self._base_url + '/{}'.format(service_driver_id)
-        )
-
+    def _list(self, filters: entities.Filters):
+        url = self._base_url + '/query'
+        success, response = self._client_api.gen_request(req_type='POST',
+                                                         path=url,
+                                                         json_req=filters.prepare())
         if not success:
             raise exceptions.PlatformException(response)
 
-        service_driver = entities.ServiceDriver.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
-
-        return service_driver
-
-    def delete(self, service_driver_id: str):
+        return response.json()
+
+    def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Compute]:
+        pool = self._client_api.thread_pools(pool_name='entity.create')
+        jobs = [None for _ in range(len(response_items))]
+        for i_item, item in enumerate(response_items):
+            jobs[i_item] = pool.submit(entities.Compute._protected_from_json,
+                                       **{'client_api': self._client_api,
+                                          '_json': item})
+        results = [j.result() for j in jobs]
+        _ = [logger.warning(r[1]) for r in results if r[0] is False]
+        items = miscellaneous.List([r[1] for r in results if r[0] is True])
+        return items
+
+    def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
         """
-        Delete a service driver
-
-        :param service_driver_id: Service driver ID
-        """
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='delete',
-            path=self._base_url + '/{}'.format(service_driver_id)
-        )
-
-        if not success:
-            raise exceptions.PlatformException(response)
+        List all services drivers
 
-        return True
+        :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
+        :return: Paged entity
+        :rtype: dtlpy.entities.paged_entities.PagedEntities
 
-    def set_default(self, service_driver_id: str, org_id: str, update_existing_services=False):
-        """
-        Set a service driver as default
+        **Example**:
 
-        :param service_driver_id: Compute name
-        :param org_id: Organization ID
-        :param update_existing_services: Update existing services
+        .. code-block:: python
 
-        :return: Service driver
+            services = dl.service_drivers.list()
         """
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='post',
-            path=self._base_url + '/default',
-            json_req={
-                'organizationId': org_id,
-                'updateExistingServices': update_existing_services,
-                'driverName': service_driver_id
-            }
-        )
-
-        if not success:
-            raise exceptions.PlatformException(response)
-
-        service_driver = entities.ServiceDriver.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
-
-        return service_driver
+        # default filters
+        if filters is None:
+            filters = entities.Filters(resource=entities.FiltersResource.COMPUTE)
+
+        if filters.resource != entities.FiltersResource.COMPUTE:
+            raise exceptions.PlatformException(
+                error='400',
+                message='Filters resource must to be FiltersResource.COMPUTE. Got: {!r}'.format(
+                    filters.resource))
+
+        if not isinstance(filters, entities.Filters):
+            raise exceptions.PlatformException('400', 'Unknown filters type')
+
+        paged = entities.PagedEntities(items_repository=self,
+                                       filters=filters,
+                                       page_offset=filters.page,
+                                       page_size=filters.page_size,
+                                       client_api=self._client_api)
+        paged.get_page()
+        return paged

dtlpy/repositories/datasets.py

@@ -128,7 +128,7 @@ class Datasets:
 
     @staticmethod
     def _build_payload(filters, include_feature_vectors, include_annotations,
-                       export_type, annotation_filters, feature_vector_filters, dataset_lock, lock_timeout_sec):
+                       export_type, annotation_filters, feature_vector_filters, dataset_lock, lock_timeout_sec, export_summary):
         valid_list = [e.value for e in entities.ExportType]
         valid_types = ', '.join(valid_list)
         if export_type not in ['json', 'zip']:
@@ -161,6 +161,9 @@ class Datasets:
         if dataset_lock:
             payload['datasetLock'] = dataset_lock
 
+        if export_summary:
+            payload['summary'] = export_summary
+
         if lock_timeout_sec:
             payload['lockTimeoutSec'] = lock_timeout_sec
    
@@ -636,7 +639,8 @@ class Datasets:
                export_type: entities.ExportType = entities.ExportType.JSON,
                timeout: int = 0,
                dataset_lock: bool = False,
-               lock_timeout_sec: int = None):
+               lock_timeout_sec: int = None,
+               export_summary: bool = False):
         """
         Export dataset items and annotations.
 
@@ -654,6 +658,7 @@ class Datasets:
         :param bool include_feature_vectors: Include item feature vectors in the export
         :param bool include_annotations: Include item annotations in the export
         :param bool dataset_lock: Make dataset readonly during the export
+        :param bool export_summary: Get Summary of the dataset export
         :param int lock_timeout_sec: Timeout for locking the dataset during export in seconds
         :param entities.ExportType export_type: Type of export ('json' or 'zip')
         :param int timeout: Maximum time in seconds to wait for the export to complete
@@ -669,12 +674,14 @@ class Datasets:
                                                   include_feature_vectors=True,
                                                   include_annotations=True,
                                                   export_type=dl.ExportType.JSON,
-                                                  dataset_lock=True
-                                                  lock_timeout_sec=300)
+                                                  dataset_lock=True,
+                                                  lock_timeout_sec=300,
+                                                  export_summary=False)
         """
         dataset_id = self._resolve_dataset_id(dataset, dataset_name, dataset_id)
         payload = self._build_payload(filters, include_feature_vectors, include_annotations,
-                                       export_type, annotation_filters, feature_vector_filters, dataset_lock, lock_timeout_sec)
+                                       export_type, annotation_filters, feature_vector_filters, 
+                                       dataset_lock, lock_timeout_sec, export_summary)
 
         success, response = self._client_api.gen_request(req_type='post', path=f'/datasets/{dataset_id}/export',
                                                          json_req=payload)
@@ -940,7 +947,8 @@ class Datasets:
                              alpha: float = None,
                              export_version=entities.ExportVersion.V1,
                              dataset_lock: bool = False, 
-                             lock_timeout_sec: int = None                
+                             lock_timeout_sec: int = None,
+                             export_summary: bool = False,              
                              ) -> str:
         """
         Download dataset's annotations by filters.
@@ -968,6 +976,7 @@ class Datasets:
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
         :return: local_path of the directory where all the downloaded item
         :param bool dataset_lock: optional - default = False
+        :param bool export_summary: optional - default = False
         :param int lock_timeout_sec: optional
         :rtype: str
 
@@ -982,8 +991,9 @@ class Datasets:
                                                  thickness=1,
                                                  with_text=False,
                                                  alpha=1,
-                                                 dataset_lock=False
-                                                 lock_timeout_sec=300                                                 
+                                                 dataset_lock=False,
+                                                 lock_timeout_sec=300,
+                                                 export_summary=False                                               
                                                  )
         """
         if annotation_options is None:
@@ -1045,7 +1055,8 @@ class Datasets:
                                         filter_output_annotations=filter_output_annotations,
                                         export_version=export_version,
                                         dataset_lock=dataset_lock,
-                                        lock_timeout_sec=lock_timeout_sec                                        
+                                        lock_timeout_sec=lock_timeout_sec,
+                                        export_summary=export_summary                                      
                                         )
         if annotation_options:
             pages = dataset.items.list(filters=filters)

dtlpy/repositories/downloader.py

@@ -48,7 +48,8 @@ class Downloader:
                  alpha=1,
                  export_version=entities.ExportVersion.V1,
                  dataset_lock=False,
-                 lock_timeout_sec=None
+                 lock_timeout_sec=None,
+                 export_summary=False
                  ):
         """
         Download dataset by filters.
@@ -75,6 +76,7 @@ class Downloader:
         :param alpha: opacity value [0 1], default 1
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
         :param bool dataset_lock: optional - default = False
+        :param bool export_summary: optional - default = False
         :param int lock_timeout_sec: optional
         :return: Output (list)
         """
@@ -201,7 +203,8 @@ class Downloader:
                 'filter_output_annotations': filter_output_annotations,
                 'export_version': export_version,
                 'dataset_lock': dataset_lock,
-                'lock_timeout_sec': lock_timeout_sec
+                'lock_timeout_sec': lock_timeout_sec,
+                'export_summary': export_summary
             })
         ###############
         # downloading #
@@ -369,7 +372,8 @@ class Downloader:
                              filter_output_annotations=False,
                              export_version=entities.ExportVersion.V1,
                              dataset_lock=False,
-                             lock_timeout_sec=None                          
+                             lock_timeout_sec=None,
+                             export_summary=False                         
                              ):
         """
         Download annotations json for entire dataset
@@ -384,6 +388,7 @@ class Downloader:
         :param filter_output_annotations: default - False, given an export by filter - determine if to filter out annotations
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
         :param bool dataset_lock: optional - default = False
+        :param bool export_summary: optional - default = False
         :param int lock_timeout_sec: optional
         :return:
         """
@@ -409,6 +414,9 @@ class Downloader:
                 payload['annotations']['filter'] = filter_output_annotations
             if dataset_lock:
                 payload['datasetLock'] = dataset_lock
+
+            if export_summary:
+                payload['summary'] = export_summary
                 
             if lock_timeout_sec:
                 payload['lockTimeoutSec'] = lock_timeout_sec
@@ -694,7 +702,8 @@ class Downloader:
                         raise PlatformException(response)
                 else:
                     _, ext = os.path.splitext(item.metadata['system']['shebang']['linkInfo']['ref'].split('?')[0])
-                    local_filepath += ext
+                    if local_filepath:
+                        local_filepath += ext
                     response = self.get_url_stream(url=url)
 
                 if save_locally:
@@ -791,9 +800,12 @@ class Downloader:
                             for chunk in response.iter_content(chunk_size=chunk_size):
                                 if chunk:  # filter out keep-alive new chunks
                                     data.write(chunk)
-                            file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
-                                                                                              download_progress=data,
-                                                                                              chunk_resume=chunk_resume)
+
+                            file_validation = True
+                            if not is_url:
+                                file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
+                                                                                                   download_progress=data,
+                                                                                                   chunk_resume=chunk_resume)
                             if file_validation:
                                 download_done = True
                             else:
@@ -804,7 +816,7 @@ class Downloader:
                     data.seek(0)
                     data.name = item.name
                     if not save_locally and to_array:
-                        if 'image' not in item.mimetype:
+                        if 'image' not in item.mimetype and not is_url:
                             raise PlatformException(
                                 error="400",
                                 message='Download element type numpy.ndarray support for image only. '

dtlpy/repositories/dpks.py

@@ -248,7 +248,7 @@ class Dpks:
             logger.warning("the project id that provide different from the dpk project id")
 
         if local_path is None:
-            if manifest_filepath=='dataloop.json':
+            if manifest_filepath == 'dataloop.json':
                 local_path = os.getcwd()
             else:
                 local_path = os.path.dirname(manifest_filepath)
@@ -407,3 +407,28 @@ class Dpks:
             dpk = self.__get_by_name(dpk_name=dpk_name, dpk_version=dpk_version)
 
         return dpk
+
+    def get_previews(self, dpk: entities.Dpk):
+        """
+        Get the preview of a specific dpk.
+
+        :param entities.Dpk dpk: the dpk entity to get the preview for.
+        :return the preview of the dpk's templates
+        :rtype dict
+
+        ** Example **
+        ..coed-block:: python
+            res = dl.dpks.get_previews(dpk=dpk)
+        """
+        url = '/app-registry/{}/previews'.format(dpk.id)
+
+        # request
+        success, response = self._client_api.gen_request(
+            req_type='get',
+            path=url
+        )
+
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        return response.json()

dtlpy/repositories/items.py

@@ -529,7 +529,8 @@ class Items:
             alpha: float = 1,
             export_version=entities.ExportVersion.V1,
             dataset_lock: bool = False,
-            lock_timeout_sec: int = None     
+            lock_timeout_sec: int = None,
+            export_summary: bool = False,    
     ):
         """
         Download dataset items by filters.
@@ -550,6 +551,7 @@ class Items:
         :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for download
         :param bool overwrite: optional - default = False
         :param bool dataset_lock: optional - default = False
+        :param bool export_summary: optional - default = False
         :param int lock_timeout_sec: optional
         :param bool to_items_folder: Create 'items' folder and download items to it
         :param int thickness: optional - line thickness, if -1 annotation will be filled, default =1
@@ -599,7 +601,8 @@ class Items:
             filter_output_annotations=filter_output_annotations,
             export_version=export_version,
             dataset_lock=dataset_lock,
-            lock_timeout_sec=lock_timeout_sec
+            lock_timeout_sec=lock_timeout_sec,
+            export_summary=export_summary
         )
 
     def upload(