dtlpy 1.105.6__py3-none-any.whl → 1.107.8__py3-none-any.whl

dtlpy/repositories/downloader.py

@@ -46,7 +46,9 @@ class Downloader:
                  export_png_files=False,
                  filter_output_annotations=False,
                  alpha=1,
-                 export_version=entities.ExportVersion.V1
+                 export_version=entities.ExportVersion.V1,
+                 dataset_lock=False,
+                 lock_timeout_sec=None
                  ):
         """
         Download dataset by filters.
@@ -72,6 +74,8 @@ class Downloader:
         :param filter_output_annotations: default - False, given an export by filter - determine if to filter out annotations
         :param alpha: opacity value [0 1], default 1
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
+        :param bool dataset_lock: optional - default = False
+        :param int lock_timeout_sec: optional
         :return: Output (list)
         """
 
@@ -195,7 +199,9 @@ class Downloader:
                 'include_annotations_in_output': include_annotations_in_output,
                 'export_png_files': export_png_files,
                 'filter_output_annotations': filter_output_annotations,
-                'export_version': export_version
+                'export_version': export_version,
+                'dataset_lock': dataset_lock,
+                'lock_timeout_sec': lock_timeout_sec
             })
         ###############
         # downloading #
@@ -361,7 +367,9 @@ class Downloader:
                              include_annotations_in_output=True,
                              export_png_files=False,
                              filter_output_annotations=False,
-                             export_version=entities.ExportVersion.V1
+                             export_version=entities.ExportVersion.V1,
+                             dataset_lock=False,
+                             lock_timeout_sec=None                          
                              ):
         """
         Download annotations json for entire dataset
@@ -375,6 +383,8 @@ class Downloader:
         :param export_png_files: default - if True, semantic annotations should be exported as png files
         :param filter_output_annotations: default - False, given an export by filter - determine if to filter out annotations
         :param str export_version:  exported items will have original extension in filename, `V1` - no original extension in filenames
+        :param bool dataset_lock: optional - default = False
+        :param int lock_timeout_sec: optional
         :return:
         """
         local_path = os.path.join(local_path, "json")
@@ -397,6 +407,11 @@ class Downloader:
             if annotation_filters is not None:
                 payload['annotationsQuery'] = annotation_filters.prepare()
                 payload['annotations']['filter'] = filter_output_annotations
+            if dataset_lock:
+                payload['datasetLock'] = dataset_lock
+                
+            if lock_timeout_sec:
+                payload['lockTimeoutSec'] = lock_timeout_sec
 
             success, response = dataset._client_api.gen_request(req_type='post',
                                                                 path='/datasets/{}/export'.format(dataset.id),
@@ -730,19 +745,15 @@ class Downloader:
                                                 one_file_pbar.update(len(chunk))
                                 except Exception as err:
                                     pass
-                            file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
-                                                                                              download_progress=temp_file_path,
-                                                                                              chunk_resume=chunk_resume)
+
+                            file_validation = True
+                            if not is_url:
+                                file_validation, start_point, chunk_resume = self.__get_next_chunk(item=item,
+                                                                                                  download_progress=temp_file_path,
+                                                                                                  chunk_resume=chunk_resume)
                             if file_validation:
                                 shutil.move(temp_file_path, local_filepath)
                                 download_done = True
-                            else:
-                                if not is_url:
-                                    continue
-                                else:
-                                    raise PlatformException(
-                                        error="400",
-                                        message='Downloaded file is corrupted. Please try again. If the issue repeats please contact support.')
                         except Exception as err:
                             if os.path.isfile(temp_file_path):
                                 os.remove(temp_file_path)

dtlpy/repositories/integrations.py

@@ -55,12 +55,15 @@ class Integrations:
     def delete(self,
                integrations_id: str,
                sure: bool = False,
-               really: bool = False) -> bool:
+               really: bool = False,
+               organization_id: str = None
+               ) -> bool:
         """
         Delete integrations from the organization.
 
         **Prerequisites**: You must be an organization *owner* to delete an integration.
 
+        :param organization_id: organization id
         :param str integrations_id: integrations id
         :param bool sure: Are you sure you want to delete?
         :param bool really: Really really sure?
@@ -74,11 +77,12 @@ class Integrations:
             project.integrations.delete(integrations_id='integrations_id', sure=True, really=True)
         """
         if sure and really:
-            if self.project is None and self.org is None:
+            if self.project is None and self.org is None and organization_id is None:
                 raise exceptions.PlatformException(
                     error='400',
                     message='Must provide an identifier in inputs')
 
+        if organization_id is None:
             if self.project is not None:
                 organization_id = self.project.org.get('id')
             else:
@@ -101,7 +105,9 @@ class Integrations:
                integrations_type: entities.IntegrationType,
                name: str,
                options: dict,
-               metadata: dict = None):
+               metadata: dict = None,
+               organization_id: str = None,
+               ):
         """
         Create an integration between an external storage and the organization.
 
@@ -123,6 +129,7 @@ class Integrations:
         :param str name: integrations name
         :param dict options: dict of storage secrets
         :param dict metadata: metadata
+        :param str organization_id: organization id
         :return: success
         :rtype: bool
 
@@ -135,15 +142,16 @@ class Integrations:
                             options={key: "Access key ID", secret: "Secret access key"})
         """
 
-        if self.project is None and self.org is None:
+        if self.project is None and self.org is None and organization_id is None:
             raise exceptions.PlatformException(
                 error='400',
                 message='Must have an organization or project')
 
-        if self.project is not None:
-            organization_id = self.project.org.get('id')
-        else:
-            organization_id = self.org.id
+        if organization_id is None:
+            if self.project is not None:
+                organization_id = self.project.org.get('id')
+            else:
+                organization_id = self.org.id
 
         url_path = '/orgs/{}/integrations'.format(organization_id)
         payload = {"type": integrations_type.value if isinstance(integrations_type,
@@ -170,7 +178,9 @@ class Integrations:
                new_name: str = None,
                integrations_id: str = None,
                integration: entities.Integration = None,
-               new_options: dict = None):
+               new_options: dict = None,
+               organization_id: str = None,
+               ):
         """
         Update the integration's name.
 
@@ -180,6 +190,7 @@ class Integrations:
         :param str integrations_id: integrations id
         :param Integration integration: integration object
         :param dict new_options: new value
+        :param str organization_id: organization id
         :return: Integration object
         :rtype: dtlpy.entities.integration.Integration
 
@@ -198,7 +209,8 @@ class Integrations:
 
             project.integrations.update(integrations_id='integrations_id', new_options={roleArn: ""})
         """
-        if self.project is None and self.org is None:
+
+        if self.project is None and self.org is None and organization_id is None:
             raise exceptions.PlatformException(
                 error='400',
                 message='Must have an organization or project')
@@ -207,10 +219,11 @@ class Integrations:
                 error='400',
                 message='Must have an integrations_id or integration')
 
-        if self.project is not None:
-            organization_id = self.project.org.get('id')
-        else:
-            organization_id = self.org.id
+        if organization_id is None:
+            if self.project is not None:
+                organization_id = self.project.org.get('id')
+            else:
+                organization_id = self.org.id
 
         url_path = '/orgs/{}/integrations/'.format(organization_id)
         payload = dict(integrationId=integrations_id if integrations_id is not None else integration.id)
@@ -230,13 +243,14 @@ class Integrations:
         return entities.Integration.from_json(_json=response.json(), client_api=self._client_api)
 
     @_api_reference.add(path='/orgs/{orgId}/integrations/{integrationId}', method='get')
-    def get(self, integrations_id: str):
+    def get(self, integrations_id: str, organization_id: str = None):
         """
         Get organization integrations. Use this method to access your integration and be able to use it in your code.
 
         **Prerequisites**: You must be an *owner* in the organization.
 
         :param str integrations_id: integrations id
+        :param str organization_id: organization id
         :return: Integration object
         :rtype: dtlpy.entities.integration.Integration
 
@@ -246,15 +260,16 @@ class Integrations:
 
             project.integrations.get(integrations_id='integrations_id')
         """
-        if self.project is None and self.org is None:
+        if self.project is None and self.org is None and organization_id is None:
             raise exceptions.PlatformException(
                 error='400',
                 message='Must have an organization or project')
 
-        if self.project is not None:
-            organization_id = self.project.org.get('id')
-        else:
-            organization_id = self.org.id
+        if organization_id is None:
+            if self.project is not None:
+                organization_id = self.project.org.get('id')
+            else:
+                organization_id = self.org.id
 
         url_path = '/orgs/{}/integrations/{}'.format(organization_id, integrations_id)
 
@@ -265,13 +280,14 @@ class Integrations:
         return entities.Integration.from_json(_json=response.json(), client_api=self._client_api)
 
     @_api_reference.add(path='/orgs/{orgId}/integrations', method='get')
-    def list(self, only_available=False):
+    def list(self, only_available=False, organization_id: str = None):
         """
         List all the organization's integrations with external storage.
 
         **Prerequisites**: You must be an *owner* in the organization.
 
         :param bool only_available: if True list only the available integrations.
+        :param str organization_id: organization id
         :return: groups list
         :rtype: list
 
@@ -281,15 +297,16 @@ class Integrations:
 
             project.integrations.list(only_available=True)
         """
-        if self.project is None and self.org is None:
+        if self.project is None and self.org is None and organization_id is None:
             raise exceptions.PlatformException(
                 error='400',
                 message='Must have an organization or project')
 
-        if self.project is not None:
-            organization_id = self.project.org.get('id')
-        else:
-            organization_id = self.org.id
+        if organization_id is None:
+            if self.project is not None:
+                organization_id = self.project.org.get('id')
+            else:
+                organization_id = self.org.id
 
         if only_available:
             url_path = '/orgs/{}/availableIntegrations'.format(organization_id)
@@ -304,7 +321,8 @@ class Integrations:
         available_integrations = miscellaneous.List(response.json())
         return available_integrations
 
-    def generate_gar_options(self, service_account: str, location: str) -> dict:
+    @staticmethod
+    def generate_gar_options(service_account: str, location: str, email: str = None) -> dict:
         """
         Generates a Google Artifact Registry JSON configuration and returns it as a base64-encoded string.
 
@@ -320,27 +338,119 @@ class Integrations:
         Returns:
             str: A base64-encoded string representation of the repository JSON configuration.
         """
-        if not service_account:
-            raise ValueError('Missing Service Account')
-        if not location:
-            raise ValueError('Missing Location')
-        user_name = "_json_key"
-        cred = f"{user_name}:{service_account}"
-        auth = str(base64.b64encode(bytes(cred, 'utf-8')))[2:-1]
+        return IntegrationUtils.generate_gar_options(service_account=service_account, location=location, email=email)
+
+    @staticmethod
+    def generate_docker_hub_options(username: str, password: str, email: str = None) -> dict:
+        """
+        Generates a Docker Hub JSON configuration and returns it as a base64-encoded string.
+
+        Parameters:
+            username (str): The Docker Hub username.
+            password (str): The Docker Hub password.
+            email (str): Optional - Docker Hub email.
+
+        Returns:
+            str: A base64-encoded string representation of the repository JSON configuration.
+        """
+        return IntegrationUtils.generate_docker_hub_options(username=username, password=password, email=email)
+
+    @staticmethod
+    def generate_ecr_options(access_key_id: str, secret_access_key: str, account: str, region: str) -> dict:
+        """
+        Generates an Amazon Elastic Container Registry (ECR) JSON configuration and returns it as a base64-encoded string.
+
+        Parameters:
+            access_key_id (str): The AWS access key ID.
+            secret_access_key (str): The AWS secret access key.
+            account (str): The AWS account ID.
+            region (str): The AWS region.
+
+        Returns:
+            str: A base64-encoded string representation of the repository JSON configuration.
+        """
+        return IntegrationUtils.generate_ecr_options(
+            access_key_id=access_key_id,
+            secret_access_key=secret_access_key,
+            account=account,
+            region=region
+        )
+
 
+class IntegrationUtils:
+
+    @staticmethod
+    def encode(st: str):
+        return str(base64.b64encode(bytes(st, 'utf-8')))[2:-1]
+
+    @staticmethod
+    def generate_json_key_options(location: str, username: str, password: str, auth: str, email: str = None):
         encoded_pass = {
             "auths": {
                 f"{location}": {
-                    "username": user_name,
-                    "password": service_account,
+                    "username": username,
+                    "password": password,
                     "auth": auth
                 }
             }
         }
 
+        if email:
+            encoded_pass['auths'][f'{location}']['email'] = email
+
         return {
             "name": "_json_key",
             "spec": {
-                "password": str(base64.b64encode(bytes(json.dumps(encoded_pass), 'utf-8')))[2:-1]
+                "password": IntegrationUtils.encode(json.dumps(encoded_pass))
+            }
+        }
+
+    @staticmethod
+    def generate_gar_options(service_account: str, location: str, email: str = None) -> dict:
+
+        if not service_account:
+            raise ValueError('Missing Service Account')
+        if not location:
+            raise ValueError('Missing Location')
+
+        username = "_json_key"
+        cred = f"{username}:{service_account}"
+        auth = IntegrationUtils.encode(cred)
+
+        return IntegrationUtils.generate_json_key_options(
+            location=location,
+            username=username,
+            password=service_account,
+            auth=auth,
+            email=email
+        )
+
+    @staticmethod
+    def generate_docker_hub_options(username: str, password: str, email: str = None) -> dict:
+
+        if not username:
+            raise ValueError('Missing Username')
+        if not password:
+            raise ValueError('Missing Password')
+
+        auth = IntegrationUtils.encode('{}:{}'.format(username, password))
+
+        return IntegrationUtils.generate_json_key_options(
+            location='docker.io',
+            username=username,
+            password=password,
+            auth=auth,
+            email=email
+        )
+
+    @staticmethod
+    def generate_ecr_options(access_key_id: str, secret_access_key: str, account: str, region: str) -> dict:
+        return {
+            "name": "AWS",
+            "spec": {
+                "accessKeyId": access_key_id,
+                "secretAccessKey": secret_access_key,
+                "account": account,
+                "region": region,
             }
         }

dtlpy/repositories/items.py

@@ -527,7 +527,9 @@ class Items:
             export_png_files: bool = False,
             filter_output_annotations: bool = False,
             alpha: float = 1,
-            export_version=entities.ExportVersion.V1
+            export_version=entities.ExportVersion.V1,
+            dataset_lock: bool = False,
+            lock_timeout_sec: int = None     
     ):
         """
         Download dataset items by filters.
@@ -547,6 +549,8 @@ class Items:
         :param list annotation_options: download annotations options:  list(dl.ViewAnnotationOptions)
         :param dtlpy.entities.filters.Filters annotation_filters: Filters entity to filter annotations for download
         :param bool overwrite: optional - default = False
+        :param bool dataset_lock: optional - default = False
+        :param int lock_timeout_sec: optional
         :param bool to_items_folder: Create 'items' folder and download items to it
         :param int thickness: optional - line thickness, if -1 annotation will be filled, default =1
         :param bool with_text: optional - add text to annotations, default = False
@@ -593,7 +597,9 @@ class Items:
             include_annotations_in_output=include_annotations_in_output,
             export_png_files=export_png_files,
             filter_output_annotations=filter_output_annotations,
-            export_version=export_version
+            export_version=export_version,
+            dataset_lock=dataset_lock,
+            lock_timeout_sec=lock_timeout_sec
         )
 
     def upload(
@@ -869,3 +875,38 @@ class Items:
             raise exceptions.PlatformException(response)
 
         return success
+    
+    def task_scores(self, item_id: str, task_id: str, page_offset: int = 0, page_size: int = 100):
+        """
+        Get item score
+
+        **Prerequisites**: You must be able to read the task
+
+        :param str item_id: item id
+        :param str task_id: task id
+        :param int page_offset: start page
+        :param int page_size: page size
+        :return: page of item scores
+
+        **Example**:
+
+        .. code-block:: python
+        
+            dataset.items.item_score(item_id='item_id', task_id='task_id')
+
+        """
+
+        if item_id is None:
+            raise exceptions.PlatformException('400', 'Must provide item id')
+
+        if task_id is None:
+            raise exceptions.PlatformException('400', 'Must provide task id')
+
+        success, response = self._client_api.gen_request(req_type="get",
+                                                         path="/scores/tasks/{}/items/{}?page={}&pageSize={}"
+                                                         .format(task_id, item_id, page_offset, page_size))
+        if success:
+            return response.json()
+        else:
+            raise exceptions.PlatformException(response)
+

dtlpy/repositories/projects.py

@@ -432,14 +432,6 @@ class Projects:
         assert isinstance(project, entities.Project)
         if checkout:
             self.checkout(project=project)
-        if project.id not in self._client_api.platform_settings.working_projects:
-            self._client_api.platform_settings.add_project(project.id)
-            try:
-                settings_list = project.settings.resolve(user_email=self._client_api.info()['user_email'],
-                                                         project_id=project.id)
-                self._client_api.platform_settings.add_bulk(settings_list)
-            except:
-                logger.warning("failed to add project settings")
         return project
 
     @_api_reference.add(path='/projects/{projectId}', method='delete')

dtlpy/repositories/settings.py

@@ -112,12 +112,6 @@ class Settings:
         else:
             raise exceptions.PlatformException(response)
 
-        # add settings to cookies
-        self._client_api.platform_settings.add(setting.name,
-                                               {
-                                                   setting.scope.id: setting.value
-                                               }
-                                               )
         return constructor(
             _json=_json,
             client_api=self._client_api,
@@ -172,13 +166,6 @@ class Settings:
         else:
             raise exceptions.PlatformException(response)
 
-        # add settings to cookies
-        self._client_api.platform_settings.add(setting.name,
-                                               {
-                                                   setting.scope.id: setting.value
-                                               }
-                                               )
-
         return constructor(
             _json=_json,
             client_api=self._client_api,

dtlpy/repositories/tasks.py

@@ -2,6 +2,7 @@ import datetime
 import logging
 import json
 from typing import Union, List
+import warnings
 
 from .. import exceptions, miscellaneous, entities, repositories, _api_reference
 from ..services.api_client import ApiClient
@@ -426,7 +427,7 @@ class Tasks:
         **Prerequisites**: You must be in the role of an *owner* or *developer* or *annotation manager* who created that task.
 
         :param dtlpy.entities.task.Task task: the task object
-        :param bool system_metadata: True, if you want to change metadata system
+        :param bool system_metadata: DEPRECATED
         :return: Task object
         :rtype: dtlpy.entities.task.Task
 
@@ -440,7 +441,7 @@ class Tasks:
         url = '{}/{}'.format(url, task.id)
 
         if system_metadata:
-            url += '?system=true'
+            warnings.warn("Task system metadata updates are not permitted. Please store custom metadata in 'task.metadata['user']' instead.", DeprecationWarning)
 
         success, response = self._client_api.gen_request(req_type='patch',
                                                          path=url,
@@ -1089,3 +1090,36 @@ class Tasks:
                     success_count=len(updated_items), failed_items=failed_items)
             logger.info(msg=log_msg)
         return True
+    
+    def task_scores(self, task_id: str = None, page_offset: int = 0, page_size: int = 100):
+        """
+        Get all entities scores in a task.
+
+        :param str task_id: the id of the task
+        :param int page_offset: the page offset
+        :param int page_size: the page size
+        :return: page of the task scores
+
+        **Example**:
+
+        .. code-block:: python
+
+            dataset.tasks.task_scores(task_id= 'task_id')
+        """
+        if task_id is None:
+            raise exceptions.PlatformException('400', 'Please provide task_id')
+    
+        url = '/scores/tasks/{task_id}?page={page_offset}&pageSize={page_size}'.format(
+            task_id=task_id,
+            page_offset=page_offset,
+            page_size=page_size
+        )
+        success, response = self._client_api.gen_request(
+            req_type='get',
+            path=url
+        )
+
+        if success:
+            return response.json()
+        else:
+            raise exceptions.PlatformException(response)

dtlpy/services/api_client.py

@@ -430,51 +430,6 @@ class Attributes2:
         os.environ["USE_ATTRIBUTE_2"] = json.dumps(val)
         self.to_cookie()
 
-
-class PlatformSettings:
-
-    def __init__(self):
-        self._working_projects = list()
-        self._settings = dict()
-
-    @property
-    def settings(self) -> dict:
-        return self._settings
-
-    @property
-    def working_projects(self) -> list:
-        return self._working_projects
-
-    @settings.setter
-    def settings(self, val: dict):
-        if not isinstance(val, dict):
-            raise exceptions.PlatformException(error=400,
-                                               message="input must be of type dict")
-
-        self._settings = val
-
-    def add(self, setting_name: str, setting: dict):
-        if setting_name in self.settings:
-            self._settings[setting_name].update(setting)
-        else:
-            self._settings[setting_name] = setting
-
-    def add_project(self, project_id: str):
-        if not isinstance(project_id, str):
-            raise exceptions.PlatformException(error=400,
-                                               message="input must be of type str")
-        self._working_projects.append(project_id)
-
-    def add_bulk(self, settings_list):
-        settings_dict = {s.name: {s.scope.id: s.value}
-                         for s in settings_list}
-        for setting_name, settings_val in settings_dict.items():
-            if setting_name in self._settings:
-                self._settings[setting_name].update(settings_val)
-            else:
-                self._settings[setting_name] = settings_val
-
-
 class Decorators:
     @staticmethod
     def token_expired_decorator(method):
@@ -522,7 +477,6 @@ class ApiClient:
         self._callbacks = None
         self._cache_state = None
         self._attributes_mode = None
-        self._platform_settings = None
         self._cache_configs = None
         self._sdk_cache = None
         self._fetch_entities = None
@@ -793,13 +747,6 @@ class ApiClient:
         assert isinstance(self._attributes_mode, Attributes2)
         return self._attributes_mode
 
-    @property
-    def platform_settings(self):
-        if self._platform_settings is None:
-            self._platform_settings = PlatformSettings()
-        assert isinstance(self._platform_settings, PlatformSettings)
-        return self._platform_settings
-
     @property
     def sdk_cache(self):
         if self._sdk_cache is None: