dtlpy 1.107.8__py3-none-any.whl → 1.109.19__py3-none-any.whl

dtlpy/repositories/collections.py

@@ -1,8 +1,5 @@
-from venv import logger
-from dtlpy import entities, exceptions, repositories
-from dtlpy.entities.dataset import Dataset
-from dtlpy.entities.filters import FiltersMethod
-from dtlpy.services.api_client import ApiClient
+from .. import entities, exceptions
+from ..services.api_client import ApiClient
 from typing import List
 
 class Collections:
@@ -15,6 +12,26 @@ class Collections:
         self._dataset = dataset
         self._item = item
 
+    @property
+    def dataset(self) -> entities.Dataset:
+        if self._dataset is None:
+            raise ValueError("Must set dataset for this action.")
+        return self._dataset
+    
+    @dataset.setter
+    def dataset(self, dataset: entities.Dataset):
+        self._dataset = dataset
+
+    @property
+    def item(self) -> entities.Item:
+        if self._item is None:
+            raise ValueError("Must set item for this action.")
+        return self._item
+    
+    @item.setter
+    def item(self, item: entities.Item):
+        self._item = item
+    
     def create(self, name: str) -> entities.Collection:
         """
         Creates a new collection in the dataset.
@@ -22,12 +39,11 @@ class Collections:
         :param name: The name of the new collection.
         :return: The created collection details.
         """
-        dataset_id = self._dataset.id
         self.validate_max_collections()
         self.validate_collection_name(name)
         payload = {"name": name}
         success, response = self._client_api.gen_request(
-            req_type="post", path=f"/datasets/{dataset_id}/items/collections", json_req=payload
+            req_type="post", path=f"/datasets/{self.dataset.id}/items/collections", json_req=payload
         )
         if success:
             collection_json = self._single_collection(data=response.json(), name=name)
@@ -43,11 +59,10 @@ class Collections:
         :param new_name: The new name for the collection.
         :return: The updated collection details.
         """
-        dataset_id = self._dataset.id
         self.validate_collection_name(new_name)
         payload = {"name": new_name}
         success, response = self._client_api.gen_request(
-            req_type="patch", path=f"/datasets/{dataset_id}/items/collections/{collection_name}", json_req=payload
+            req_type="patch", path=f"/datasets/{self.dataset.id}/items/collections/{collection_name}", json_req=payload
         )
         if success:
             collection_json = self._single_collection(data=response.json(), name=new_name)
@@ -61,9 +76,8 @@ class Collections:
 
         :param collection_name: The name of the collection to delete.
         """
-        dataset_id = self._dataset.id
         success, response = self._client_api.gen_request(
-            req_type="delete", path=f"/datasets/{dataset_id}/items/collections/{collection_name}"
+            req_type="delete", path=f"/datasets/{self.dataset.id}/items/collections/{collection_name}"
         )
         if success:
             # Wait for the split operation to complete
@@ -74,7 +88,7 @@ class Collections:
         else:
             raise exceptions.PlatformException(response)
 
-    def clone(self, collection_name: str) -> dict:
+    def clone(self, collection_name: str) -> entities.Collection:
         """
         Clones an existing collection, creating a new one with a unique name.
 
@@ -99,7 +113,10 @@ class Collections:
 
         # Create the cloned collection
         cloned_collection = self.create(name=clone_name)
-        self.assign(dataset_id=self._dataset.id, collections=[cloned_collection.name], collection_key=original_collection['key'])
+        filters = entities.Filters()
+        filters.add(field=f'metadata.system.collections.{original_collection["key"]}', values=True)
+        self.assign(collections=[cloned_collection.name], 
+                    filters=filters)
         return cloned_collection
 
 
@@ -109,9 +126,8 @@ class Collections:
 
         :return: A list of collections in the dataset.
         """
-        dataset_id = self._dataset.id
         success, response = self._client_api.gen_request(
-            req_type="GET", path=f"/datasets/{dataset_id}/items/collections"
+            req_type="GET", path=f"/datasets/{self.dataset.id}/items/collections"
         )
         if success:
             data = response.json()
@@ -140,6 +156,17 @@ class Collections:
         if len(collections) >= 10:
             raise ValueError("The dataset already has the maximum number of collections (10).")
         
+    def list_missing_collections(self) -> List[str]:
+        """
+        List all items in the dataset that are not assigned to any collection.
+
+        :return: A list of item IDs that are not part of any collection.
+        """
+        filters = entities.Filters()
+        filters.add(field='metadata.system.collections', values=None)
+        filters.add(field='datasetId', values=self._dataset.id)
+        return self._dataset.items.list(filters=filters)
+    
     def list_unassigned_items(self) -> list:
         """
         List unassigned items in a dataset (items where all collection fields are false).
@@ -147,7 +174,7 @@ class Collections:
         :return: List of unassigned item IDs
         :rtype: list
         """
-        filters = entities.Filters(method=FiltersMethod.AND)  # Use AND method for all conditions
+        filters = entities.Filters(method=entities.FiltersMethod.AND)  # Use AND method for all conditions
         collection_fields = [
             "collections0",
             "collections1",
@@ -163,7 +190,7 @@ class Collections:
 
         # Add each field to the filter with a value of False
         for field in collection_fields:
-            filters.add(field=field, values=False, method=FiltersMethod.AND)
+            filters.add(field=field, values=False, method=entities.FiltersMethod.AND)
 
         missing_ids = []
         pages = self._dataset.items.list(filters=filters)
@@ -176,31 +203,33 @@ class Collections:
 
     def assign(
         self, 
-        dataset_id: str, 
         collections: List[str],
+        dataset_id: str = None, 
         item_id: str = None, 
-        collection_key: str = None
+        filters: entities.Filters = None
     ) -> bool:
         """
         Assign an item to a collection. Creates the collection if it does not exist.
 
-        :param dataset_id: ID of the dataset.
         :param collections: List of the collections to assign the item to.
+        :param dataset_id: ID of the dataset.
         :param item_id: (Optional) ID of the item to assign. If not provided, all items in the dataset will be updated.
-        :param collection_key: (Optional) Key for the bulk assignment. If not provided, no specific metadata will be updated.
+        :param filters: (Optional) Filters of items to assign to the collections.
         :return: True if the assignment was successful, otherwise raises an exception.
         """
+        if not isinstance(collections, list):
+            raise ValueError("collections must be a list.")
+        if dataset_id is None and self._dataset is not None:
+            dataset_id = self.dataset.id
+        if item_id is None and self._item is not None:
+            item_id = self.item.id 
         # Build the query structure
-        if collection_key:
-            query = {
-                "filter": {
-                    f"metadata.system.collections.{collection_key}": True
-                }
-            }
-        elif item_id:
+        if item_id is not None:
             query = {
                 "id": {"$eq": item_id}
             }
+        elif filters is not None:
+            query = filters.prepare().get("filter")
         else:
             raise ValueError("Either collection_key or item_id must be provided.")
 
@@ -226,16 +255,39 @@ class Collections:
             raise exceptions.PlatformException(f"Failed to assign item to collections: {response}")
 
         
-    def unassign(self, dataset_id: str, item_id: str, collections: List[str]) -> bool:
+    def unassign(self, 
+                 collections: List[str], 
+                 dataset_id: str = None,
+                 item_id: str = None, 
+                 filters: entities.Filters = None) -> bool:
         """
         Unassign an item from a collection.
-        :param item_id: ID of the item.
         :param collections: List of collection names to unassign.
+        :param dataset_id: ID of the dataset.
+        :param item_id: ID of the item.
+        :param filters: (Optional) Filters of items to unassign from the collections.
         """
-        payload = {
-            "query": {"id": {"$eq": item_id}},
-            "collections": collections,
-        }
+        if not isinstance(collections, list):
+            raise ValueError("collections must be a list.")
+        # build the context
+        if dataset_id is None and self._dataset is not None:
+            dataset_id = self._dataset.id
+        if item_id is None and self._item is not None:
+            item_id = self._item.id
+            
+        # build the payload
+        if item_id is not None and filters is None:
+            payload = {
+                "query": {"id": {"$eq": item_id}},
+                "collections": collections,
+            }
+        elif filters is not None and item_id is None:
+            payload = {
+                "query": filters.prepare().get("filter"),
+                "collections": collections,
+            }
+        else:
+            raise ValueError("Either item_id or filters must be provided but not both.")
         success, response = self._client_api.gen_request(
             req_type="post",
             path=f"/datasets/{dataset_id}/items/collections/bulk-remove",

dtlpy/repositories/commands.py

@@ -70,7 +70,7 @@ class Commands:
         return entities.Command.from_json(client_api=self._client_api,
                                           _json=response.json())
 
-    def wait(self, command_id, timeout=0, step=None, url=None, backoff_factor=1):
+    def wait(self, command_id, timeout=0, step=None, url=None, backoff_factor=1, iteration_callback=None):
         """
         Wait for command to finish
 
@@ -84,6 +84,7 @@ class Commands:
         :param int timeout: int, seconds to wait until TimeoutError is raised. if 0 - wait until done
         :param int step: int, seconds between polling
         :param str url: url to the command
+        :param function iteration_callback: function to call on each iteration
         :param float backoff_factor: A backoff factor to apply between attempts after the second try
         :return: Command  object
         """
@@ -112,9 +113,18 @@ class Commands:
             elapsed = time.time() - start
             sleep_time = np.min([timeout - elapsed, backoff_factor * (2 ** num_tries), MAX_SLEEP_TIME])
             num_tries += 1
-            logger.debug("Command {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(command.id,
-                                                                                                         elapsed,
-                                                                                                         sleep_time))
+            logger.debug(
+                "Command {!r} is running for {:.2f}[s] and now Going to sleep {:.2f}[s]".format(
+                    command.id,
+                    elapsed,
+                    sleep_time
+                )
+            )
+            if iteration_callback is not None:
+                try:
+                    iteration_callback()
+                except Exception as e:
+                    logger.warning('iteration_callback failed: {}'.format(e.__str__()))
             time.sleep(sleep_time)
         pbar.close()
         if command is None:

dtlpy/repositories/computes.py

@@ -2,11 +2,16 @@ import base64
 import datetime
 import json
 
+from dtlpy import miscellaneous
+
 from ..services.api_client import ApiClient
 from .. import exceptions, entities, repositories
 from typing import List, Optional, Dict
-from ..entities import ComputeCluster, ComputeContext, ComputeType, Project
+from ..entities import ComputeCluster, ComputeContext, ComputeType
 from ..entities.integration import IntegrationType
+import logging
+
+logger = logging.getLogger(name='dtlpy')
 
 
 class Computes:
@@ -17,6 +22,7 @@ class Computes:
         self._commands = None
         self._projects = None
         self._organizations = None
+        self.log_cache = dict()
 
     @property
     def commands(self) -> repositories.Commands:
@@ -46,7 +52,8 @@ class Computes:
             is_global: Optional[bool] = False,
             features: Optional[Dict] = None,
             wait=True,
-            status: entities.ComputeStatus = None
+            status: entities.ComputeStatus = None,
+            settings: entities.ComputeSettings = None
     ):
         """
         Create a new compute
@@ -60,18 +67,25 @@ class Computes:
         :param features: Features
         :param wait: Wait for compute creation
         :param status: Compute status
+        :param settings: Compute settings
         :return: Compute
+        :rtype: dl.entities.compute.Compute
         """
 
+        shared_contexts_json = []
+        for shared_context in shared_contexts:
+            src_json = shared_context.to_json() if isinstance(shared_context, entities.ComputeContext) else shared_context
+            shared_contexts_json.append(src_json)
         payload = {
             'name': name,
             'context': context.to_json(),
             'type': type.value,
             'global': is_global,
             'features': features,
-            'shared_contexts': [sc.to_json() for sc in shared_contexts],
+            'sharedContexts': shared_contexts_json,
             'cluster': cluster.to_json(),
-            'status': status
+            'status': status,
+            "settings": settings.to_json() if isinstance(settings, entities.ComputeSettings) else settings
         }
 
         # request
@@ -84,26 +98,69 @@ class Computes:
         if not success:
             raise exceptions.PlatformException(response)
 
-        compute = entities.Compute.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
+        compute = self._build_compute_by_type(response.json())
 
         if wait:
             command_id = compute.metadata.get('system', {}).get('commands', {}).get('create', None)
             if command_id is not None:
                 command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
-                command.wait()
+                try:
+                    command.wait(iteration_callback=self.__get_log_compute_progress_callback(compute.id))
+                except Exception as e:
+                    self.log_cache.pop(compute.id, None)
+                    raise e
                 compute = self.get(compute_id=compute.id)
 
         return compute
 
+    def _build_compute_by_type(self, _json):
+        if _json.get('type') == 'kubernetes':
+            compute = entities.KubernetesCompute.from_json(
+                _json=_json,
+                client_api=self._client_api
+            )
+        else:
+            compute = entities.Compute.from_json(
+                _json=_json,
+                client_api=self._client_api
+            )
+        return compute
+
+    def __get_log_compute_progress_callback(self, compute_id: str):
+        def func():
+            compute = self.get(compute_id=compute_id)
+            bootstrap_progress = compute.metadata.get('system', {}).get('bootstrapProcess', {}).get('progress', None)
+            bootstrap_logs = compute.metadata.get('system', {}).get('bootstrapProcess', {}).get('logs', None)
+            validation_progress = compute.metadata.get('system', {}).get('validation', {}).get('progress', None)
+            validation_logs = compute.metadata.get('system', {}).get('validation', {}).get('logs', None)
+            if bootstrap_progress not in [None, 100]:
+                logger.info(f"Bootstrap in progress: {bootstrap_progress}%")
+                last_index = len(self.log_cache.get(compute_id, {}).get('bootstrap', []))
+                new_logs = bootstrap_logs[last_index:]
+                if new_logs:
+                    logger.info("Bootstrap Logs: {}".format('\n'.join(new_logs)))
+                    if compute_id not in self.log_cache:
+                        self.log_cache[compute_id] = {}
+                    self.log_cache[compute_id]['bootstrap'] = bootstrap_logs
+            if validation_progress not in [None, 100]:
+                logger.info(f"Validating created compute. Progress: {validation_progress}%")
+                last_index = len(self.log_cache.get(compute_id, {}).get('validation', []))
+                new_logs = validation_logs[last_index:]
+                if new_logs:
+                    logger.info("Validation Logs: {}".format('\n'.join(new_logs)))
+                    if compute_id not in self.log_cache:
+                        self.log_cache[compute_id] = {}
+                    self.log_cache[compute_id]['validation'] = validation_logs
+        return func
+
+
     def get(self, compute_id: str):
         """
         Get a compute
 
         :param compute_id: Compute ID
         :return: Compute
+        :rtype: dl.entities.compute.Compute
         """
 
         # request
@@ -115,10 +172,7 @@ class Computes:
         if not success:
             raise exceptions.PlatformException(response)
 
-        compute = entities.Compute.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
+        compute = self._build_compute_by_type(response.json())
 
         return compute
 
@@ -128,6 +182,7 @@ class Computes:
 
         :param compute: Compute
         :return: Compute
+        :rtype: dl.entities.compute.Compute
         """
 
         # request
@@ -140,10 +195,7 @@ class Computes:
         if not success:
             raise exceptions.PlatformException(response)
 
-        compute = entities.Compute.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
+        compute = self._build_compute_by_type(response.json())
 
         return compute
 
@@ -165,6 +217,60 @@ class Computes:
 
         return True
 
+    def validate(self, compute_id: str, wait: bool = True):
+        """
+        Validate a compute
+
+        :param str compute_id: Compute ID
+        :param bool wait: Wait for validation
+        :return: Compute
+        :rtype: dl.entities.compute.Compute
+        """
+
+        # request
+        success, response = self._client_api.gen_request(
+            req_type='post',
+            path=self._base_url + '/{}/validate'.format(compute_id)
+        )
+
+        if not success:
+            raise exceptions.PlatformException(response)
+
+        compute = self._build_compute_by_type(response.json())
+
+        if wait:
+            command_id = compute.metadata.get('system', {}).get('commands', {}).get('validate', None)
+            if command_id is not None:
+                command = self.commands.get(command_id=command_id, url='api/v1/commands/faas/{}'.format(command_id))
+                try:
+                    command.wait(iteration_callback=self.__get_log_compute_progress_callback(compute.id))
+                except Exception as e:
+                    self.log_cache.pop(compute.id, None)
+                    raise e
+                compute = self.get(compute_id=compute.id)
+
+        return compute
+
+    def list_global(self):
+        """
+        List computes
+
+        :return: List of computes
+        :rtype: list[str]
+        """
+
+        # request
+        success, response = self._client_api.gen_request(
+            req_type='get',
+            path=self._base_url + '/globals',
+        )
+
+        if not success:
+            raise exceptions.PlatformException(response)
+
+
+        return response.json()
+
     @staticmethod
     def read_file(file_path):
         try:
@@ -205,7 +311,9 @@ class Computes:
             [],
             cluster,
             ComputeType.KUBERNETES,
-            status=config['config'].get('status', None))
+            status=config['config'].get('status', None),
+            settings=config['config'].get('settings', None))
+
         return compute
 
     def create_from_config_file(self, config_file_path, org_id, project_name: Optional[str] = None):
@@ -221,121 +329,59 @@ class Computes:
         return compute
 
 
-class ServiceDrivers:
-
-    def __init__(self, client_api: ApiClient):
-        self._client_api = client_api
-        self._base_url = '/serviceDrivers'
-
-    def create(
-            self,
-            name: str,
-            compute_id: str,
-            context: entities.ComputeContext
-    ):
-        """
-        Create a new service driver
-
-        :param name: Service driver name
-        :param compute_id: Compute ID
-        :param context: Compute context
-        :return: Service driver
-
-        """
-
-        payload = {
-            'name': name,
-            'computeId': compute_id,
-            'context': context.to_json()
-        }
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='post',
-            path=self._base_url,
-            json_req=payload
-        )
-
-        if not success:
-            raise exceptions.PlatformException(response)
-
-        service_driver = entities.ServiceDriver.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
-
-        return service_driver
-
-    def get(self, service_driver_id: str):
-        """
-        Get a service driver
-
-        :param service_driver_id: Service driver ID
-        :return: Service driver
-        """
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='get',
-            path=self._base_url + '/{}'.format(service_driver_id)
-        )
-
+    def _list(self, filters: entities.Filters):
+        url = self._base_url + '/query'
+        success, response = self._client_api.gen_request(req_type='POST',
+                                                         path=url,
+                                                         json_req=filters.prepare())
         if not success:
             raise exceptions.PlatformException(response)
 
-        service_driver = entities.ServiceDriver.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
-
-        return service_driver
-
-    def delete(self, service_driver_id: str):
+        return response.json()
+
+    def _build_entities_from_response(self, response_items) -> miscellaneous.List[entities.Compute]:
+        pool = self._client_api.thread_pools(pool_name='entity.create')
+        jobs = [None for _ in range(len(response_items))]
+        for i_item, item in enumerate(response_items):
+            jobs[i_item] = pool.submit(entities.Compute._protected_from_json,
+                                       **{'client_api': self._client_api,
+                                          '_json': item})
+        results = [j.result() for j in jobs]
+        _ = [logger.warning(r[1]) for r in results if r[0] is False]
+        items = miscellaneous.List([r[1] for r in results if r[0] is True])
+        return items
+
+    def list(self, filters: entities.Filters = None) -> entities.PagedEntities:
         """
-        Delete a service driver
-
-        :param service_driver_id: Service driver ID
-        """
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='delete',
-            path=self._base_url + '/{}'.format(service_driver_id)
-        )
-
-        if not success:
-            raise exceptions.PlatformException(response)
+        List all services drivers
 
-        return True
+        :param dtlpy.entities.filters.Filters filters: Filters entity or a dictionary containing filters parameters
+        :return: Paged entity
+        :rtype: dtlpy.entities.paged_entities.PagedEntities
 
-    def set_default(self, service_driver_id: str, org_id: str, update_existing_services=False):
-        """
-        Set a service driver as default
+        **Example**:
 
-        :param service_driver_id: Compute name
-        :param org_id: Organization ID
-        :param update_existing_services: Update existing services
+        .. code-block:: python
 
-        :return: Service driver
+            services = dl.service_drivers.list()
         """
-
-        # request
-        success, response = self._client_api.gen_request(
-            req_type='post',
-            path=self._base_url + '/default',
-            json_req={
-                'organizationId': org_id,
-                'updateExistingServices': update_existing_services,
-                'driverName': service_driver_id
-            }
-        )
-
-        if not success:
-            raise exceptions.PlatformException(response)
-
-        service_driver = entities.ServiceDriver.from_json(
-            _json=response.json(),
-            client_api=self._client_api
-        )
-
-        return service_driver
+        # default filters
+        if filters is None:
+            filters = entities.Filters(resource=entities.FiltersResource.COMPUTE)
+
+        if filters.resource != entities.FiltersResource.COMPUTE:
+            raise exceptions.PlatformException(
+                error='400',
+                message='Filters resource must to be FiltersResource.COMPUTE. Got: {!r}'.format(
+                    filters.resource))
+
+        if not isinstance(filters, entities.Filters):
+            raise exceptions.PlatformException('400', 'Unknown filters type')
+
+        paged = entities.PagedEntities(items_repository=self,
+                                       filters=filters,
+                                       page_offset=filters.page,
+                                       page_size=filters.page_size,
+                                       client_api=self._client_api)
+        paged.get_page()
+        return paged