qarnot 2.13.0__tar.gz → 2.14.5__tar.gz

{qarnot-2.13.0/qarnot.egg-info → qarnot-2.14.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qarnot
-Version: 2.13.0
+Version: 2.14.5
 Summary: Qarnot Computing SDK
 Home-page: https://computing.qarnot.com
 Author: Qarnot computing
@@ -16,11 +16,16 @@ Classifier: Intended Audience :: Information Technology
 Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: boto3
+Requires-Dist: wheel
+Requires-Dist: deprecation
+Requires-Dist: simplejson
 
 Qarnot computing Python SDK
 ===========================
 
-|travis-badge|_ |pypi-badge|_ |readthedocs-badge|_
+|pypi-badge|_ |readthedocs-badge|_
 
 This package allows you to use Qarnot cloud computing service.
 
@@ -63,5 +68,3 @@ The index of the doc is then generated in `doc/_build/html/index.html`
 .. _pypi-badge: https://pypi.python.org/pypi/qarnot/
 .. |readthedocs-badge| image:: https://readthedocs.org/projects/qarnot/badge/?version=latest
 .. _readthedocs-badge: https://qarnot.readthedocs.io/en/latest/
-.. |travis-badge| image:: https://app.travis-ci.com/qarnot/qarnot-sdk-python.svg?branch=master
-.. _travis-badge: https://app.travis-ci.com/qarnot/qarnot-sdk-python

{qarnot-2.13.0 → qarnot-2.14.5}/README.rst

@@ -1,7 +1,7 @@
 Qarnot computing Python SDK
 ===========================
 
-|travis-badge|_ |pypi-badge|_ |readthedocs-badge|_
+|pypi-badge|_ |readthedocs-badge|_
 
 This package allows you to use Qarnot cloud computing service.
 
@@ -44,5 +44,3 @@ The index of the doc is then generated in `doc/_build/html/index.html`
 .. _pypi-badge: https://pypi.python.org/pypi/qarnot/
 .. |readthedocs-badge| image:: https://readthedocs.org/projects/qarnot/badge/?version=latest
 .. _readthedocs-badge: https://qarnot.readthedocs.io/en/latest/
-.. |travis-badge| image:: https://app.travis-ci.com/qarnot/qarnot-sdk-python.svg?branch=master
-.. _travis-badge: https://app.travis-ci.com/qarnot/qarnot-sdk-python

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/__init__.py

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 
-from .exceptions import QarnotGenericException, UnauthorizedException
+from .exceptions import QarnotGenericException, SecretConflictException, SecretNotFoundException, UnauthorizedException
 from ._util import get_error_message_from_http_response
 
 __all__ = ["task", "connection", "bucket", "pool",
@@ -35,6 +35,13 @@ def raise_on_error(response):
             raise QarnotGenericException(response.text) from value
 
 
+def raise_on_secrets_specific_error(response):
+    if response.status_code == 404:
+        raise SecretNotFoundException()
+    if response.status_code == 409:
+        raise SecretConflictException()
+
+
 def get_url(key, **kwargs):
     """Get and format the url for the given key.
     """
@@ -69,6 +76,8 @@ def get_url(key, **kwargs):
         'pool stderr': '/pools/{uuid}/stderr',  # GET -> pool stderr
         'pool instance stdout': '/pools/{uuid}/stdout/{instanceId}',  # GET -> pool instance stdout
         'pool instance stderr': '/pools/{uuid}/stderr/{instanceId}',  # GET -> pool instance stderr
+        'secrets data': '/secrets-manager/data/{secret_key}',  # GET -> get secret , PUT -> create secret, PATCH -> update secret, DELETE -> delete secret
+        'secrets search': '/secrets-manager/search/{secret_prefix}',  # GET -> lists secrets starting with prefix
         'user': '/info',  # GET -> user info
         'profiles': '/profiles',  # GET -> profiles list
         'profile details': '/profiles/{profile}',  # GET -> profile details
@@ -84,3 +93,4 @@ __version__ = get_versions()['version']  # type: ignore
 del get_versions
 
 from .connection import Connection  # noqa
+from .secrets import Secrets  # noqa

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/_util.py

@@ -144,7 +144,11 @@ def get_sanitized_bucket_path(path: str, show_warning: bool = True):
 def get_error_message_from_http_response(response: Response, message_is_status_code_if_null: bool = False) -> str:
     error_message = ""
     try:
-        error_message = response.json()['message']
+        error_response = response.json()
+        if 'message' in error_response:
+            error_message = error_response['message']
+        elif 'error' in error_response:
+            error_message = error_response['error']
     except (JSONDecodeError, simpleJsonDecodeError):
         error_message = response.text
     if (error_message is None or error_message == "" or len(error_message) < 1) and message_is_status_code_if_null:

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-20T09:58:50+0200",
+ "date": "2024-03-29T14:48:51+0100",
  "dirty": false,
  "error": null,
- "full-revisionid": "8bf9157ea1fc05fc9fe5d296372308330f100280",
- "version": "v2.13.0"
+ "full-revisionid": "ff28a98ad04060443b7c6b1f487e492c61e31be1",
+ "version": "v2.14.5"
 }
 '''  # END VERSION_JSON
 

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/bucket.py

@@ -227,8 +227,11 @@ class Bucket(Storage):  # pylint: disable=W0223
         :returns: A list of ObjectSummary resources
 
         """
-        bucket = self._connection.s3resource.Bucket(self._uuid)
-        return bucket.objects.all()
+        try:
+            bucket = self._connection.s3resource.Bucket(self._uuid)
+            return [b for b in bucket.objects.all() if b.key is not None]
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot list files. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
 
     def directory(self, directory=''):
         """List files in a directory of the bucket according to prefix.
@@ -282,6 +285,7 @@ class Bucket(Storage):  # pylint: disable=W0223
         :param dict files: Dictionary of synchronized files
         :param bool verbose: Print information about synchronization operations
         :param str remote: path of the directory on remote node (defaults to *local*)
+        :raises ~qarnot.exceptions.MissingBucketException: the bucket is not on the server
 
         Dictionary key is the remote file path while value is the local file
         path.
@@ -348,53 +352,56 @@ class Bucket(Storage):  # pylint: disable=W0223
         def objectsummarytocomparable(object_):
             return Comparable(object_.key, object_.e_tag, None)
 
-        localfiles = set()
-        if self._connection._sanitize_bucket_paths:
-            remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
-        for name, filepath in files.items():
-            localfiles.add(localtocomparable(name.replace(os.path.sep, '/'), filepath, remote))
+        try:
+            localfiles = set()
+            if self._connection._sanitize_bucket_paths:
+                remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
+            for name, filepath in files.items():
+                localfiles.add(localtocomparable(name.replace(os.path.sep, '/'), filepath, remote))
 
-        remotefiles = set(map(objectsummarytocomparable, self.list_files()))
+            remotefiles = set(map(objectsummarytocomparable, self.list_files()))
 
-        adds = localfiles - remotefiles
-        removes = remotefiles - localfiles
+            adds = localfiles - remotefiles
+            removes = remotefiles - localfiles
 
-        seen_tags = set()  # To avoid copying the same objects multiple times when renaming
-        for file_ in removes:
-            if remote is not None and not file_.name.startswith(remote):
-                continue
-            renames = (x for x in adds if x.e_tag not in seen_tags and x.e_tag == file_.e_tag
-                       and all(rem.name != x.name for rem in remotefiles))
-            for dup in renames:
-                if verbose:
-                    self._connection.logger.info("Copy", file_.name, "to", dup.name)
-                self.copy_file(file_.name, dup.name)
-            if verbose:
-                self._connection.logger.info("Remove:", file_.name)
-            self.delete_file(file_.name)
-            seen_tags.add(file_.e_tag)
-
-        remotefiles = set(map(objectsummarytocomparable, self.list_files()))
-
-        sadds = sorted(adds, key=lambda x: x.e_tag)
-        groupedadds = (list(g) for _, g in itertools.groupby(sadds, lambda x: x.e_tag))
-
-        for entry in groupedadds:
-            try:
-                rem = next(x for x in remotefiles if x.e_tag == entry[0].e_tag)
-                if rem.name == entry[0].name:
+            seen_tags = set()  # To avoid copying the same objects multiple times when renaming
+            for file_ in removes:
+                if remote is not None and not file_.name.startswith(remote):
                     continue
+                renames = (x for x in adds if x.e_tag not in seen_tags and x.e_tag == file_.e_tag
+                           and all(rem.name != x.name for rem in remotefiles))
+                for dup in renames:
+                    if verbose:
+                        self._connection.logger.info("Copy %s to %s" % (file_.name, dup.name))
+                    self.copy_file(file_.name, dup.name)
                 if verbose:
-                    self._connection.logger.info("Copy", rem.name, "to", entry[0].name)
-                self.copy_file(rem.name, entry[0].name)
-            except StopIteration:
-                if verbose:
-                    self._connection.logger.info("Upload:", entry[0].filepath, '->', entry[0].name)
-                self.add_file(entry[0].filepath, entry[0].name)
-            for link in entry[1:]:  # duplicate files
-                if verbose:
-                    self._connection.logger.info("Copy", entry[0].name, "to", link.name)
-                self.copy_file(entry[0].name, link.name)
+                    self._connection.logger.info("Remove: %s" % file_.name)
+                self.delete_file(file_.name)
+                seen_tags.add(file_.e_tag)
+
+            remotefiles = set(map(objectsummarytocomparable, self.list_files()))
+
+            sadds = sorted(adds, key=lambda x: x.e_tag)
+            groupedadds = (list(g) for _, g in itertools.groupby(sadds, lambda x: x.e_tag))
+
+            for entry in groupedadds:
+                try:
+                    rem = next(x for x in remotefiles if x.e_tag == entry[0].e_tag)
+                    if rem.name == entry[0].name:
+                        continue
+                    if verbose:
+                        self._connection.logger.info("Copy %s to %s" % (rem.name, entry[0].name))
+                    self.copy_file(rem.name, entry[0].name)
+                except StopIteration:
+                    if verbose:
+                        self._connection.logger.info("Upload: %s -> %s" % (entry[0].filepath, entry[0].name))
+                    self.add_file(entry[0].filepath, entry[0].name)
+                for link in entry[1:]:  # duplicate files
+                    if verbose:
+                        self._connection.logger.info("Copy %s to %s" % (entry[0].name, link.name))
+                    self.copy_file(entry[0].name, link.name)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot sync files. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
 
     def add_string(self, string, remote):
         """Add a string on the storage.
@@ -416,14 +423,21 @@ class Bucket(Storage):  # pylint: disable=W0223
             file_ = local_or_file
         dest = remote or os.path.basename(file_.name)
 
-        self._connection.s3client.upload_fileobj(file_, self._uuid, dest, Config=s3_multipart_config)
-        if tobeclosed:
-            file_.close()
+        try:
+            self._connection.s3client.upload_fileobj(file_, self._uuid, dest, Config=s3_multipart_config)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot add string. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
+        finally:
+            if tobeclosed:
+                file_.close()
 
     @_util.copy_docs(Storage.get_all_files)
     def get_all_files(self, output_dir, progress=None):
-        list_files_only = [x for x in self.list_files() if not x.key.endswith('/')]
-        list_directories_only = [x for x in self.list_files() if x.key.endswith('/')]
+        try:
+            list_files_only = [x for x in self.list_files() if not x.key.endswith('/')]
+            list_directories_only = [x for x in self.list_files() if x.key.endswith('/')]
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot get files. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
 
         for directory in list_directories_only:
             if not os.path.isdir(os.path.join(output_dir, directory.key.lstrip('/'))):
@@ -465,11 +479,14 @@ class Bucket(Storage):  # pylint: disable=W0223
 
     @_util.copy_docs(Storage.copy_file)
     def copy_file(self, source, dest):
-        copy_source = {
-            'Bucket': self._uuid,
-            'Key': source
-        }
-        return self._connection.s3client.copy_object(CopySource=copy_source, Bucket=self._uuid, Key=dest)
+        try:
+            copy_source = {
+                'Bucket': self._uuid,
+                'Key': source
+            }
+            return self._connection.s3client.copy_object(CopySource=copy_source, Bucket=self._uuid, Key=dest)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot copy file {} to {} from bucket {}. Bucket not found.".format(source, dest, err.response['Error']['BucketName'])) from err
 
     @deprecation.deprecated(deprecated_in="2.6.0", removed_in="3.0",
                             current_version=__version__,  # type: ignore
@@ -490,14 +507,20 @@ class Bucket(Storage):  # pylint: disable=W0223
             if hasattr(remote, 'read'):
                 shutil.copyfileobj(remote, data)
             else:
-                self._connection.s3client.download_fileobj(self._uuid, remote, data)
+                try:
+                    self._connection.s3client.download_fileobj(self._uuid, remote, data)
+                except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+                    raise MissingBucketException("Cannot download file {} from bucket {}. Bucket not found.".format(remote, err.response['Error']['BucketName'])) from err
         return local
 
     @_util.copy_docs(Storage.delete_file)
     def delete_file(self, remote):
-        if self._connection._sanitize_bucket_paths:
-            remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
-        self._connection.s3client.delete_object(Bucket=self._uuid, Key=remote)
+        try:
+            if self._connection._sanitize_bucket_paths:
+                remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
+            self._connection.s3client.delete_object(Bucket=self._uuid, Key=remote)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot delete file {} from bucket {}. Bucket not found.".format(remote, err.response['Error']['BucketName'])) from err
 
     @property
     def uuid(self):

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/exceptions.py

@@ -30,7 +30,9 @@ __all__ = ['QarnotException',
            'NotEnoughCreditsException',
            'MissingBucketException',
            'MaxJobException',
-           'MissingJobException']
+           'MissingJobException',
+           'SecretNotFoundException',
+           'SecretConflictException']
 
 
 class QarnotException(Exception):
@@ -93,3 +95,11 @@ class MissingJobException(Exception):
 
 class MaxJobException(Exception):
     """Max number of jobs reached."""
+
+
+class SecretNotFoundException(Exception):
+    """Requested secret was not found."""
+
+
+class SecretConflictException(Exception):
+    """Secret already exists."""

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/forced_network_rule.py

@@ -76,6 +76,66 @@ class ForcedNetworkRule(object):
 
         Whether the network endpoint to access is in the payload."""
 
+    @classmethod
+    def from_json(cls, json: Dict[str, Union[str, bool]]):
+        """Create the forced network rule from json.
+
+        :param dict json: Dictionary representing the forced network rule
+        :returns: The created :class:`~qarnot.retry_settings.ForcedNetworkRule`
+        """
+
+        inbound: bool = bool(json["inbound"])
+        proto: str = str(json["proto"])
+
+        port: str = None
+        if 'port' in json:
+            port = str(json["port"])
+
+        to: str = None
+        if 'to' in json:
+            to = str(json["to"])
+
+        public_host: str = None
+        if 'public_host' in json:
+            public_host = str(json["public_host"])
+
+        public_port: str = None
+        if 'public_port' in json:
+            public_port = str(json["public_port"])
+
+        forwarder: str = None
+        if 'forwarder' in json:
+            forwarder = str(json["forwarder"])
+
+        priority: str = None
+        if 'priority' in json:
+            priority = str(json["priority"])
+
+        description: str = None
+        if 'description' in json:
+            description = str(json["description"])
+
+        to_qbox: Optional[bool] = None
+        if 'to_qbox' in json:
+            to_qbox = bool(json["to_qbox"])
+
+        to_payload: Optional[bool] = None
+        if 'to_payload' in json:
+            to_payload = bool(json["to_payload"])
+
+        return ForcedNetworkRule(
+            inbound,
+            proto,
+            port,
+            to,
+            public_host,
+            public_port,
+            forwarder,
+            priority,
+            description,
+            to_qbox,
+            to_payload)
+
     def to_json(self):
         result: Dict[str, Union[str, bool]] = {
             "inbound": self.inbound,

{qarnot-2.13.0 → qarnot-2.14.5}/qarnot/pool.py

@@ -19,6 +19,7 @@ from typing import Dict, List, Optional
 
 from qarnot.retry_settings import RetrySettings
 from qarnot.forced_network_rule import ForcedNetworkRule
+from qarnot.secrets import SecretsAccessRights
 
 from . import raise_on_error, get_url, _util
 from .bucket import Bucket
@@ -124,6 +125,7 @@ class Pool(object):
         self._privileges: Privileges = Privileges()
         self._default_retry_settings: RetrySettings = RetrySettings()
         self._forced_network_rules: List[ForcedNetworkRule] = []
+        self._secrets_access_rights: SecretsAccessRights = SecretsAccessRights()
 
     @classmethod
     def _retrieve(cls, connection, uuid):
@@ -236,8 +238,9 @@ class Pool(object):
             self._default_retry_settings = RetrySettings.from_json(json_pool["defaultRetrySettings"])
         if 'schedulingType' in json_pool:
             self._scheduling_type = SchedulingType.from_string(json_pool["schedulingType"])
-        if 'forcedNetworkRules' in json_pool:
-            self._forced_network_rules = json_pool["forcedNetworkRules"]
+        self._forced_network_rules = [ForcedNetworkRule.from_json(forced_network_dict) for forced_network_dict in json_pool.get("forcedNetworkRules", [])]
+        if 'secretsAccessRights' in json_pool:
+            self._secrets_access_rights = SecretsAccessRights.from_json(json_pool["secretsAccessRights"])
 
     def _to_json(self):
         """Get a dict ready to be json packed from this pool."""
@@ -293,7 +296,10 @@ class Pool(object):
             json_pool['targetedReservedMachineKey'] = self._targeted_reserved_machine_key
 
         if self._forced_network_rules is not None:
-            json_pool['forcedNetworkRules'] = self._forced_network_rules
+            json_pool['forcedNetworkRules'] = [x.to_json() for x in self._forced_network_rules]
+
+        if self._secrets_access_rights:
+            json_pool['secretsAccessRights'] = self._secrets_access_rights.to_json()
 
         return json_pool
 
@@ -1068,6 +1074,31 @@ class Pool(object):
 
         self._constraints = value
 
+    @property
+    def secrets_access_rights(self):
+        """:type: :class:`~qarnot.secrets.SecretsAccessRights`
+        :getter: Returns the description of the secrets the tasks in this pool will have access to when running.
+        :setter: set the secrets this pool will have access to when running.
+
+        Secrets can be accessible either by exact match on the key or by using a prefix
+        in order to match all the secrets starting with said prefix.
+        """
+        self._update_if_summary()
+        if self._auto_update:
+            self.update()
+
+        return self._secrets_access_rights
+
+    @secrets_access_rights.setter
+    def secrets_access_rights(self, value: SecretsAccessRights):
+        """Setter for secrets access rights
+        """
+        self._update_if_summary()
+        if self._auto_update:
+            self.update()
+
+        self._secrets_access_rights = value
+
     @property
     def forced_network_rules(self):
         """:type: list{:class:`~qarnot.forced_network_rule.ForcedNetworkRule`}