qarnot 2.13.1__tar.gz → 2.15.0__tar.gz

{qarnot-2.13.1/qarnot.egg-info → qarnot-2.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qarnot
-Version: 2.13.1
+Version: 2.15.0
 Summary: Qarnot Computing SDK
 Home-page: https://computing.qarnot.com
 Author: Qarnot computing
@@ -16,11 +16,16 @@ Classifier: Intended Audience :: Information Technology
 Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: boto3
+Requires-Dist: wheel
+Requires-Dist: deprecation
+Requires-Dist: simplejson
 
 Qarnot computing Python SDK
 ===========================
 
-|travis-badge|_ |pypi-badge|_ |readthedocs-badge|_
+|pypi-badge|_ |readthedocs-badge|_
 
 This package allows you to use Qarnot cloud computing service.
 
@@ -63,5 +68,3 @@ The index of the doc is then generated in `doc/_build/html/index.html`
 .. _pypi-badge: https://pypi.python.org/pypi/qarnot/
 .. |readthedocs-badge| image:: https://readthedocs.org/projects/qarnot/badge/?version=latest
 .. _readthedocs-badge: https://qarnot.readthedocs.io/en/latest/
-.. |travis-badge| image:: https://app.travis-ci.com/qarnot/qarnot-sdk-python.svg?branch=master
-.. _travis-badge: https://app.travis-ci.com/qarnot/qarnot-sdk-python

{qarnot-2.13.1 → qarnot-2.15.0}/README.rst

@@ -1,7 +1,7 @@
 Qarnot computing Python SDK
 ===========================
 
-|travis-badge|_ |pypi-badge|_ |readthedocs-badge|_
+|pypi-badge|_ |readthedocs-badge|_
 
 This package allows you to use Qarnot cloud computing service.
 
@@ -44,5 +44,3 @@ The index of the doc is then generated in `doc/_build/html/index.html`
 .. _pypi-badge: https://pypi.python.org/pypi/qarnot/
 .. |readthedocs-badge| image:: https://readthedocs.org/projects/qarnot/badge/?version=latest
 .. _readthedocs-badge: https://qarnot.readthedocs.io/en/latest/
-.. |travis-badge| image:: https://app.travis-ci.com/qarnot/qarnot-sdk-python.svg?branch=master
-.. _travis-badge: https://app.travis-ci.com/qarnot/qarnot-sdk-python

{qarnot-2.13.1 → qarnot-2.15.0}/qarnot/__init__.py

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 
-from .exceptions import QarnotGenericException, UnauthorizedException
+from .exceptions import QarnotGenericException, SecretConflictException, SecretNotFoundException, UnauthorizedException
 from ._util import get_error_message_from_http_response
 
 __all__ = ["task", "connection", "bucket", "pool",
@@ -35,6 +35,13 @@ def raise_on_error(response):
             raise QarnotGenericException(response.text) from value
 
 
+def raise_on_secrets_specific_error(response):
+    if response.status_code == 404:
+        raise SecretNotFoundException()
+    if response.status_code == 409:
+        raise SecretConflictException()
+
+
 def get_url(key, **kwargs):
     """Get and format the url for the given key.
     """
@@ -69,6 +76,8 @@ def get_url(key, **kwargs):
         'pool stderr': '/pools/{uuid}/stderr',  # GET -> pool stderr
         'pool instance stdout': '/pools/{uuid}/stdout/{instanceId}',  # GET -> pool instance stdout
         'pool instance stderr': '/pools/{uuid}/stderr/{instanceId}',  # GET -> pool instance stderr
+        'secrets data': '/secrets-manager/data/{secret_key}',  # GET -> get secret , PUT -> create secret, PATCH -> update secret, DELETE -> delete secret
+        'secrets search': '/secrets-manager/search/{secret_prefix}',  # GET -> lists secrets starting with prefix
         'user': '/info',  # GET -> user info
         'profiles': '/profiles',  # GET -> profiles list
         'profile details': '/profiles/{profile}',  # GET -> profile details
@@ -84,3 +93,4 @@ __version__ = get_versions()['version']  # type: ignore
 del get_versions
 
 from .connection import Connection  # noqa
+from .secrets import Secrets  # noqa

{qarnot-2.13.1 → qarnot-2.15.0}/qarnot/_util.py

@@ -20,11 +20,6 @@ from http.client import responses
 
 import re
 
-_IS_PY2 = bytes is str
-
-if not _IS_PY2:
-    unicode = str
-
 
 def copy_docs(docs_source):
     def decorator(obj):
@@ -43,7 +38,7 @@ def decode(string, encoding='utf-8'):
 
 def is_string(x):
     """Check if x is a string (bytes or unicode)."""
-    return isinstance(x, (str, unicode))
+    return isinstance(x, (str, bytes))
 
 
 def parse_to_timespan_string(value):
@@ -144,7 +139,11 @@ def get_sanitized_bucket_path(path: str, show_warning: bool = True):
 def get_error_message_from_http_response(response: Response, message_is_status_code_if_null: bool = False) -> str:
     error_message = ""
     try:
-        error_message = response.json()['message']
+        error_response = response.json()
+        if 'message' in error_response:
+            error_message = error_response['message']
+        elif 'error' in error_response:
+            error_message = error_response['error']
     except (JSONDecodeError, simpleJsonDecodeError):
         error_message = response.text
     if (error_message is None or error_message == "" or len(error_message) < 1) and message_is_status_code_if_null:

{qarnot-2.13.1 → qarnot-2.15.0}/qarnot/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-20T13:38:48+0200",
+ "date": "2024-08-27T11:29:32+0200",
  "dirty": false,
  "error": null,
- "full-revisionid": "098a8560da06f8c3cf58082163ad5b4f316cd6e5",
- "version": "v2.13.1"
+ "full-revisionid": "8fe176158583e7c2db929a8a25031075991e9e36",
+ "version": "v2.15.0"
 }
 '''  # END VERSION_JSON
 

{qarnot-2.13.1 → qarnot-2.15.0}/qarnot/bucket.py

@@ -227,8 +227,11 @@ class Bucket(Storage):  # pylint: disable=W0223
         :returns: A list of ObjectSummary resources
 
         """
-        bucket = self._connection.s3resource.Bucket(self._uuid)
-        return bucket.objects.all()
+        try:
+            bucket = self._connection.s3resource.Bucket(self._uuid)
+            return [b for b in bucket.objects.all() if b.key is not None]
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot list files. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
 
     def directory(self, directory=''):
         """List files in a directory of the bucket according to prefix.
@@ -239,6 +242,64 @@ class Bucket(Storage):  # pylint: disable=W0223
         bucket = self._connection.s3resource.Bucket(self._uuid)
         return bucket.objects.filter(Prefix=directory)
 
+    def sync_remote_to_local(self, local_directoy, remote_directory=None):
+        """Synchronize a remote directory to a local directory.
+
+        :param str local_directoy: The local directory to use for synchronization
+        :param str remote_directory: path of the directory on remote node (defaults to whole bucket)
+
+        .. warning::
+           Distant changes are reflected on the local filesystem, a file not present on the
+           bucket but in the local directory might be deleted from the local filesystem.
+
+        .. note::
+           The following parameters are used to determine whether
+           synchronization is required :
+
+              * name
+              * size
+              * sha1sum
+        """
+
+        def get_key_for_local(remote_key: str) -> str:
+            if remote_directory:
+                return removeprefix(remote_key, remote_directory).lstrip('/')
+            return remote_key.lstrip('/')
+
+        def removeprefix(target_str: str, prefix: str) -> str:
+            if target_str.startswith(prefix):
+                return target_str[len(prefix):]
+            else:
+                return target_str[:]
+
+        try:
+            if remote_directory:
+                entries = self.directory(remote_directory)
+            else:
+                entries = self.list_files()
+
+            list_files_only = [x for x in entries if not x.key.endswith('/')]
+            list_directories_only = [x for x in entries if x.key.endswith('/')]
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot synchronize. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
+
+        for directory in list_directories_only:
+            if not os.path.isdir(os.path.join(local_directoy, get_key_for_local(directory.key))):
+                os.makedirs(os.path.join(local_directoy, get_key_for_local(directory.key)), exist_ok=True)
+
+        for _, dupes in groupby(sorted(list_files_only, key=attrgetter('e_tag')), attrgetter('e_tag')):
+            file_info = next(dupes)
+            first_file = os.path.join(local_directoy, get_key_for_local(file_info.key))
+            self.get_file(file_info.get()['Body'], local=first_file)  # avoids making a useless HEAD request
+
+            for dupe in dupes:
+                local = os.path.join(local_directoy, get_key_for_local(dupe.key))
+                directory = os.path.dirname(local)
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+                if (os.path.abspath(os.path.realpath(local)) is not os.path.abspath(os.path.realpath(first_file))):
+                    shutil.copy(first_file, local)
+
     def sync_directory(self, directory, verbose=False, remote=None):
         """Synchronize a local directory with the remote buckets.
 
@@ -282,6 +343,7 @@ class Bucket(Storage):  # pylint: disable=W0223
         :param dict files: Dictionary of synchronized files
         :param bool verbose: Print information about synchronization operations
         :param str remote: path of the directory on remote node (defaults to *local*)
+        :raises ~qarnot.exceptions.MissingBucketException: the bucket is not on the server
 
         Dictionary key is the remote file path while value is the local file
         path.
@@ -348,53 +410,56 @@ class Bucket(Storage):  # pylint: disable=W0223
         def objectsummarytocomparable(object_):
             return Comparable(object_.key, object_.e_tag, None)
 
-        localfiles = set()
-        if self._connection._sanitize_bucket_paths:
-            remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
-        for name, filepath in files.items():
-            localfiles.add(localtocomparable(name.replace(os.path.sep, '/'), filepath, remote))
+        try:
+            localfiles = set()
+            if self._connection._sanitize_bucket_paths:
+                remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
+            for name, filepath in files.items():
+                localfiles.add(localtocomparable(name.replace(os.path.sep, '/'), filepath, remote))
 
-        remotefiles = set(map(objectsummarytocomparable, self.list_files()))
+            remotefiles = set(map(objectsummarytocomparable, self.list_files()))
 
-        adds = localfiles - remotefiles
-        removes = remotefiles - localfiles
+            adds = localfiles - remotefiles
+            removes = remotefiles - localfiles
 
-        seen_tags = set()  # To avoid copying the same objects multiple times when renaming
-        for file_ in removes:
-            if remote is not None and not file_.name.startswith(remote):
-                continue
-            renames = (x for x in adds if x.e_tag not in seen_tags and x.e_tag == file_.e_tag
-                       and all(rem.name != x.name for rem in remotefiles))
-            for dup in renames:
-                if verbose:
-                    self._connection.logger.info("Copy", file_.name, "to", dup.name)
-                self.copy_file(file_.name, dup.name)
-            if verbose:
-                self._connection.logger.info("Remove:", file_.name)
-            self.delete_file(file_.name)
-            seen_tags.add(file_.e_tag)
-
-        remotefiles = set(map(objectsummarytocomparable, self.list_files()))
-
-        sadds = sorted(adds, key=lambda x: x.e_tag)
-        groupedadds = (list(g) for _, g in itertools.groupby(sadds, lambda x: x.e_tag))
-
-        for entry in groupedadds:
-            try:
-                rem = next(x for x in remotefiles if x.e_tag == entry[0].e_tag)
-                if rem.name == entry[0].name:
+            seen_tags = set()  # To avoid copying the same objects multiple times when renaming
+            for file_ in removes:
+                if remote is not None and not file_.name.startswith(remote):
                     continue
+                renames = (x for x in adds if x.e_tag not in seen_tags and x.e_tag == file_.e_tag
+                           and all(rem.name != x.name for rem in remotefiles))
+                for dup in renames:
+                    if verbose:
+                        self._connection.logger.info("Copy %s to %s" % (file_.name, dup.name))
+                    self.copy_file(file_.name, dup.name)
                 if verbose:
-                    self._connection.logger.info("Copy", rem.name, "to", entry[0].name)
-                self.copy_file(rem.name, entry[0].name)
-            except StopIteration:
-                if verbose:
-                    self._connection.logger.info("Upload:", entry[0].filepath, '->', entry[0].name)
-                self.add_file(entry[0].filepath, entry[0].name)
-            for link in entry[1:]:  # duplicate files
-                if verbose:
-                    self._connection.logger.info("Copy", entry[0].name, "to", link.name)
-                self.copy_file(entry[0].name, link.name)
+                    self._connection.logger.info("Remove: %s" % file_.name)
+                self.delete_file(file_.name)
+                seen_tags.add(file_.e_tag)
+
+            remotefiles = set(map(objectsummarytocomparable, self.list_files()))
+
+            sadds = sorted(adds, key=lambda x: x.e_tag)
+            groupedadds = (list(g) for _, g in itertools.groupby(sadds, lambda x: x.e_tag))
+
+            for entry in groupedadds:
+                try:
+                    rem = next(x for x in remotefiles if x.e_tag == entry[0].e_tag)
+                    if rem.name == entry[0].name:
+                        continue
+                    if verbose:
+                        self._connection.logger.info("Copy %s to %s" % (rem.name, entry[0].name))
+                    self.copy_file(rem.name, entry[0].name)
+                except StopIteration:
+                    if verbose:
+                        self._connection.logger.info("Upload: %s -> %s" % (entry[0].filepath, entry[0].name))
+                    self.add_file(entry[0].filepath, entry[0].name)
+                for link in entry[1:]:  # duplicate files
+                    if verbose:
+                        self._connection.logger.info("Copy %s to %s" % (entry[0].name, link.name))
+                    self.copy_file(entry[0].name, link.name)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot sync files. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
 
     def add_string(self, string, remote):
         """Add a string on the storage.
@@ -416,31 +481,17 @@ class Bucket(Storage):  # pylint: disable=W0223
             file_ = local_or_file
         dest = remote or os.path.basename(file_.name)
 
-        self._connection.s3client.upload_fileobj(file_, self._uuid, dest, Config=s3_multipart_config)
-        if tobeclosed:
-            file_.close()
+        try:
+            self._connection.s3client.upload_fileobj(file_, self._uuid, dest, Config=s3_multipart_config)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot add string. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
+        finally:
+            if tobeclosed:
+                file_.close()
 
     @_util.copy_docs(Storage.get_all_files)
     def get_all_files(self, output_dir, progress=None):
-        list_files_only = [x for x in self.list_files() if not x.key.endswith('/')]
-        list_directories_only = [x for x in self.list_files() if x.key.endswith('/')]
-
-        for directory in list_directories_only:
-            if not os.path.isdir(os.path.join(output_dir, directory.key.lstrip('/'))):
-                os.makedirs(os.path.join(output_dir, directory.key.lstrip('/')))
-
-        for _, dupes in groupby(sorted(list_files_only, key=attrgetter('e_tag')), attrgetter('e_tag')):
-            file_info = next(dupes)
-            first_file = os.path.join(output_dir, file_info.key.lstrip('/'))
-            self.get_file(file_info.get()['Body'], local=first_file)  # avoids making a useless HEAD request
-
-            for dupe in dupes:
-                local = os.path.join(output_dir, dupe.key.lstrip('/'))
-                directory = os.path.dirname(local)
-                if not os.path.exists(directory):
-                    os.makedirs(directory)
-                if (os.path.abspath(os.path.realpath(local)) is not os.path.abspath(os.path.realpath(first_file))):
-                    shutil.copy(first_file, local)
+        self.sync_remote_to_local(output_dir, None)
 
     @_util.copy_docs(Storage.get_file)
     def get_file(self, remote, local=None, progress=None):
@@ -465,11 +516,14 @@ class Bucket(Storage):  # pylint: disable=W0223
 
     @_util.copy_docs(Storage.copy_file)
     def copy_file(self, source, dest):
-        copy_source = {
-            'Bucket': self._uuid,
-            'Key': source
-        }
-        return self._connection.s3client.copy_object(CopySource=copy_source, Bucket=self._uuid, Key=dest)
+        try:
+            copy_source = {
+                'Bucket': self._uuid,
+                'Key': source
+            }
+            return self._connection.s3client.copy_object(CopySource=copy_source, Bucket=self._uuid, Key=dest)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot copy file {} to {} from bucket {}. Bucket not found.".format(source, dest, err.response['Error']['BucketName'])) from err
 
     @deprecation.deprecated(deprecated_in="2.6.0", removed_in="3.0",
                             current_version=__version__,  # type: ignore
@@ -490,14 +544,20 @@ class Bucket(Storage):  # pylint: disable=W0223
             if hasattr(remote, 'read'):
                 shutil.copyfileobj(remote, data)
             else:
-                self._connection.s3client.download_fileobj(self._uuid, remote, data)
+                try:
+                    self._connection.s3client.download_fileobj(self._uuid, remote, data)
+                except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+                    raise MissingBucketException("Cannot download file {} from bucket {}. Bucket not found.".format(remote, err.response['Error']['BucketName'])) from err
         return local
 
     @_util.copy_docs(Storage.delete_file)
     def delete_file(self, remote):
-        if self._connection._sanitize_bucket_paths:
-            remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
-        self._connection.s3client.delete_object(Bucket=self._uuid, Key=remote)
+        try:
+            if self._connection._sanitize_bucket_paths:
+                remote = _util.get_sanitized_bucket_path(remote, self._connection._show_bucket_warnings)
+            self._connection.s3client.delete_object(Bucket=self._uuid, Key=remote)
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot delete file {} from bucket {}. Bucket not found.".format(remote, err.response['Error']['BucketName'])) from err
 
     @property
     def uuid(self):

{qarnot-2.13.1 → qarnot-2.15.0}/qarnot/exceptions.py

@@ -30,7 +30,9 @@ __all__ = ['QarnotException',
            'NotEnoughCreditsException',
            'MissingBucketException',
            'MaxJobException',
-           'MissingJobException']
+           'MissingJobException',
+           'SecretNotFoundException',
+           'SecretConflictException']
 
 
 class QarnotException(Exception):
@@ -93,3 +95,11 @@ class MissingJobException(Exception):
 
 class MaxJobException(Exception):
     """Max number of jobs reached."""
+
+
+class SecretNotFoundException(Exception):
+    """Requested secret was not found."""
+
+
+class SecretConflictException(Exception):
+    """Secret already exists."""

{qarnot-2.13.1 → qarnot-2.15.0}/qarnot/pool.py

@@ -19,6 +19,7 @@ from typing import Dict, List, Optional
 
 from qarnot.retry_settings import RetrySettings
 from qarnot.forced_network_rule import ForcedNetworkRule
+from qarnot.secrets import SecretsAccessRights
 
 from . import raise_on_error, get_url, _util
 from .bucket import Bucket
@@ -124,6 +125,7 @@ class Pool(object):
         self._privileges: Privileges = Privileges()
         self._default_retry_settings: RetrySettings = RetrySettings()
         self._forced_network_rules: List[ForcedNetworkRule] = []
+        self._secrets_access_rights: SecretsAccessRights = SecretsAccessRights()
 
     @classmethod
     def _retrieve(cls, connection, uuid):
@@ -237,6 +239,8 @@ class Pool(object):
         if 'schedulingType' in json_pool:
             self._scheduling_type = SchedulingType.from_string(json_pool["schedulingType"])
         self._forced_network_rules = [ForcedNetworkRule.from_json(forced_network_dict) for forced_network_dict in json_pool.get("forcedNetworkRules", [])]
+        if 'secretsAccessRights' in json_pool:
+            self._secrets_access_rights = SecretsAccessRights.from_json(json_pool["secretsAccessRights"])
 
     def _to_json(self):
         """Get a dict ready to be json packed from this pool."""
@@ -294,6 +298,9 @@ class Pool(object):
         if self._forced_network_rules is not None:
             json_pool['forcedNetworkRules'] = [x.to_json() for x in self._forced_network_rules]
 
+        if self._secrets_access_rights:
+            json_pool['secretsAccessRights'] = self._secrets_access_rights.to_json()
+
         return json_pool
 
     def submit(self):
@@ -1067,6 +1074,31 @@ class Pool(object):
 
         self._constraints = value
 
+    @property
+    def secrets_access_rights(self):
+        """:type: :class:`~qarnot.secrets.SecretsAccessRights`
+        :getter: Returns the description of the secrets the tasks in this pool will have access to when running.
+        :setter: set the secrets this pool will have access to when running.
+
+        Secrets can be accessible either by exact match on the key or by using a prefix
+        in order to match all the secrets starting with said prefix.
+        """
+        self._update_if_summary()
+        if self._auto_update:
+            self.update()
+
+        return self._secrets_access_rights
+
+    @secrets_access_rights.setter
+    def secrets_access_rights(self, value: SecretsAccessRights):
+        """Setter for secrets access rights
+        """
+        self._update_if_summary()
+        if self._auto_update:
+            self.update()
+
+        self._secrets_access_rights = value
+
     @property
     def forced_network_rules(self):
         """:type: list{:class:`~qarnot.forced_network_rule.ForcedNetworkRule`}