qarnot 2.14.5__py3-none-any.whl → 2.16.0__py3-none-any.whl

qarnot/_util.py

@@ -20,11 +20,6 @@ from http.client import responses
 
 import re
 
-_IS_PY2 = bytes is str
-
-if not _IS_PY2:
-    unicode = str
-
 
 def copy_docs(docs_source):
     def decorator(obj):
@@ -43,7 +38,7 @@ def decode(string, encoding='utf-8'):
 
 def is_string(x):
     """Check if x is a string (bytes or unicode)."""
-    return isinstance(x, (str, unicode))
+    return isinstance(x, (str, bytes))
 
 
 def parse_to_timespan_string(value):

qarnot/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-03-29T14:48:51+0100",
+ "date": "2025-02-07T16:27:29+0100",
  "dirty": false,
  "error": null,
- "full-revisionid": "ff28a98ad04060443b7c6b1f487e492c61e31be1",
- "version": "v2.14.5"
+ "full-revisionid": "23daecdc71207c207e56afae3e55190f0734bd0d",
+ "version": "v2.16.0"
 }
 '''  # END VERSION_JSON
 

qarnot/bucket.py

@@ -242,6 +242,64 @@ class Bucket(Storage):  # pylint: disable=W0223
         bucket = self._connection.s3resource.Bucket(self._uuid)
         return bucket.objects.filter(Prefix=directory)
 
+    def sync_remote_to_local(self, local_directoy, remote_directory=None):
+        """Synchronize a remote directory to a local directory.
+
+        :param str local_directoy: The local directory to use for synchronization
+        :param str remote_directory: path of the directory on remote node (defaults to whole bucket)
+
+        .. warning::
+           Distant changes are reflected on the local filesystem, a file not present on the
+           bucket but in the local directory might be deleted from the local filesystem.
+
+        .. note::
+           The following parameters are used to determine whether
+           synchronization is required :
+
+              * name
+              * size
+              * sha1sum
+        """
+
+        def get_key_for_local(remote_key: str) -> str:
+            if remote_directory:
+                return removeprefix(remote_key, remote_directory).lstrip('/')
+            return remote_key.lstrip('/')
+
+        def removeprefix(target_str: str, prefix: str) -> str:
+            if target_str.startswith(prefix):
+                return target_str[len(prefix):]
+            else:
+                return target_str[:]
+
+        try:
+            if remote_directory:
+                entries = self.directory(remote_directory)
+            else:
+                entries = self.list_files()
+
+            list_files_only = [x for x in entries if not x.key.endswith('/')]
+            list_directories_only = [x for x in entries if x.key.endswith('/')]
+        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
+            raise MissingBucketException("Cannot synchronize. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
+
+        for directory in list_directories_only:
+            if not os.path.isdir(os.path.join(local_directoy, get_key_for_local(directory.key))):
+                os.makedirs(os.path.join(local_directoy, get_key_for_local(directory.key)), exist_ok=True)
+
+        for _, dupes in groupby(sorted(list_files_only, key=attrgetter('e_tag')), attrgetter('e_tag')):
+            file_info = next(dupes)
+            first_file = os.path.join(local_directoy, get_key_for_local(file_info.key))
+            self.get_file(file_info.get()['Body'], local=first_file)  # avoids making a useless HEAD request
+
+            for dupe in dupes:
+                local = os.path.join(local_directoy, get_key_for_local(dupe.key))
+                directory = os.path.dirname(local)
+                if not os.path.exists(directory):
+                    os.makedirs(directory)
+                if (os.path.abspath(os.path.realpath(local)) is not os.path.abspath(os.path.realpath(first_file))):
+                    shutil.copy(first_file, local)
+
     def sync_directory(self, directory, verbose=False, remote=None):
         """Synchronize a local directory with the remote buckets.
 
@@ -433,28 +491,7 @@ class Bucket(Storage):  # pylint: disable=W0223
 
     @_util.copy_docs(Storage.get_all_files)
     def get_all_files(self, output_dir, progress=None):
-        try:
-            list_files_only = [x for x in self.list_files() if not x.key.endswith('/')]
-            list_directories_only = [x for x in self.list_files() if x.key.endswith('/')]
-        except self._connection.s3resource.meta.client.exceptions.NoSuchBucket as err:
-            raise MissingBucketException("Cannot get files. Bucket {} not found.".format(err.response['Error']['BucketName'])) from err
-
-        for directory in list_directories_only:
-            if not os.path.isdir(os.path.join(output_dir, directory.key.lstrip('/'))):
-                os.makedirs(os.path.join(output_dir, directory.key.lstrip('/')))
-
-        for _, dupes in groupby(sorted(list_files_only, key=attrgetter('e_tag')), attrgetter('e_tag')):
-            file_info = next(dupes)
-            first_file = os.path.join(output_dir, file_info.key.lstrip('/'))
-            self.get_file(file_info.get()['Body'], local=first_file)  # avoids making a useless HEAD request
-
-            for dupe in dupes:
-                local = os.path.join(output_dir, dupe.key.lstrip('/'))
-                directory = os.path.dirname(local)
-                if not os.path.exists(directory):
-                    os.makedirs(directory)
-                if (os.path.abspath(os.path.realpath(local)) is not os.path.abspath(os.path.realpath(first_file))):
-                    shutil.copy(first_file, local)
+        self.sync_remote_to_local(output_dir, None)
 
     @_util.copy_docs(Storage.get_file)
     def get_file(self, remote, local=None, progress=None):

qarnot/connection.py

@@ -196,7 +196,7 @@ class Connection(object):
 
         user = self.user_info
         session = boto3.session.Session()
-        conf = botocore.config.Config(user_agent=self._version)
+        conf = botocore.config.Config(user_agent=self._version, request_checksum_calculation="when_required")
 
         should_verify_or_certificate_path = True
         if storage_unsafe:

qarnot/forced_constant.py

@@ -0,0 +1,44 @@
+from typing import Optional, Union, Dict
+from enum import Enum
+
+
+class ForcedConstantAccess(Enum):
+    ReadWrite = "ReadWrite"
+    ReadOnly = "ReadOnly"
+
+
+class ForcedConstant(object):
+    """Forced Constant Information
+
+    .. note:: For internal usage only
+    """
+
+    def __init__(self, forced_value: str, force_export_in_environment: Optional[bool] = None, access: Optional[ForcedConstantAccess] = None):
+        self.forced_value = forced_value
+        """:type: :class:`str`
+
+        Forced value for the constant."""
+
+        self.force_export_in_environment = force_export_in_environment
+        """:type: :class:`bool`
+
+        Whether the constant should be forced in the execution environment or not."""
+
+        self.access = access
+        """:type: :class:`~qarnot.forced_constant.ForcedConstantAccess`
+
+        The access level of the constant: ReadOnly or ReadWrite."""
+
+    def to_json(self, name: str):
+        result: Dict[str, Union[str, bool]] = {
+            "constantName": name,
+            "forcedValue": self.forced_value,
+        }
+
+        if self.force_export_in_environment is not None:
+            result["forceExportInEnvironment"] = self.force_export_in_environment
+
+        if self.access is not None:
+            result["access"] = self.access.value
+
+        return result

qarnot/forced_network_rule.py

@@ -81,7 +81,7 @@ class ForcedNetworkRule(object):
         """Create the forced network rule from json.
 
         :param dict json: Dictionary representing the forced network rule
-        :returns: The created :class:`~qarnot.retry_settings.ForcedNetworkRule`
+        :returns: The created :class:`~qarnot.forced_network_rule.ForcedNetworkRule`
         """
 
         inbound: bool = bool(json["inbound"])

qarnot/pool.py

@@ -23,6 +23,7 @@ from qarnot.secrets import SecretsAccessRights
 
 from . import raise_on_error, get_url, _util
 from .bucket import Bucket
+from .forced_constant import ForcedConstant
 from .status import Status
 from .hardware_constraint import HardwareConstraint
 from .scheduling_type import SchedulingType
@@ -77,6 +78,7 @@ class Pool(object):
               with :meth:`qarnot.connection.Connection.retrieve_profile`.
         """
         self._constraints: Dict[str, str] = {}
+        self._forced_constants: Dict[str, ForcedConstant] = {}
         self._labels: Dict[str, str] = {}
         self._auto_update = True
         self._last_auto_update_state = self._auto_update
@@ -252,6 +254,10 @@ class Pool(object):
             {'key': key, 'value': value}
             for key, value in self._constraints.items()
         ]
+        forced_const_list = [
+            value.to_json(key)
+            for key, value in self._forced_constants.items()
+        ]
 
         elastic_dict = {
             "isElastic": self._is_elastic,
@@ -267,6 +273,7 @@ class Pool(object):
             'name': self._name,
             'profile': self._profile,
             'constants': const_list,
+            'forcedConstants': forced_const_list,
             'constraints': constr_list,
             'instanceCount': self._instancecount,
             'tags': self._tags,
@@ -1099,6 +1106,31 @@ class Pool(object):
 
         self._secrets_access_rights = value
 
+    @property
+    def forced_constants(self):
+        """:type: dictionary{:class:`str` : :class:`~qarnot.forced_constant.ForcedConstant`}
+        :getter: Returns this pool's forced constants dictionary.
+        :setter: set the pool's forced constants dictionary.
+
+        Update the forced constants if needed.
+        Forced constants are reserved for internal use.
+        """
+        self._update_if_summary()
+        if self._auto_update:
+            self.update()
+
+        return self._forced_constants
+
+    @forced_constants.setter
+    def forced_constants(self, value: Dict[str, "ForcedConstant"]):
+        """Setter for forced_constants
+        """
+        self._update_if_summary()
+        if self._auto_update:
+            self.update()
+
+        self._forced_constants = value
+
     @property
     def forced_network_rules(self):
         """:type: list{:class:`~qarnot.forced_network_rule.ForcedNetworkRule`}
@@ -1116,7 +1148,7 @@ class Pool(object):
 
     @forced_network_rules.setter
     def forced_network_rules(self, value: List["ForcedNetworkRule"]):
-        """Setter for forced_constants
+        """Setter for forced_network_rules
         """
         if self.uuid is not None:
             raise AttributeError("can't set attribute on a launched pool")

qarnot/secrets.py

@@ -97,7 +97,7 @@ class SecretsAccessRights(object):
 
         :param by_secret: the list of secrets the task will have access to, described using an exact key match
         :type by_secret: `List[~qarnot.secrets.SecretAccessRightBySecret]`
-        :param by_prefix:the list of secrets the task will have access to, described using a prefix key match
+        :param by_prefix: the list of secrets the task will have access to, described using a prefix key match
         :type by_prefix: `List[~qarnot.secrets.SecretAccessRightByPrefix]`
         """
         self._by_secret: List[SecretAccessRightBySecret] = by_secret or []

qarnot/task.py

@@ -19,7 +19,6 @@ from os import makedirs, path
 import time
 import warnings
 import sys
-from enum import Enum
 from typing import Dict, Optional, Union, List, Any, Callable
 
 from qarnot.retry_settings import RetrySettings
@@ -29,6 +28,7 @@ from qarnot.secrets import SecretsAccessRights
 from . import get_url, raise_on_error, _util
 from .status import Status
 from .hardware_constraint import HardwareConstraint
+from .forced_constant import ForcedConstant
 from .scheduling_type import SchedulingType
 from .privileges import Privileges
 from .bucket import Bucket
@@ -458,6 +458,9 @@ class Task(object):
         if 'resultBucket' in json_task and json_task['resultBucket']:
             self._result_object_id = json_task['resultBucket']
 
+        if 'ResultsCacheTTLSec' in json_task and self._result_object is not None:
+            self._result_object._cache_ttl_sec = json_task['ResultsCacheTTLSec']
+
         if 'status' in json_task:
             self._status = json_task['status']
         self._creation_date = _util.parse_datetime(json_task['creationDate'])
@@ -1340,7 +1343,7 @@ class Task(object):
 
     @property
     def forced_constants(self):
-        """:type: dictionary{:class:`str` : :class:`~qarnot.task.ForcedConstant`}
+        """:type: dictionary{:class:`str` : :class:`~qarnot.forced_constant.ForcedConstant`}
         :getter: Returns this task's forced constants dictionary.
         :setter: set the task's forced constants dictionary.
 
@@ -1380,7 +1383,7 @@ class Task(object):
 
     @forced_network_rules.setter
     def forced_network_rules(self, value: List["ForcedNetworkRule"]):
-        """Setter for forced_constants
+        """Setter for forced_network_rules
         """
         if self.uuid is not None:
             raise AttributeError("can't set attribute on a launched task")
@@ -1815,6 +1818,8 @@ class Task(object):
 
         if self._result_object is not None:
             json_task['resultBucket'] = self._result_object.uuid
+            if self._result_object._cache_ttl_sec is not None:
+                json_task['ResultsCacheTTLSec'] = self._result_object._cache_ttl_sec
 
         if self._advanced_range is not None:
             json_task['advancedRanges'] = self._advanced_range
@@ -1975,45 +1980,3 @@ class BulkTaskResponse(object):
             return ', '.join("{0}={1}".format(key, val) for (key, val) in self.__dict__.items())
         else:
             return ', '.join("{0}={1}".format(key, val) for (key, val) in self.__dict__.iteritems())  # pylint: disable=no-member
-
-
-class ForcedConstantAccess(Enum):
-    ReadWrite = "ReadWrite"
-    ReadOnly = "ReadOnly"
-
-
-class ForcedConstant(object):
-    """Forced Constant Information
-
-    .. note:: For internal usage only
-    """
-
-    def __init__(self, forced_value: str, force_export_in_environment: Optional[bool] = None, access: Optional[ForcedConstantAccess] = None):
-        self.forced_value = forced_value
-        """:type: :class:`str`
-
-        Forced value for the constant."""
-
-        self.force_export_in_environment = force_export_in_environment
-        """:type: :class:`bool`
-
-        Whether the constant should be forced in the execution environment or not."""
-
-        self.access = access
-        """:type: :class:`~qarnot.task.ForcedConstantAccess`
-
-        The access level of the constant: ReadOnly or ReadWrite."""
-
-    def to_json(self, name: str):
-        result: Dict[str, Union[str, bool]] = {
-            "constantName": name,
-            "forcedValue": self.forced_value,
-        }
-
-        if self.force_export_in_environment is not None:
-            result["forceExportInEnvironment"] = self.force_export_in_environment
-
-        if self.access is not None:
-            result["access"] = self.access.value
-
-        return result

{qarnot-2.14.5.dist-info → qarnot-2.16.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: qarnot
-Version: 2.14.5
+Version: 2.16.0
 Summary: Qarnot Computing SDK
 Home-page: https://computing.qarnot.com
 Author: Qarnot computing
@@ -17,10 +17,19 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.6
 License-File: LICENSE
 Requires-Dist: requests
-Requires-Dist: boto3
+Requires-Dist: boto3>=1.36
 Requires-Dist: wheel
 Requires-Dist: deprecation
 Requires-Dist: simplejson
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 Qarnot computing Python SDK
 ===========================

{qarnot-2.14.5.dist-info → qarnot-2.16.0.dist-info}/RECORD

@@ -1,28 +1,29 @@
 qarnot/__init__.py,sha256=pLDjFGU-0WROKi7nb73a7M3tafVYpgM4Tj9iOwIx-tE,5092
 qarnot/_filter.py,sha256=J--0lOY2rverPEE3zrvuipYkOd9T_4HrW4eVNJqheac,10109
 qarnot/_retry.py,sha256=4QdtI5Y_Jshja8zDxhx_g17dzABCIUGXQsjcPl65u7g,890
-qarnot/_util.py,sha256=e8Luej_ZNqjH3autsqgGz9xGw_vDl-m6qxzGNOi3kUQ,5734
-qarnot/_version.py,sha256=nt_FGe1ljkcgIy8ysOp0AqmO22RW1L8z7Tn3aHCViys,499
+qarnot/_util.py,sha256=fmAq55CmfDTTB7yaOLspayC3Mvre7SCzp0YwWsnwpDA,5673
+qarnot/_version.py,sha256=LyIkPD7PhRJmr2rXGj5uL8sQ6xl7teOzkl-Mv5wd4FE,499
 qarnot/advanced_bucket.py,sha256=_uKSxRULmpYj4IBbSzssfUu5lWj8ZICUjxs-MvI4lvs,7923
-qarnot/bucket.py,sha256=R4PHl5FQeQsGYmUqj2i6fqUXJH2wz_XmdEKgEM0-rjc,22958
-qarnot/connection.py,sha256=-CPPZnGAYNCTyTIsi_wBlX6dnAbIh6Llvt3WCQyQG1s,44848
+qarnot/bucket.py,sha256=kXA16XwL2NxGqqN0z6SeywN18MucDGX8UFgISMKJj5k,24361
+qarnot/connection.py,sha256=GM8Hi8bYiGgKsccx8GPBXejJ8CsjC10AjSIIYOkT4cE,44894
 qarnot/error.py,sha256=LB_fXUiFZMkF5dgSi7QKpw-arELV043G5qu4HN-yA1k,722
 qarnot/exceptions.py,sha256=yt_iwCw_9pFdoKeOTxsr05kqW5Gu-th3gSfos5zI26g,2729
-qarnot/forced_network_rule.py,sha256=SvKg41BEvmIT1f0WXWIvzaWJDAw0lWXQH1gVxEKFz3M,4466
+qarnot/forced_constant.py,sha256=-i4b_JO10YiWuJ7Q0bmWE_TEwtf8qSeLlkMTAbH55EM,1309
+qarnot/forced_network_rule.py,sha256=diGTZEjWOSR2QbtC00tjf_7zXw0c_D6OIV3ujVq4zvg,4471
 qarnot/hardware_constraint.py,sha256=qJukigRpv8kT9EAg5am0j3ypHtuLRsbcIQ5bUyrMS3c,13658
 qarnot/helper.py,sha256=SAMcXt2GEcPuhV_ZzTOYs1Ucy4OdYGHVur472k36uoI,1498
 qarnot/job.py,sha256=d5rH7LUY-QgQfZabsRV0jHVC6KhPezFy-mkMZw9XhKQ,19444
 qarnot/paginate.py,sha256=DaUYDPAS0M8hf0hph8GuMSzTASymSUOpp-WqbfevX-s,1387
-qarnot/pool.py,sha256=C6ps9O4NRBCNQvjlS9Ve-wOlxYo5bUaNGdMk-eraEnc,54167
+qarnot/pool.py,sha256=NK341qpU37kGPEwZiyffGBUVSTYNBwNFhtehV4E2Byc,55251
 qarnot/privileges.py,sha256=yUnIbAyb9G410nifFdUXEGvm-FHR_PQYGh-OGq_iiwk,1892
 qarnot/retry_settings.py,sha256=NSJB54LLdjdD3w7t5G0gZVQD-SDl_2DUUwJphjcfgfg,2504
 qarnot/scheduling_type.py,sha256=YZgmeAgWo_tQorwM1aqHUHq2xEZytKWB3hTViYn-hwQ,2175
-qarnot/secrets.py,sha256=LJo4B0K5X2CPAIdapfU_HIv-eGoFPmMU3HcmmhA7PZ4,12329
+qarnot/secrets.py,sha256=apGb6dD2kyNVMGRf402wo7kPQKF7EtRZabl8kxHxqRA,12330
 qarnot/status.py,sha256=zzB0bFeiWMv-XiHuDH7Zj12sWhp1TYFVmQtNNt3Gzmg,12084
 qarnot/storage.py,sha256=jAist_J_6yzmRrkF5jqYNG3mhEP_y7KqCdNv4dJcHuM,6929
-qarnot/task.py,sha256=1-NjnZ5bMc4IJk9RbKgVCqtlg53vqSjkcXMyW8msl2s,75101
-qarnot-2.14.5.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-qarnot-2.14.5.dist-info/METADATA,sha256=Sy40-M4D2AXs3Pjgfj1MqkxEBfPqm8pgpZLazOCPxLU,2335
-qarnot-2.14.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qarnot-2.14.5.dist-info/top_level.txt,sha256=acRyoLZNyf_kuGTwHQgfZv2MfdTcZstyNjBhOxFtHzU,7
-qarnot-2.14.5.dist-info/RECORD,,
+qarnot/task.py,sha256=FZd8sZeT7DZa9_-ldHTy2qfAzx1Q6Hd9l3dHXnVWLA8,74215
+qarnot-2.16.0.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+qarnot-2.16.0.dist-info/METADATA,sha256=TSdbTmPqnwqJSC02Wrt4JoolHDb1fKQvc8HwYzuSVUM,2521
+qarnot-2.16.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+qarnot-2.16.0.dist-info/top_level.txt,sha256=acRyoLZNyf_kuGTwHQgfZv2MfdTcZstyNjBhOxFtHzU,7
+qarnot-2.16.0.dist-info/RECORD,,

{qarnot-2.14.5.dist-info → qarnot-2.16.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any