pyPreservica 2.9.4__py3-none-any.whl → 3.0.1__py3-none-any.whl

pyPreservica/__init__.py

@@ -23,6 +23,6 @@ from .mdformsAPI import MetadataGroupsAPI, Group, GroupField, GroupFieldType
 __author__ = "James Carr (drjamescarr@gmail.com)"
 
 # Version of the pyPreservica package
-__version__ = "2.9.4"
+__version__ = "3.0.1"
 
 __license__ = "Apache License Version 2.0"

pyPreservica/common.py

@@ -880,10 +880,10 @@ class AuthenticatedAPI:
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook=None):
+                 protocol: str = "https", request_hook=None, credentials_path: str = 'credentials.properties'):
 
         config = configparser.ConfigParser(interpolation=configparser.Interpolation())
-        config.read('credentials.properties', encoding='utf-8')
+        config.read(os.path.relpath(credentials_path), encoding='utf-8')
         self.session: Session = requests.Session()
 
         if request_hook is not None:

pyPreservica/contentAPI.py

@@ -35,11 +35,12 @@ class Field:
 
 class ContentAPI(AuthenticatedAPI):
 
-    def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False,
-                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None):
+    def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
+                 use_shared_secret: bool = False, two_fa_secret_key: str = None,
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
         self.callback = None
 
     class SearchResult:

pyPreservica/entityAPI.py

@@ -35,10 +35,10 @@ class EntityAPI(AuthenticatedAPI):
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
 
         xml.etree.ElementTree.register_namespace("oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/")
         xml.etree.ElementTree.register_namespace("ead", "urn:isbn:1-931666-22-9")
@@ -2281,7 +2281,7 @@ class EntityAPI(AuthenticatedAPI):
             logger.error(exception)
             raise exception
 
-    def delete_asset(self, asset: Asset, operator_comment: str, supervisor_comment: str):
+    def delete_asset(self, asset: Asset, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
         Delete an asset from the repository
 
@@ -2290,11 +2290,11 @@ class EntityAPI(AuthenticatedAPI):
         :param supervisor_comment:  The supervisor comment on the deletion
         """
         if isinstance(asset, Asset):
-            return self._delete_entity(asset, operator_comment, supervisor_comment)
+            return self._delete_entity(asset, operator_comment, supervisor_comment, credentials_path)
         else:
             raise RuntimeError("delete_asset only deletes assets")
 
-    def delete_folder(self, folder: Folder, operator_comment: str, supervisor_comment: str):
+    def delete_folder(self, folder: Folder, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
          Delete an asset from the repository
 
@@ -2304,11 +2304,11 @@ class EntityAPI(AuthenticatedAPI):
          :param supervisor_comment:  The supervisor comment on the deletion
          """
         if isinstance(folder, Folder):
-            return self._delete_entity(folder, operator_comment, supervisor_comment)
+            return self._delete_entity(folder, operator_comment, supervisor_comment, credentials_path)
         else:
             raise RuntimeError("delete_folder only deletes folders")
 
-    def _delete_entity(self, entity: Entity, operator_comment: str, supervisor_comment: str):
+    def _delete_entity(self, entity: Entity, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
         Delete an asset from the repository
 
@@ -2319,7 +2319,7 @@ class EntityAPI(AuthenticatedAPI):
 
         # check manager password is available:
         config = configparser.ConfigParser()
-        config.read('credentials.properties', encoding='utf-8')
+        config.read(credentials_path, encoding='utf-8')
         try:
             manager_username = config['credentials']['manager.username']
             manager_password = config['credentials']['manager.password']
@@ -2373,7 +2373,7 @@ class EntityAPI(AuthenticatedAPI):
                                        headers=headers)
         elif request.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
-            return self._delete_entity(entity, operator_comment, supervisor_comment)
+            return self._delete_entity(entity, operator_comment, supervisor_comment, credentials_path)
         if request.status_code == requests.codes.unprocessable:
             logger.error(request.content.decode('utf-8'))
             raise RuntimeError(request.status_code, "no active workflow context for full deletion exists in the system")
@@ -2385,4 +2385,4 @@ class EntityAPI(AuthenticatedAPI):
         exception = HTTPException(entity.reference, request.status_code, request.url,
                                   "_delete_entity", request.content.decode('utf-8'))
         logger.error(exception)
-        raise exception
+        raise exception

pyPreservica/mdformsAPI.py

@@ -134,12 +134,13 @@ def _json_from_object_(group: Group) -> dict:
 
 
 class MetadataGroupsAPI(AuthenticatedAPI):
+
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
 
         xml.etree.ElementTree.register_namespace("oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/")
         xml.etree.ElementTree.register_namespace("ead", "urn:isbn:1-931666-22-9")

pyPreservica/parAPI.py

@@ -23,43 +23,7 @@ def __get_contents__(document) -> AnyStr:
         return json.dumps(json.loads(document))
 
 
-class PreservationActionRegistry:
-
-    def __init__(self, server: str = None, username: str = None, password: str = None, protocol: str = 'https'):
-        self.protocol = protocol
-        self.session = requests.Session()
-        self.session.headers.update({'accept': 'application/json;charset=UTF-8'})
-        config = configparser.ConfigParser()
-        config.read('credentials.properties')
-        if not server:
-            server = os.environ.get('PRESERVICA_SERVER')
-            if server is None:
-                try:
-                    server = config['credentials']['server']
-                except KeyError:
-                    pass
-        if not server:
-            msg = "No valid server found in method arguments, environment variables or credentials.properties file"
-            logger.error(msg)
-            raise RuntimeError(msg)
-        else:
-            self.server = server
-        if not username:
-            username = os.environ.get('PRESERVICA_USERNAME')
-            if username is None:
-                try:
-                    username = config['credentials']['username']
-                except KeyError:
-                    pass
-        self.username = username
-        if not password:
-            password = os.environ.get('PRESERVICA_PASSWORD')
-            if password is None:
-                try:
-                    password = config['credentials']['password']
-                except KeyError:
-                    pass
-        self.password = password
+class PreservationActionRegistry(AuthenticatedAPI):
 
     def format_family(self, guid: str) -> str:
         return self.__guid__(guid, "format-families")

pyPreservica/retentionAPI.py

@@ -58,9 +58,10 @@ class RetentionPolicy:
 class RetentionAPI(AuthenticatedAPI):
 
     def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False,
-                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None):
+                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
+
         if self.major_version < 7 and self.minor_version < 2:
             raise RuntimeError("Retention API is only available when connected to a v6.2 System")
 

pyPreservica/uploadAPI.py

@@ -13,20 +13,22 @@ import shutil
 import tempfile
 import uuid
 import xml
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from time import sleep
 from xml.dom import minidom
 from xml.etree import ElementTree
 from xml.etree.ElementTree import Element, SubElement
 
 import boto3
+import botocore
 import s3transfer.tasks
 import s3transfer.upload
-
+from botocore.session import get_session
 from boto3.s3.transfer import TransferConfig, S3Transfer
 from botocore.config import Config
 from botocore.credentials import RefreshableCredentials
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError
+from dateutil.tz import tzlocal
 from s3transfer import S3UploadFailedError
 from tqdm import tqdm
 
@@ -1916,10 +1918,32 @@ class UploadAPI(AuthenticatedAPI):
             'mode': 'adaptive'
         }
 
-        s3_client = boto3.client('s3', endpoint_url=endpoint, aws_access_key_id=self.token,
-                                 aws_secret_access_key="NOT_USED",
+        def new_credentials():
+            metadata: dict = {}
+            metadata['access_key'] =  self.__token__()
+            metadata['secret_key'] = "NOT_USED"
+            metadata['token'] = ""
+            metadata["expiry_time"] = (datetime.now(tzlocal()) + timedelta(minutes=12)).isoformat()
+            logger.info("Refreshing credentials at: " + str(datetime.now(tzlocal())))
+            return metadata
+
+        session = get_session()
+
+        session_credentials = RefreshableCredentials.create_from_metadata(
+            metadata=new_credentials(),
+            refresh_using=new_credentials,
+            advisory_timeout = 4 * 60,
+            mandatory_timeout = 12 * 60,
+            method = 'Preservica'
+        )
+
+        autorefresh_session = boto3.Session(botocore_session=session)
+
+        session._credentials = session_credentials
+
+        s3_client = autorefresh_session.client('s3', endpoint_url=endpoint,
                                  config=Config(s3={'addressing_style': 'path'}, read_timeout=120, connect_timeout=120,
-                                               retries=retries, tcp_keepalive=True))
+                                 retries=retries, tcp_keepalive=True))
 
         metadata = {}
         if folder is not None:
@@ -1932,21 +1956,48 @@ class UploadAPI(AuthenticatedAPI):
             try:
                 key_id = str(uuid.uuid4()) + ".zip"
 
+
+                # how big is the package
+                package_size = os.path.getsize(path_to_zip_package)
+                if package_size > 1 * GB:
+                    transfer_config.multipart_chunksize = 16 * MB   ## Min 64 Chunks
+                if package_size > 8 * GB:
+                    transfer_config.multipart_chunksize = 32 * MB   ## Min 256 Chunks
+                if package_size > 24 * GB:
+                    transfer_config.multipart_chunksize = 48 * MB   ## Min 512 Chunks
+                if package_size > 48 * GB:
+                    transfer_config.multipart_chunksize = 64 * MB
+
+                logger.info("Using Multipart Chunk Size: " + str(transfer_config.multipart_chunksize))
+
                 transfer = S3Transfer(client=s3_client, config=transfer_config)
 
                 transfer.PutObjectTask = PutObjectTask
                 transfer.CompleteMultipartUploadTask = CompleteMultipartUploadTask
                 transfer.upload_file = upload_file
 
-                response = transfer.upload_file(self=transfer, filename=path_to_zip_package, bucket=bucket, key=key_id,
+
+                response = transfer.upload_file(self=transfer, filename=path_to_zip_package, bucket=bucket,
+                                                key=key_id,
                                                 extra_args=metadata,
                                                 callback=callback)
 
+
                 if delete_after_upload:
                     os.remove(path_to_zip_package)
 
                 return response['ResponseMetadata']['HTTPHeaders']['preservica-progress-token']
 
-            except ClientError as e:
-                logger.error(e)
-                raise e
+            except (NoCredentialsError, PartialCredentialsError) as ex:
+                logger.error(ex)
+                raise ex
+
+            except ClientError as ex:
+                logger.error(ex)
+                raise ex
+
+
+
+
+
+

pyPreservica/workflowAPI.py

@@ -81,10 +81,10 @@ class WorkflowAPI(AuthenticatedAPI):
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
         self.base_url = "sdb/rest/workflow"
 
     def get_workflow_contexts_by_type(self, workflow_type: str):

{pyPreservica-2.9.4.dist-info → pyPreservica-3.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyPreservica
-Version: 2.9.4
+Version: 3.0.1
 Summary: Python library for the Preservica API
 Home-page: https://pypreservica.readthedocs.io/
 Author: James Carr

pyPreservica-3.0.1.dist-info/RECORD

@@ -0,0 +1,19 @@
+pyPreservica/__init__.py,sha256=6lYm_cDE5Z9aKXe_UfgNabaGq1MoIDPWUb16GgDGeS0,1177
+pyPreservica/adminAPI.py,sha256=511bc5KtrCAXbDyBk39dmDnxUVDaOu6xaiyu0jYhxa4,37781
+pyPreservica/authorityAPI.py,sha256=Eule8g6LXr8c8SFcJgpRah4lH1FgevUItO5HhHDEaZE,9172
+pyPreservica/common.py,sha256=YvONehtYu2StoPY2oydWy86-zsyIK-r2FD79TnLi01Q,37707
+pyPreservica/contentAPI.py,sha256=z_IwndqzpTMtsDKUgneqWec5YPhi2Yb110Z-4tC42qU,22182
+pyPreservica/entityAPI.py,sha256=ljqiN2EjF20xprjrXjcb8OACtrgKn2K0Kl11uqh7uQU,119261
+pyPreservica/mdformsAPI.py,sha256=cTHxHgPV-EZAFN6C1JfnxrVNv3FLzqI1SV_fb5ZOxeE,19196
+pyPreservica/monitorAPI.py,sha256=HD-PUPdSI9wGAa07e2_2_-FLINH8PoWUwpFogz7F-j4,6269
+pyPreservica/opex.py,sha256=ccra1S4ojUXS3PlbU8WfxajOkJrwG4OykBnNrYP_jus,4875
+pyPreservica/parAPI.py,sha256=f0ZUxLd0U-BW6kBx5K7W2Pv7NjG3MkTNydmxQ3U1ZVE,9296
+pyPreservica/retentionAPI.py,sha256=F6okFSyqtnLhfMbcyChd_5V-D_PAxNLwyx8XohH2DEM,24840
+pyPreservica/uploadAPI.py,sha256=vIji6AUajiSvA2wuemHvU6RqHFd7PlKQToypIGx3KMw,98537
+pyPreservica/webHooksAPI.py,sha256=_K3KUOsmwYf8qMa-mD47sAmNUW7Pzb9oKVpS0VoSbC0,6827
+pyPreservica/workflowAPI.py,sha256=OcOiiUdrQerbPllrkj1lWpmuW0jTuyyV0urwPSYcd_U,17561
+pyPreservica-3.0.1.dist-info/LICENSE.txt,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+pyPreservica-3.0.1.dist-info/METADATA,sha256=SKMkfDkZMetZdqGhV_lxqqDT8meV1mJwTXLpzuSNWN8,2779
+pyPreservica-3.0.1.dist-info/WHEEL,sha256=YiKiUUeZQGmGJoR_0N1Y933DOBowq4AIvDe2-UIy8E4,91
+pyPreservica-3.0.1.dist-info/top_level.txt,sha256=iIBh6NAznYQHOV8mv_y_kGKSDITek9rANyFDwJsbU-c,13
+pyPreservica-3.0.1.dist-info/RECORD,,

pyPreservica-2.9.4.dist-info/RECORD

@@ -1,19 +0,0 @@
-pyPreservica/__init__.py,sha256=dDaBZZwmdmrfqa4DsObHFxNfXtscI8P13o7RkqUs0IU,1177
-pyPreservica/adminAPI.py,sha256=511bc5KtrCAXbDyBk39dmDnxUVDaOu6xaiyu0jYhxa4,37781
-pyPreservica/authorityAPI.py,sha256=Eule8g6LXr8c8SFcJgpRah4lH1FgevUItO5HhHDEaZE,9172
-pyPreservica/common.py,sha256=qVmH5xazlxX02pZaYAX1GT-k0fDuzF8JmEIrYBGiTtw,37648
-pyPreservica/contentAPI.py,sha256=6L7z5KZZDuBBLTm9GQ9RLOhugAawPnuQIgv1NZf0vIU,22060
-pyPreservica/entityAPI.py,sha256=BxrJ-irL0-ohZGVd5uduvFgDgMvrB3Xf2pzTaZ00-Cg,118999
-pyPreservica/mdformsAPI.py,sha256=v_PKjddWHr2-Lo3zqHXikupq4PJvyrwZEEhPHeKDdGY,19126
-pyPreservica/monitorAPI.py,sha256=HD-PUPdSI9wGAa07e2_2_-FLINH8PoWUwpFogz7F-j4,6269
-pyPreservica/opex.py,sha256=ccra1S4ojUXS3PlbU8WfxajOkJrwG4OykBnNrYP_jus,4875
-pyPreservica/parAPI.py,sha256=bgaQvYfWNnzdD7ibKMV3ZV85pNkEdSoLsgVigoiFFfw,10771
-pyPreservica/retentionAPI.py,sha256=31yKHbatxj9kt5vbqyTeg98nrMotd2iL4jsiNhcOLg4,24770
-pyPreservica/uploadAPI.py,sha256=WbMGbp7uVJJm_GGvrwxgIP51M5IVXJU2M_JV6Vv5KJw,96687
-pyPreservica/webHooksAPI.py,sha256=_K3KUOsmwYf8qMa-mD47sAmNUW7Pzb9oKVpS0VoSbC0,6827
-pyPreservica/workflowAPI.py,sha256=wDDR5_CsJ3dhX79E5mJaziAtgYb830J0ZpNJppzgvqk,17493
-pyPreservica-2.9.4.dist-info/LICENSE.txt,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-pyPreservica-2.9.4.dist-info/METADATA,sha256=jOR9wFM424u6iERFDApsXB_Ta5bEKHHEOz85yoG-nVY,2779
-pyPreservica-2.9.4.dist-info/WHEEL,sha256=YiKiUUeZQGmGJoR_0N1Y933DOBowq4AIvDe2-UIy8E4,91
-pyPreservica-2.9.4.dist-info/top_level.txt,sha256=iIBh6NAznYQHOV8mv_y_kGKSDITek9rANyFDwJsbU-c,13
-pyPreservica-2.9.4.dist-info/RECORD,,