pyPreservica 2.9.4__tar.gz → 3.0.1__tar.gz

{pypreservica-2.9.4 → pypreservica-3.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyPreservica
-Version: 2.9.4
+Version: 3.0.1
 Summary: Python library for the Preservica API
 Home-page: https://pypreservica.readthedocs.io/
 Author: James Carr

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/__init__.py

@@ -23,6 +23,6 @@ from .mdformsAPI import MetadataGroupsAPI, Group, GroupField, GroupFieldType
 __author__ = "James Carr (drjamescarr@gmail.com)"
 
 # Version of the pyPreservica package
-__version__ = "2.9.4"
+__version__ = "3.0.1"
 
 __license__ = "Apache License Version 2.0"

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/common.py

@@ -880,10 +880,10 @@ class AuthenticatedAPI:
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook=None):
+                 protocol: str = "https", request_hook=None, credentials_path: str = 'credentials.properties'):
 
         config = configparser.ConfigParser(interpolation=configparser.Interpolation())
-        config.read('credentials.properties', encoding='utf-8')
+        config.read(os.path.relpath(credentials_path), encoding='utf-8')
         self.session: Session = requests.Session()
 
         if request_hook is not None:

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/contentAPI.py

@@ -35,11 +35,12 @@ class Field:
 
 class ContentAPI(AuthenticatedAPI):
 
-    def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False,
-                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None):
+    def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
+                 use_shared_secret: bool = False, two_fa_secret_key: str = None,
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
         self.callback = None
 
     class SearchResult:

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/entityAPI.py

@@ -35,10 +35,10 @@ class EntityAPI(AuthenticatedAPI):
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
 
         xml.etree.ElementTree.register_namespace("oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/")
         xml.etree.ElementTree.register_namespace("ead", "urn:isbn:1-931666-22-9")
@@ -2281,7 +2281,7 @@ class EntityAPI(AuthenticatedAPI):
             logger.error(exception)
             raise exception
 
-    def delete_asset(self, asset: Asset, operator_comment: str, supervisor_comment: str):
+    def delete_asset(self, asset: Asset, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
         Delete an asset from the repository
 
@@ -2290,11 +2290,11 @@ class EntityAPI(AuthenticatedAPI):
         :param supervisor_comment:  The supervisor comment on the deletion
         """
         if isinstance(asset, Asset):
-            return self._delete_entity(asset, operator_comment, supervisor_comment)
+            return self._delete_entity(asset, operator_comment, supervisor_comment, credentials_path)
         else:
             raise RuntimeError("delete_asset only deletes assets")
 
-    def delete_folder(self, folder: Folder, operator_comment: str, supervisor_comment: str):
+    def delete_folder(self, folder: Folder, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
          Delete an asset from the repository
 
@@ -2304,11 +2304,11 @@ class EntityAPI(AuthenticatedAPI):
          :param supervisor_comment:  The supervisor comment on the deletion
          """
         if isinstance(folder, Folder):
-            return self._delete_entity(folder, operator_comment, supervisor_comment)
+            return self._delete_entity(folder, operator_comment, supervisor_comment, credentials_path)
         else:
             raise RuntimeError("delete_folder only deletes folders")
 
-    def _delete_entity(self, entity: Entity, operator_comment: str, supervisor_comment: str):
+    def _delete_entity(self, entity: Entity, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
         Delete an asset from the repository
 
@@ -2319,7 +2319,7 @@ class EntityAPI(AuthenticatedAPI):
 
         # check manager password is available:
         config = configparser.ConfigParser()
-        config.read('credentials.properties', encoding='utf-8')
+        config.read(credentials_path, encoding='utf-8')
         try:
             manager_username = config['credentials']['manager.username']
             manager_password = config['credentials']['manager.password']
@@ -2373,7 +2373,7 @@ class EntityAPI(AuthenticatedAPI):
                                        headers=headers)
         elif request.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
-            return self._delete_entity(entity, operator_comment, supervisor_comment)
+            return self._delete_entity(entity, operator_comment, supervisor_comment, credentials_path)
         if request.status_code == requests.codes.unprocessable:
             logger.error(request.content.decode('utf-8'))
             raise RuntimeError(request.status_code, "no active workflow context for full deletion exists in the system")
@@ -2385,4 +2385,4 @@ class EntityAPI(AuthenticatedAPI):
         exception = HTTPException(entity.reference, request.status_code, request.url,
                                   "_delete_entity", request.content.decode('utf-8'))
         logger.error(exception)
-        raise exception
+        raise exception

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/mdformsAPI.py

@@ -134,12 +134,13 @@ def _json_from_object_(group: Group) -> dict:
 
 
 class MetadataGroupsAPI(AuthenticatedAPI):
+
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
 
         xml.etree.ElementTree.register_namespace("oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/")
         xml.etree.ElementTree.register_namespace("ead", "urn:isbn:1-931666-22-9")

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/parAPI.py

@@ -23,43 +23,7 @@ def __get_contents__(document) -> AnyStr:
         return json.dumps(json.loads(document))
 
 
-class PreservationActionRegistry:
-
-    def __init__(self, server: str = None, username: str = None, password: str = None, protocol: str = 'https'):
-        self.protocol = protocol
-        self.session = requests.Session()
-        self.session.headers.update({'accept': 'application/json;charset=UTF-8'})
-        config = configparser.ConfigParser()
-        config.read('credentials.properties')
-        if not server:
-            server = os.environ.get('PRESERVICA_SERVER')
-            if server is None:
-                try:
-                    server = config['credentials']['server']
-                except KeyError:
-                    pass
-        if not server:
-            msg = "No valid server found in method arguments, environment variables or credentials.properties file"
-            logger.error(msg)
-            raise RuntimeError(msg)
-        else:
-            self.server = server
-        if not username:
-            username = os.environ.get('PRESERVICA_USERNAME')
-            if username is None:
-                try:
-                    username = config['credentials']['username']
-                except KeyError:
-                    pass
-        self.username = username
-        if not password:
-            password = os.environ.get('PRESERVICA_PASSWORD')
-            if password is None:
-                try:
-                    password = config['credentials']['password']
-                except KeyError:
-                    pass
-        self.password = password
+class PreservationActionRegistry(AuthenticatedAPI):
 
     def format_family(self, guid: str) -> str:
         return self.__guid__(guid, "format-families")

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/retentionAPI.py

@@ -58,9 +58,10 @@ class RetentionPolicy:
 class RetentionAPI(AuthenticatedAPI):
 
     def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False,
-                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None):
+                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
+
         if self.major_version < 7 and self.minor_version < 2:
             raise RuntimeError("Retention API is only available when connected to a v6.2 System")
 

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/uploadAPI.py

@@ -13,20 +13,22 @@ import shutil
 import tempfile
 import uuid
 import xml
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from time import sleep
 from xml.dom import minidom
 from xml.etree import ElementTree
 from xml.etree.ElementTree import Element, SubElement
 
 import boto3
+import botocore
 import s3transfer.tasks
 import s3transfer.upload
-
+from botocore.session import get_session
 from boto3.s3.transfer import TransferConfig, S3Transfer
 from botocore.config import Config
 from botocore.credentials import RefreshableCredentials
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError
+from dateutil.tz import tzlocal
 from s3transfer import S3UploadFailedError
 from tqdm import tqdm
 
@@ -1916,10 +1918,32 @@ class UploadAPI(AuthenticatedAPI):
             'mode': 'adaptive'
         }
 
-        s3_client = boto3.client('s3', endpoint_url=endpoint, aws_access_key_id=self.token,
-                                 aws_secret_access_key="NOT_USED",
+        def new_credentials():
+            metadata: dict = {}
+            metadata['access_key'] =  self.__token__()
+            metadata['secret_key'] = "NOT_USED"
+            metadata['token'] = ""
+            metadata["expiry_time"] = (datetime.now(tzlocal()) + timedelta(minutes=12)).isoformat()
+            logger.info("Refreshing credentials at: " + str(datetime.now(tzlocal())))
+            return metadata
+
+        session = get_session()
+
+        session_credentials = RefreshableCredentials.create_from_metadata(
+            metadata=new_credentials(),
+            refresh_using=new_credentials,
+            advisory_timeout = 4 * 60,
+            mandatory_timeout = 12 * 60,
+            method = 'Preservica'
+        )
+
+        autorefresh_session = boto3.Session(botocore_session=session)
+
+        session._credentials = session_credentials
+
+        s3_client = autorefresh_session.client('s3', endpoint_url=endpoint,
                                  config=Config(s3={'addressing_style': 'path'}, read_timeout=120, connect_timeout=120,
-                                               retries=retries, tcp_keepalive=True))
+                                 retries=retries, tcp_keepalive=True))
 
         metadata = {}
         if folder is not None:
@@ -1932,21 +1956,48 @@ class UploadAPI(AuthenticatedAPI):
             try:
                 key_id = str(uuid.uuid4()) + ".zip"
 
+
+                # how big is the package
+                package_size = os.path.getsize(path_to_zip_package)
+                if package_size > 1 * GB:
+                    transfer_config.multipart_chunksize = 16 * MB   ## Min 64 Chunks
+                if package_size > 8 * GB:
+                    transfer_config.multipart_chunksize = 32 * MB   ## Min 256 Chunks
+                if package_size > 24 * GB:
+                    transfer_config.multipart_chunksize = 48 * MB   ## Min 512 Chunks
+                if package_size > 48 * GB:
+                    transfer_config.multipart_chunksize = 64 * MB
+
+                logger.info("Using Multipart Chunk Size: " + str(transfer_config.multipart_chunksize))
+
                 transfer = S3Transfer(client=s3_client, config=transfer_config)
 
                 transfer.PutObjectTask = PutObjectTask
                 transfer.CompleteMultipartUploadTask = CompleteMultipartUploadTask
                 transfer.upload_file = upload_file
 
-                response = transfer.upload_file(self=transfer, filename=path_to_zip_package, bucket=bucket, key=key_id,
+
+                response = transfer.upload_file(self=transfer, filename=path_to_zip_package, bucket=bucket,
+                                                key=key_id,
                                                 extra_args=metadata,
                                                 callback=callback)
 
+
                 if delete_after_upload:
                     os.remove(path_to_zip_package)
 
                 return response['ResponseMetadata']['HTTPHeaders']['preservica-progress-token']
 
-            except ClientError as e:
-                logger.error(e)
-                raise e
+            except (NoCredentialsError, PartialCredentialsError) as ex:
+                logger.error(ex)
+                raise ex
+
+            except ClientError as ex:
+                logger.error(ex)
+                raise ex
+
+
+
+
+
+

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica/workflowAPI.py

@@ -81,10 +81,10 @@ class WorkflowAPI(AuthenticatedAPI):
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
         self.base_url = "sdb/rest/workflow"
 
     def get_workflow_contexts_by_type(self, workflow_type: str):

{pypreservica-2.9.4 → pypreservica-3.0.1}/pyPreservica.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyPreservica
-Version: 2.9.4
+Version: 3.0.1
 Summary: Python library for the Preservica API
 Home-page: https://pypreservica.readthedocs.io/
 Author: James Carr

{pypreservica-2.9.4 → pypreservica-3.0.1}/setup.py

@@ -21,7 +21,7 @@ if sys.argv[-1] == 'publish':
 # This call to setup() does all the work
 setup(
     name=PKG,
-    version="2.9.4",
+    version="3.0.1",
     description="Python library for the Preservica API",
     long_description=README,
     long_description_content_type="text/markdown",