pyPreservica 2.9.3__py3-none-any.whl → 3.3.3__py3-none-any.whl

pyPreservica/uploadAPI.py

@@ -13,7 +13,7 @@ import shutil
 import tempfile
 import uuid
 import xml
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from time import sleep
 from xml.dom import minidom
 from xml.etree import ElementTree
@@ -22,11 +22,12 @@ from xml.etree.ElementTree import Element, SubElement
 import boto3
 import s3transfer.tasks
 import s3transfer.upload
-
+from botocore.session import get_session
 from boto3.s3.transfer import TransferConfig, S3Transfer
 from botocore.config import Config
 from botocore.credentials import RefreshableCredentials
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError
+from dateutil.tz import tzlocal
 from s3transfer import S3UploadFailedError
 from tqdm import tqdm
 
@@ -37,7 +38,7 @@ logger = logging.getLogger(__name__)
 
 MB = 1024 * 1024
 GB = 1024 ** 3
-transfer_config = TransferConfig(multipart_threshold=int((1 * GB) / 16))
+transfer_config = TransferConfig(multipart_threshold=int(32 * MB))
 
 CONTENT_FOLDER = "content"
 PRESERVATION_CONTENT_FOLDER = "p1"
@@ -481,7 +482,7 @@ def generic_asset_package(preservation_files_dict=None, access_files_dict=None,
     content_type = kwargs.get('CustomType', "")
 
     if not compress:
-        shutil.register_archive_format("szip", _make_stored_zipfile, None, "UnCompressed ZIP file")
+        shutil.register_archive_format(name="szip", function=_make_stored_zipfile, extra_args=None, description="UnCompressed ZIP file")
 
     has_preservation_files = bool((preservation_files_dict is not None) and (len(preservation_files_dict) > 0))
     has_access_files = bool((access_files_dict is not None) and (len(access_files_dict) > 0))
@@ -910,17 +911,22 @@ def complex_asset_package(preservation_files_list=None, access_files_list=None,
     if has_preservation_files:
         if default_asset_title is None:
             default_asset_title = os.path.splitext(os.path.basename(preservation_files_list[0]))[0]
-
         # create the asset
-        xip, io_ref = __create_io__(file_name=default_asset_title, parent_folder=parent_folder, **kwargs)
+        if io_ref is None:
+            xip, io_ref = __create_io__(file_name=default_asset_title, parent_folder=parent_folder, **kwargs)
 
     if has_access_files:
         if default_asset_title is None:
             default_asset_title = os.path.splitext(os.path.basename(access_files_list[0]))[0]
-
         if io_ref is None:
             xip, io_ref = __create_io__(file_name=default_asset_title, parent_folder=parent_folder, **kwargs)
 
+    if io_ref is None:
+        default_asset_title = kwargs.get('Title', None)
+        if default_asset_title is None:
+            default_asset_title = "New Asset"
+        xip, io_ref = __create_io__(file_name=default_asset_title, parent_folder=parent_folder, **kwargs)
+
     if has_preservation_files:
         # add the content objects
         representation_name = kwargs.get('Preservation_Representation_Name', "Preservation")
@@ -1153,383 +1159,14 @@ def _unpad(s):
 
 class UploadAPI(AuthenticatedAPI):
 
-    def ingest_tweet(self, twitter_user=None, tweet_id: int = 0, twitter_consumer_key=None, twitter_secret_key=None,
-                     folder=None,
-                     callback=None, **kwargs):
-
-        """
-            Ingest tweets from a twitter stream by twitter username
-
-            :param tweet_id:
-            :param str twitter_user: Twitter Username
-            :param str twitter_consumer_key: Optional asset title
-            :param str twitter_secret_key: Optional asset description
-            :param str folder: Folder to ingest into
-            :param callback callback: Optional upload progress callback
-            :raises RuntimeError:
 
 
-        """
-
-        def get_image(m, has_video_element):
-            media_url_https_ = m["media_url_https"]
-            if media_url_https_:
-                req = requests.get(media_url_https_)
-                if req.status_code == requests.codes.ok:
-                    if has_video_element:
-                        image_name_ = f"{{{media_id_str}}}_[{twitter_user}]_thumb.jpg"
-                    else:
-                        image_name_ = f"{{{media_id_str}}}_[{twitter_user}].jpg"
-                    image_name_document_ = open(image_name_, "wb")
-                    image_name_document_.write(req.content)
-                    image_name_document_.close()
-                    return image_name_
-
-        def get_video(m):
-            video_info_ = m["video_info"]
-            variants_ = video_info_["variants"]
-            for v_ in variants_:
-                video_url_ = v_["url"]
-                req = requests.get(video_url_)
-                if req.status_code == requests.codes.ok:
-                    video_name_ = f"{{{media_id_str}}}_[{twitter_user}].mp4"
-                    video_name_document_ = open(video_name_, "wb")
-                    video_name_document_.write(req.content)
-                    video_name_document_.close()
-                    return video_name_, True
-
-        entity_client = pyPreservica.EntityAPI(username=self.username, password=self.password, server=self.server,
-                                               tenant=self.tenant)
-        if hasattr(folder, "reference"):
-            folder = entity_client.folder(folder.reference)
-        else:
-            folder = entity_client.folder(folder)
-        try:
-            import tweepy
-        except ImportError:
-            logger.error("Package tweepy is required for twitter harvesting. pip install --upgrade tweepy")
-            raise RuntimeError("Package tweepy is required for twitter harvesting. pip install --upgrade tweepy")
-        config = configparser.ConfigParser()
-        config.read('credentials.properties')
-        if twitter_consumer_key is None:
-            twitter_consumer_key = os.environ.get('TWITTER_CONSUMER_KEY')
-            if twitter_consumer_key is None:
-                try:
-                    twitter_consumer_key = config['credentials']['TWITTER_CONSUMER_KEY']
-                except KeyError:
-                    logger.error("No valid TWITTER_CONSUMER_KEY found in method arguments, "
-                                 "environment variables or credentials.properties file")
-                    raise RuntimeError("No valid TWITTER_CONSUMER_KEY found in method arguments, "
-                                       "environment variables or credentials.properties file")
-        if twitter_secret_key is None:
-            twitter_secret_key = os.environ.get('TWITTER_SECRET_KEY')
-            if twitter_secret_key is None:
-                try:
-                    twitter_secret_key = config['credentials']['TWITTER_SECRET_KEY']
-                except KeyError:
-                    logger.error("No valid TWITTER_SECRET_KEY found in method arguments, "
-                                 "environment variables or credentials.properties file")
-                    raise RuntimeError("No valid TWITTER_SECRET_KEY found in method arguments, "
-                                       "environment variables or credentials.properties file")
-
-        api = None
-        try:
-            auth = tweepy.AppAuthHandler(twitter_consumer_key, twitter_secret_key)
-            api = tweepy.API(auth, wait_on_rate_limit=True)
-        except TweepError:
-            logger.error("No valid Twitter API keys. Could not authenticate")
-            raise RuntimeError("No valid Twitter API keys. Could not authenticate")
-        if api is not None:
-            logger.debug(api)
-            tweet = api.get_status(tweet_id, tweet_mode="extended", include_entities=True)
-            created_at = tweet.created_at
-            id_str = tweet.id_str
-            author = tweet.author.name
-            tweet_entities = tweet.entities
-            hashtags = dict()
-            if 'hashtags' in tweet_entities:
-                hashtags = tweet.entities['hashtags']
-            entities = entity_client.identifier("tweet_id", id_str.strip())
-            if len(entities) > 0:
-                logger.warning("Tweet already exists, skipping....")
-                return
-            logger.info(f"Processing tweet {id_str} ...")
-            tid = tweet.id
-            content_objects = list()
-            full_tweet = api.get_status(tid, tweet_mode="extended", include_entities=True)
-            text = tweet.full_text
-            full_text = full_tweet.full_text
-            file_name = f"{{{id_str}}}_[{twitter_user}].json"
-            json_doc = json.dumps(full_tweet._json)
-            json_file = open(file_name, "wt", encoding="utf-8")
-            json_file.write(json_doc)
-            json_file.close()
-            content_objects.append(file_name)
-            if hasattr(full_tweet, "extended_entities"):
-                extended_entities = full_tweet.extended_entities
-                if "media" in extended_entities:
-                    media = extended_entities["media"]
-                    for med in media:
-                        media_id_str = med["id_str"]
-                        has_video = False
-                        if "video_info" in med:
-                            co, has_video = get_video(med)
-                            content_objects.append(co)
-                            if has_video:
-                                co = get_image(med, has_video)
-                                content_objects.append(co)
-                            continue
-                        if "media_url_https" in med:
-                            co = get_image(med, has_video)
-                            content_objects.append(co)
-            identifiers = dict()
-            asset_metadata = dict()
-            identifiers["tweet_id"] = id_str
-
-            user = full_tweet._json['user']
-
-            if full_tweet._json.get('retweeted_status'):
-                retweeted_status = full_tweet._json['retweeted_status']
-                if retweeted_status.get("extended_entities"):
-                    extended_entities = retweeted_status["extended_entities"]
-                    if "media" in extended_entities:
-                        media = extended_entities["media"]
-                        for med in media:
-                            media_id_str = med["id_str"]
-                            has_video = False
-                            if "video_info" in med:
-                                co, has_video = get_video(med)
-                                content_objects.append(co)
-                                continue
-                            if "media_url_https" in med:
-                                co = get_image(med, has_video)
-                                content_objects.append(co)
-
-            xml_object = xml.etree.ElementTree.Element('tweet', {"xmlns": "http://www.preservica.com/tweets/v1"})
-            xml.etree.ElementTree.SubElement(xml_object, "id").text = id_str
-            xml.etree.ElementTree.SubElement(xml_object, "full_text").text = full_text
-            xml.etree.ElementTree.SubElement(xml_object, "created_at").text = str(created_at)
-            xml.etree.ElementTree.SubElement(xml_object, "screen_name_sender").text = user.get('screen_name')
-            for h in hashtags:
-                xml.etree.ElementTree.SubElement(xml_object, "hashtag").text = str(h['text'])
-
-            xml.etree.ElementTree.SubElement(xml_object, "name").text = author
-            xml.etree.ElementTree.SubElement(xml_object, "retweet").text = str(full_tweet._json['retweet_count'])
-            xml.etree.ElementTree.SubElement(xml_object, "likes").text = str(full_tweet._json['favorite_count'])
-
-            xml_request = xml.etree.ElementTree.tostring(xml_object, encoding='utf-8')
-
-            metadata_document = open("metadata.xml", "wt", encoding="utf-8")
-            metadata_document.write(xml_request.decode("utf-8"))
-            metadata_document.close()
-
-            asset_metadata["http://www.preservica.com/tweets/v1"] = "metadata.xml"
-
-            security_tag = kwargs.get("SecurityTag", "open")
-            asset_title = kwargs.get("Title", text)
-            asset_description = kwargs.get("Description", full_text)
-
-            p = complex_asset_package(preservation_files_list=content_objects, parent_folder=folder, Title=asset_title,
-                                      Description=asset_description, CustomType="Tweet", Identifiers=identifiers,
-                                      Asset_Metadata=asset_metadata, SecurityTag=security_tag)
-            self.upload_zip_package(p, folder=folder, callback=callback)
-            for ob in content_objects:
-                os.remove(ob)
-            os.remove("metadata.xml")
-
-    def ingest_twitter_feed(self, twitter_user=None, num_tweets: int = 25, twitter_consumer_key=None,
-                            twitter_secret_key=None, folder=None,
-                            callback=None, **kwargs):
-
-        """
-            Ingest tweets from a twitter stream by twitter username
-
-            :param str twitter_user: Twitter Username
-            :param int num_tweets: The number of tweets from the stream
-            :param str twitter_consumer_key: Optional asset title
-            :param str twitter_secret_key: Optional asset description
-            :param str folder: Folder to ingest into
-            :param callback callback: Optional upload progress callback
-            :raises RuntimeError:
-
-
-        """
-
-        def get_image(m, has_video_element):
-            media_url_https_ = m["media_url_https"]
-            if media_url_https_:
-                req = requests.get(media_url_https_)
-                if req.status_code == requests.codes.ok:
-                    if has_video_element:
-                        image_name_ = f"{{{media_id_str}}}_[{twitter_user}]_thumb.jpg"
-                    else:
-                        image_name_ = f"{{{media_id_str}}}_[{twitter_user}].jpg"
-                    image_name_document_ = open(image_name_, "wb")
-                    image_name_document_.write(req.content)
-                    image_name_document_.close()
-                    return image_name_
-
-        def get_video(m):
-            video_info_ = m["video_info"]
-            variants_ = video_info_["variants"]
-            for v_ in variants_:
-                if v_['content_type'] == 'video/mp4':
-                    video_url_ = v_["url"]
-                    with requests.get(video_url_, stream=True) as req:
-                        video_name_ = f"{{{media_id_str}}}_[{twitter_user}].mp4"
-                        with open(video_name_, 'wb') as video_name_document_:
-                            for chunk in req.iter_content(chunk_size=1024):
-                                video_name_document_.write(chunk)
-                                video_name_document_.flush()
-                        return video_name_, True
-
-        entity_client = pyPreservica.EntityAPI(username=self.username, password=self.password, server=self.server,
-                                               tenant=self.tenant)
-        if hasattr(folder, "reference"):
-            folder = entity_client.folder(folder.reference)
-        else:
-            folder = entity_client.folder(folder)
-        try:
-            import tweepy
-        except ImportError:
-            logger.error("Package tweepy is required for twitter harvesting. pip install --upgrade tweepy")
-            raise RuntimeError("Package tweepy is required for twitter harvesting. pip install --upgrade tweepy")
-        config = configparser.ConfigParser()
-        config.read('credentials.properties')
-        if twitter_consumer_key is None:
-            twitter_consumer_key = os.environ.get('TWITTER_CONSUMER_KEY')
-            if twitter_consumer_key is None:
-                try:
-                    twitter_consumer_key = config['credentials']['TWITTER_CONSUMER_KEY']
-                except KeyError:
-                    logger.error("No valid TWITTER_CONSUMER_KEY found in method arguments, "
-                                 "environment variables or credentials.properties file")
-                    raise RuntimeError("No valid TWITTER_CONSUMER_KEY found in method arguments, "
-                                       "environment variables or credentials.properties file")
-        if twitter_secret_key is None:
-            twitter_secret_key = os.environ.get('TWITTER_SECRET_KEY')
-            if twitter_secret_key is None:
-                try:
-                    twitter_secret_key = config['credentials']['TWITTER_SECRET_KEY']
-                except KeyError:
-                    logger.error("No valid TWITTER_SECRET_KEY found in method arguments, "
-                                 "environment variables or credentials.properties file")
-                    raise RuntimeError("No valid TWITTER_SECRET_KEY found in method arguments, "
-                                       "environment variables or credentials.properties file")
-
-        api = None
-        try:
-            auth = tweepy.AppAuthHandler(twitter_consumer_key, twitter_secret_key)
-            api = tweepy.API(auth, wait_on_rate_limit=True)
-        except RuntimeError:
-            logger.error("No valid Twitter API keys. Could not authenticate")
-            raise RuntimeError("No valid Twitter API keys. Could not authenticate")
-        if api is not None:
-            logger.debug(api)
-            for tweet in tweepy.Cursor(api.user_timeline, id=twitter_user).items(int(num_tweets)):
-                created_at = tweet.created_at
-                id_str = tweet.id_str
-                author = tweet.author.name
-                tweet_entities = tweet.entities
-                hashtags = dict()
-                if 'hashtags' in tweet_entities:
-                    hashtags = tweet.entities['hashtags']
-                entities = entity_client.identifier("tweet_id", id_str.strip())
-                if len(entities) > 0:
-                    logger.warning("Tweet already exists, skipping....")
-                    continue
-                logger.info(f"Processing tweet {id_str} ...")
-                tid = tweet.id
-                content_objects = list()
-                full_tweet = api.get_status(tid, tweet_mode="extended", include_entities=True)
-                text = tweet.text
-                logger.debug(text)
-                full_text = full_tweet.full_text
-                file_name = f"{{{id_str}}}_[{twitter_user}].json"
-                json_doc = json.dumps(full_tweet._json)
-                json_file = open(file_name, "wt", encoding="utf-8")
-                json_file.write(json_doc)
-                json_file.close()
-                content_objects.append(file_name)
-                if hasattr(full_tweet, "extended_entities"):
-                    extended_entities = full_tweet.extended_entities
-                    if "media" in extended_entities:
-                        media = extended_entities["media"]
-                        for med in media:
-                            media_id_str = med["id_str"]
-                            has_video = False
-                            if "video_info" in med:
-                                co, has_video = get_video(med)
-                                content_objects.append(co)
-                                if has_video:
-                                    co = get_image(med, has_video)
-                                    content_objects.append(co)
-                                continue
-                            if "media_url_https" in med:
-                                co = get_image(med, has_video)
-                                content_objects.append(co)
-                identifiers = {}
-                asset_metadata = {}
-                identifiers["tweet_id"] = id_str
-
-                user = full_tweet._json['user']
-
-                if full_tweet._json.get('retweeted_status'):
-                    retweeted_status = full_tweet._json['retweeted_status']
-                    if retweeted_status.get("extended_entities"):
-                        extended_entities = retweeted_status["extended_entities"]
-                        if "media" in extended_entities:
-                            media = extended_entities["media"]
-                            for med in media:
-                                media_id_str = med["id_str"]
-                                has_video = False
-                                if "video_info" in med:
-                                    co, has_video = get_video(med)
-                                    content_objects.append(co)
-                                    continue
-                                if "media_url_https" in med:
-                                    co = get_image(med, has_video)
-                                    content_objects.append(co)
-
-                xml_object = xml.etree.ElementTree.Element('tweet', {"xmlns": "http://www.preservica.com/tweets/v1"})
-                xml.etree.ElementTree.SubElement(xml_object, "id").text = id_str
-                xml.etree.ElementTree.SubElement(xml_object, "full_text").text = full_text
-                xml.etree.ElementTree.SubElement(xml_object, "created_at").text = str(created_at)
-                xml.etree.ElementTree.SubElement(xml_object, "screen_name_sender").text = user.get('screen_name')
-                for h in hashtags:
-                    xml.etree.ElementTree.SubElement(xml_object, "hashtag").text = str(h['text'])
-
-                xml.etree.ElementTree.SubElement(xml_object, "name").text = author
-                xml.etree.ElementTree.SubElement(xml_object, "retweet").text = str(full_tweet._json['retweet_count'])
-                xml.etree.ElementTree.SubElement(xml_object, "likes").text = str(full_tweet._json['favorite_count'])
-
-                xml_request = xml.etree.ElementTree.tostring(xml_object, encoding='utf-8')
-
-                metadata_document = open("metadata.xml", "wt", encoding="utf-8")
-                metadata_document.write(xml_request.decode("utf-8"))
-                metadata_document.close()
-
-                asset_metadata["http://www.preservica.com/tweets/v1"] = "metadata.xml"
-
-                security_tag = kwargs.get("SecurityTag", "open")
-                asset_title = kwargs.get("Title", text)
-                asset_description = kwargs.get("Description", full_text)
-
-                p = complex_asset_package(preservation_files_list=content_objects, parent_folder=folder,
-                                          Title=asset_title,
-                                          Description=asset_description, CustomType="Tweet", Identifiers=identifiers,
-                                          Asset_Metadata=asset_metadata, SecurityTag=security_tag)
-                self.upload_zip_package(p, folder=folder, callback=callback)
-                for ob in content_objects:
-                    os.remove(ob)
-                os.remove("metadata.xml")
-                sleep(2)
 
     def ingest_web_video(self, url=None, parent_folder=None, **kwargs):
         """
             Ingest a web video such as YouTube etc based on the URL
 
-            :param str url: URL to the youtube video
+            :param str url: URL to the YouTube video
             :param Folder parent_folder: The folder to ingest the video into
             :param str Title: Optional asset title
             :param str Description: Optional asset description
@@ -1627,6 +1264,52 @@ class UploadAPI(AuthenticatedAPI):
             logger.error(exception)
             raise exception
 
+    def clean_upload_bucket(self, bucket_name: str,  older_than_days: int = 90):
+        """
+        Clean up objects in an upload bucket which are older than older_than_days.
+
+        """
+        from azure.storage.blob import ContainerClient
+
+        for location in self.upload_locations():
+            if location['containerName'] == bucket_name:
+
+                if location['type'] != 'AWS':
+                    credentials = self.upload_credentials(location['apiId'])
+                    account_key = credentials['key']
+                    session_token = credentials['sessionToken']
+                    sas_url = f"https://{account_key}.blob.core.windows.net/{bucket_name}"
+                    container = ContainerClient.from_container_url(container_url=sas_url, credential=session_token)
+                    now = datetime.now(timezone.utc)
+                    for blob in container.list_blobs():
+                        if abs((blob.last_modified - now).days) > older_than_days:
+                            logger.debug(f"Deleting expired object {blob.name}")
+                            container.delete_blob(blob.name)
+
+                if location['type'] == 'AWS':
+                    credentials = self.upload_credentials(location['apiId'])
+                    access_key = credentials['key']
+                    secret_key = credentials['secret']
+                    session_token = credentials['sessionToken']
+                    session = boto3.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key,
+                                            aws_session_token=session_token)
+                    s3_client = session.client("s3")
+                    paginator = s3_client.get_paginator('list_objects_v2')
+                    now = datetime.now(timezone.utc)
+                    for page in paginator.paginate(Bucket=bucket_name):
+                        if 'Contents' in page:
+                            for key in page['Contents']:
+                                last_modified = key['LastModified']
+                                if abs((last_modified - now).days) > older_than_days:
+                                    logger.debug(f"Deleting expired object {key['Key']}")
+                                    s3_client.delete_object(Bucket=bucket_name, Key=key['Key'])
+
+
+
+
+
+
+
     def upload_locations(self):
         """
         Upload locations are configured on the Sources page as 'SIP Upload'.
@@ -1659,30 +1342,52 @@ class UploadAPI(AuthenticatedAPI):
                          security_tag: str = "open",
                          delete_after_upload: bool = True, max_MB_ingested: int = -1):
 
+        from pyPreservica import EntityAPI
+
+        def entity_value(client: EntityAPI, identifier: str) -> Entity:
+            back_off: int = 5
+            while True:
+                try:
+                    entities = client.identifier("code", identifier)
+                    if bool(len(entities) > 0):
+                        return entities.pop()
+                    else:
+                        return None
+                except HTTPException as e:
+                    sleep(back_off)
+                    back_off = back_off * 2
+
+        def entity_exists(client: EntityAPI, identifier: str) -> bool:
+            back_off: int = 5
+            while True:
+                try:
+                    entities = client.identifier("code", identifier)
+                    return bool(len(entities) > 0)
+                except HTTPException as e:
+                    sleep(back_off)
+                    back_off = back_off * 2
+
         def get_parent(client, identifier, parent_reference):
-            id = str(os.path.dirname(identifier))
-            if not id:
-                id = identifier
-            entities = client.identifier("code", id)
-            if len(entities) > 0:
-                folder = entities.pop()
+            dirname_id: str = str(os.path.dirname(identifier))
+            if not dirname_id:
+                dirname_id = identifier
+            folder = entity_value(client, dirname_id)
+            if folder is not None:
                 folder = client.folder(folder.reference)
                 return folder.reference
             else:
                 return parent_reference
 
         def get_folder(client, name, tag, parent_reference, identifier):
-            entities = client.identifier("code", identifier)
-            if len(entities) == 0:
+            folder = entity_value(client, identifier)
+            if folder is None:
                 logger.info(f"Creating new folder with name {name}")
                 folder = client.create_folder(name, name, tag, parent_reference)
                 client.add_identifier(folder, "code", identifier)
             else:
                 logger.info(f"Found existing folder with name {name}")
-                folder = entities.pop()
             return folder
 
-        from pyPreservica import EntityAPI
         entity_client = EntityAPI(username=self.username, password=self.password, server=self.server,
                                   tenant=self.tenant,
                                   two_fa_secret_key=self.two_fa_secret_key, use_shared_secret=self.shared_secret,
@@ -1712,7 +1417,7 @@ class UploadAPI(AuthenticatedAPI):
                     files.remove(file)
                     continue
                 asset_code = os.path.join(code, file)
-                if len(entity_client.identifier("code", asset_code)) == 0:
+                if not entity_exists(entity_client, asset_code):
                     bytes_ingested = bytes_ingested + os.stat(full_path).st_size
                     logger.info(f"Adding new file: {file} to package ready for upload")
                     file_identifiers = {"code": asset_code}
@@ -1735,8 +1440,8 @@ class UploadAPI(AuthenticatedAPI):
                                             delete_after_upload=delete_after_upload)
                 else:
                     self.upload_zip_to_Source(path_to_zip_package=package, container_name=bucket_name,
-                                                  show_progress= bool(progress_display is not None),
-                                                  delete_after_upload=delete_after_upload)
+                                              show_progress=bool(progress_display is not None),
+                                              delete_after_upload=delete_after_upload)
 
                 logger.info(f"Uploaded " + "{:.1f}".format(bytes_ingested / (1024 * 1024)) + " MB")
 
@@ -1910,9 +1615,42 @@ class UploadAPI(AuthenticatedAPI):
         endpoint = f'{self.protocol}://{self.server}/api/s3/buckets'
         self.token = self.__token__()
 
-        s3_client = boto3.client('s3', endpoint_url=endpoint, aws_access_key_id=self.token,
-                                 aws_secret_access_key="NOT_USED",
-                                 config=Config(s3={'addressing_style': 'path'}))
+
+        retries= {
+            'max_attempts': 5,
+            'mode': 'adaptive'
+        }
+
+        def new_credentials():
+            cred_metadata: dict = {}
+            cred_metadata['access_key'] =  self.__token__()
+            cred_metadata['secret_key'] = "NOT_USED"
+            cred_metadata['token'] = ""
+            cred_metadata["expiry_time"] = (datetime.now(tzlocal()) + timedelta(minutes=12)).isoformat()
+            logger.info("Refreshing credentials at: " + str(datetime.now(tzlocal())))
+            return cred_metadata
+
+        session = get_session()
+
+        session_credentials = RefreshableCredentials.create_from_metadata(
+            metadata=new_credentials(),
+            refresh_using=new_credentials,
+            advisory_timeout = 4 * 60,
+            mandatory_timeout = 12 * 60,
+            method = 'Preservica'
+        )
+
+        autorefresh_session = boto3.Session(botocore_session=session)
+
+        session._credentials = session_credentials
+
+        config = Config(s3={'addressing_style': 'path'}, read_timeout=120, connect_timeout=120,
+               request_checksum_calculation="WHEN_REQUIRED",
+               response_checksum_validation="WHEN_REQUIRED",
+               retries=retries, tcp_keepalive=True)
+
+
+        s3_client = autorefresh_session.client('s3', endpoint_url=endpoint, config=config)
 
         metadata = {}
         if folder is not None:
@@ -1925,21 +1663,48 @@ class UploadAPI(AuthenticatedAPI):
             try:
                 key_id = str(uuid.uuid4()) + ".zip"
 
+
+                # how big is the package
+                package_size = os.path.getsize(path_to_zip_package)
+                if package_size > 1 * GB:
+                    transfer_config.multipart_chunksize = 16 * MB   ## Min 64 Chunks
+                if package_size > 8 * GB:
+                    transfer_config.multipart_chunksize = 32 * MB   ## Min 256 Chunks
+                if package_size > 24 * GB:
+                    transfer_config.multipart_chunksize = 48 * MB   ## Min 512 Chunks
+                if package_size > 48 * GB:
+                    transfer_config.multipart_chunksize = 64 * MB
+
+                logger.info("Using Multipart Chunk Size: " + str(transfer_config.multipart_chunksize))
+
                 transfer = S3Transfer(client=s3_client, config=transfer_config)
 
                 transfer.PutObjectTask = PutObjectTask
                 transfer.CompleteMultipartUploadTask = CompleteMultipartUploadTask
                 transfer.upload_file = upload_file
 
-                response = transfer.upload_file(self=transfer, filename=path_to_zip_package, bucket=bucket, key=key_id,
+
+                response = transfer.upload_file(self=transfer, filename=path_to_zip_package, bucket=bucket,
+                                                key=key_id,
                                                 extra_args=metadata,
                                                 callback=callback)
 
+
                 if delete_after_upload:
                     os.remove(path_to_zip_package)
 
                 return response['ResponseMetadata']['HTTPHeaders']['preservica-progress-token']
 
-            except ClientError as e:
-                logger.error(e)
-                raise e
+            except (NoCredentialsError, PartialCredentialsError) as ex:
+                logger.error(ex)
+                raise ex
+
+            except ClientError as ex:
+                logger.error(ex)
+                raise ex
+
+
+
+
+
+