pyPreservica 0.9.9__py3-none-any.whl → 3.3.4__py3-none-any.whl

pyPreservica/common.py

@@ -1,36 +1,49 @@
 """
 Base class for authenticated API calls used by Entity, Content and Upload
 
+Manages the authentication token lifetime and namespace versions.
+
 author:     James Carr
 licence:    Apache License 2.0
 
 """
-
 import configparser
+import functools
 import hashlib
+import json
+import logging
 import os
+import platform
+import re
 import sys
 import threading
 import time
+import unicodedata
 import xml.etree.ElementTree
 from enum import Enum
+from pathlib import Path
+import pyotp
+from requests import Session
+from urllib3.util import Retry
 import requests
-import logging
-import unicodedata
-import re
+from requests.adapters import HTTPAdapter
+from typing import TypeVar
+from datetime import datetime
+import dateutil
 
 import pyPreservica
 
 logger = logging.getLogger(__name__)
 
-CHUNK_SIZE = 1024 * 2
-
 NS_XIP_ROOT = "http://preservica.com/XIP/"
 NS_ENTITY_ROOT = "http://preservica.com/EntityAPI/"
 NS_RM_ROOT = "http://preservica.com/RetentionManagement/"
+NS_SEC_ROOT = "http://preservica.com/SecurityAPI"
 
 NS_WORKFLOW = "http://workflow.preservica.com"
 
+NS_ADMIN = "http://preservica.com/AdminAPI"
+
 NS_XIP_V6 = "http://preservica.com/XIP/v6.0"
 NS_ENTITY = "http://preservica.com/EntityAPI/v6.0"
 
@@ -41,6 +54,8 @@ SO_PATH = "structural-objects"
 CO_PATH = "content-objects"
 
 HASH_BLOCK_SIZE = 65536
+TIME_OUT = 62
+CHUNK_SIZE = 1024 * 4
 
 
 class FileHash:
@@ -65,8 +80,39 @@ class FileHash:
         return hash_algorithm.hexdigest()
 
 
+def identifiersToDict(identifiers: set) -> dict:
+    """
+        Convert a set of tuples to a dict
+        :param identifiers:
+        :return:
+    """
+    result = {}
+    for identifier_tuple in identifiers:
+        result[identifier_tuple[0]] = identifier_tuple[1]
+    return result
+
+
+def strtobool(val) -> bool:
+    """
+    Convert a string representation of truth to true (1) or false (0).
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
+    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if
+    'val' is anything else.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    elif val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    else:
+        raise ValueError("invalid truth value %r" % (val,))
+
+
 def _make_stored_zipfile(base_name, base_dir, owner, group, verbose=0, dry_run=0, logger=None):
-    """Create a zip file from all the files under 'base_dir'.
+    """
+    Create a non compressed zip file from all the files under 'base_dir'.
+
 
     The output zip file will be named 'base_name' + ".zip".  Returns the
     name of the output zip file.
@@ -109,21 +155,13 @@ def _make_stored_zipfile(base_name, base_dir, owner, group, verbose=0, dry_run=0
     return zip_filename
 
 
-def only_assets(entity):
-    return bool(entity.entity_type is EntityType.ASSET)
-
-
-def only_folders(entity):
-    return bool(entity.entity_type is EntityType.FOLDER)
-
-
 class PagedSet:
     """
     Class to represent a page of results
     The results object contains the list of objects of interest
     """
 
-    def __init__(self, results, has_more, total, next_page):
+    def __init__(self, results, has_more: bool, total: int, next_page: str):
         self.results = results
         self.has_more = bool(has_more)
         self.total = int(total)
@@ -160,12 +198,83 @@ class Sha512FixityCallBack:
         return "SHA512", sha(full_path)
 
 
+class ReportProgressConsoleCallback:
+
+    def __init__(self, prefix='Progress:', suffix='', length=100, fill='█', printEnd="\r"):
+        self.prefix = prefix
+        self.suffix = suffix
+        self.length = length
+        self.fill = fill
+        self.printEnd = printEnd
+        self._lock = threading.Lock()
+        self.print_progress_bar(0)
+
+    def __call__(self, value):
+        with self._lock:
+            values = value.split(":")
+            self.total = int(values[1])
+            self.current = int(values[0])
+            if self.total == 0:
+                percentage = 100.0
+            else:
+                percentage = (self.current / self.total) * 100
+            self.print_progress_bar(percentage)
+            if int(percentage) == int(100):
+                self.print_progress_bar(100.0)
+                sys.stdout.write(self.printEnd)
+                sys.stdout.flush()
+
+    def print_progress_bar(self, percentage):
+        filled_length = int(self.length * (percentage / 100.0))
+        bar_sym = self.fill * filled_length + '-' * (self.length - filled_length)
+        sys.stdout.write(
+            '\r%s |%s| (%.2f%%) %s ' % (self.prefix, bar_sym, percentage, self.suffix))
+        sys.stdout.flush()
+
+
+class UploadProgressConsoleCallback:
+
+    def __init__(self, filename: str, prefix='Progress:', suffix='', length=100, fill='█', printEnd="\r"):
+        self.prefix = prefix
+        self.suffix = suffix
+        self.length = length
+        self.fill = fill
+        self.printEnd = printEnd
+        self._filename = filename
+        self._size = float(Path(filename).stat().st_size)
+        self._seen_so_far = 0
+        self.start = time.time()
+        self._lock = threading.Lock()
+        self.print_progress_bar(0, 0)
+
+    def __call__(self, bytes_amount):
+        with self._lock:
+            seconds = time.time() - self.start
+            if seconds == 0:
+                seconds = 1.0
+            self._seen_so_far += bytes_amount
+            percentage = (self._seen_so_far / self._size) * float(100.0)
+            rate = (self._seen_so_far / (1024 * 1024)) / seconds
+            self.print_progress_bar(percentage, rate)
+            if int(self._seen_so_far) == int(self._size):
+                self.print_progress_bar(100.0, rate)
+                sys.stdout.write(self.printEnd)
+                sys.stdout.flush()
+
+    def print_progress_bar(self, percentage, rate):
+        filled_length = int(self.length * (percentage / 100.0))
+        bar_sym = self.fill * filled_length + '-' * (self.length - filled_length)
+        sys.stdout.write(
+            '\r%s |%s| (%.2f%%) (%.2f %s) %s ' % (self.prefix, bar_sym, percentage, rate, "Mb/s", self.suffix))
+        sys.stdout.flush()
+
+
 class UploadProgressCallback:
     """
     Default implementation of a callback class to show upload progress of a file
     """
 
-    def __init__(self, filename):
+    def __init__(self, filename: str):
         self._filename = filename
         self._size = float(os.path.getsize(filename))
         self._seen_so_far = 0
@@ -179,6 +288,87 @@ class UploadProgressCallback:
             sys.stdout.flush()
 
 
+class RelationshipDirection(Enum):
+    FROM = "From"
+    TO = "To"
+
+
+class EntityType(Enum):
+    """
+    Enumeration of the Entity Types
+    """
+    ASSET = "IO"
+    FOLDER = "SO"
+    CONTENT_OBJECT = "CO"
+
+
+class HTTPException(Exception):
+    """
+     Custom Exception non 404 errors
+     """
+
+    def __init__(self, reference, http_status_code, url, method_name, message):
+        self.reference = reference
+        self.url = url
+        self.method_name = method_name
+        self.http_status_code = http_status_code
+        self.msg = message
+        Exception.__init__(self, self.reference, self.http_status_code, self.url, self.msg)
+
+    def __str__(self):
+        return f"Calling method {self.method_name}() {self.url} returned HTTP {self.http_status_code}. {self.msg}"
+
+
+class ReferenceNotFoundException(Exception):
+    """
+    Custom Exception for failed lookups by reference 404 Errors
+    """
+
+    def __init__(self, reference, http_status_code, url, method_name):
+        self.reference = reference
+        self.url = url
+        self.method_name = method_name
+        self.http_status_code = http_status_code
+        self.msg = f"The requested reference {self.reference} is not found in the repository"
+        Exception.__init__(self, self.reference, self.http_status_code, self.url, self.msg)
+
+    def __str__(self):
+        return f"Calling method {self.method_name}() {self.url} returned HTTP {self.http_status_code}. {self.msg}"
+
+
+class Relationship:
+    DCMI_hasFormat = "http://purl.org/dc/terms/hasFormat"
+    DCMI_isFormatOf = "http://purl.org/dc/terms/isFormatOf"
+    DCMI_hasPart = "http://purl.org/dc/terms/hasPart"
+    DCMI_isPartOf = "http://purl.org/dc/terms/isPartOf"
+    DCMI_hasVersion = "http://purl.org/dc/terms/hasVersion"
+    DCMI_isVersionOf = "http://purl.org/dc/terms/isVersionOf"
+    DCMI_isReferencedBy = "http://purl.org/dc/terms/isReferencedBy"
+    DCMI_references = "http://purl.org/dc/terms/references"
+    DCMI_isReplacedBy = "http://purl.org/dc/terms/isReplacedBy"
+    DCMI_replaces = "http://purl.org/dc/terms/replaces"
+    DCMI_isRequiredBy = "http://purl.org/dc/terms/isRequiredBy"
+    DCMI_requires = "http://purl.org/dc/terms/requires"
+    DCMI_conformsTo = "http://purl.org/dc/terms/conformsTo"
+
+    def __init__(self, relationship_id: str, relationship_type: str, direction: RelationshipDirection, other_ref: str,
+                 title: str, entity_type: EntityType, this_ref: str, api_id: str):
+        self.api_id = api_id
+        self.this_ref = this_ref
+        self.entity_type = entity_type
+        self.title = title
+        self.other_ref = other_ref
+        self.direction = direction
+        self.relationship_type = relationship_type
+        self.relationship_id = relationship_id
+
+    def __str__(self):
+        if self.direction == RelationshipDirection.FROM:
+            return f"{self.this_ref} {self.relationship_type} {self.other_ref}"
+        else:
+            return f"{self.other_ref} {self.relationship_type} {self.this_ref}"
+
+
 class IntegrityCheck:
     """
     Class to hold information about completed integrity checks
@@ -208,64 +398,75 @@ class IntegrityCheck:
         return self.success
 
 
-class Representation:
+class Bitstream:
     """
-        Class to represent the Representation Object in the Preservica data model
+        Class to represent the Bitstream Object or digital file in the Preservica data model
     """
 
-    def __init__(self, asset, rep_type, name, url):
-        self.asset = asset
-        self.rep_type = rep_type
-        self.name = name
-        self.url = url
+    def __init__(self, filename: str, length: int, fixity: dict, content_url: str):
+        self.filename = filename
+        self.length = int(length)
+        self.fixity = fixity
+        self.content_url = content_url
+        self.bs_index = None
+        self.gen_index = None
+        self.co_ref = None
 
     def __str__(self):
-        return f"Type:\t\t\t{self.rep_type}\n" \
-               f"Name:\t\t\t{self.name}\n" \
-               f"URL:\t{self.url}"
+        return f"""
+           Filename:       {self.filename}
+           File Length:    {self.length}
+           Fixity:         {self.fixity}
+        """
 
     def __repr__(self):
         return self.__str__()
 
 
-class Bitstream:
+class ExternIdentifier:
     """
-        Class to represent the Bitstream Object or digital file in the Preservica data model
+        Class to represent the External Identifier Object in the Preservica data model
     """
 
-    def __init__(self, filename, length, fixity, content_url):
-        self.filename = filename
-        self.length = int(length)
-        self.fixity = fixity
-        self.content_url = content_url
+    def __init__(self, identifier_type: str, identifier_value: str):
+        self.type = identifier_type
+        self.value = identifier_value
+        self.id = None
 
     def __str__(self):
-        return f"Filename:\t\t\t{self.filename}\n" \
-               f"FileSize:\t\t\t{self.length}\n" \
-               f"Content:\t{self.content_url}\n" \
-               f"Fixity:\t{self.fixity}"
+        return f"""
+           Identifier:          {self.id}
+           Identifier Type:     {self.type}
+           Identifier Value:    {self.value}
+        """
 
     def __repr__(self):
         return self.__str__()
 
-
 class Generation:
     """
          Class to represent the Generation Object in the Preservica data model
-     """
+    """
 
-    def __init__(self, original, active, format_group, effective_date, bitstreams):
-        self.original = original
-        self.active = active
+    def __init__(self, original: bool, active: bool, format_group: str, effective_date: str, bitstreams: list):
+        self.original = bool(original)
+        self.active = bool(active)
         self.content_object = None
         self.format_group = format_group
         self.effective_date = effective_date
         self.bitstreams = bitstreams
+        self.properties = list()
+        self.formats = list()
 
     def __str__(self):
-        return f"Active:\t\t\t{self.active}\n" \
-               f"Original:\t\t\t{self.original}\n" \
-               f"Format_group:\t{self.format_group}"
+        return f"""
+           Active:         {self.active}
+           Original:       {self.original}
+           Format Group:   {self.format_group}
+           Effective Date: {self.effective_date}
+           Formats:        {self.formats}
+           Properties:     {self.properties}
+           """
 
     def __repr__(self):
         return self.__str__()
@@ -276,7 +477,7 @@ class Entity:
         Base Class of Assets, Folders and Content Objects
     """
 
-    def __init__(self, reference, title, description, security_tag, parent, metadata):
+    def __init__(self, reference: str, title: str, description: str, security_tag: str, parent: str, metadata: dict):
         self.reference = reference
         self.title = title
         self.description = description
@@ -286,24 +487,36 @@ class Entity:
         self.entity_type = None
         self.path = None
         self.tag = None
+        self.custom_type = None
 
     def __str__(self):
-        return f"Ref:\t\t\t{self.reference}\n" \
-               f"Title:\t\t\t{self.title}\n" \
-               f"Description:\t{self.description}\n" \
-               f"Security Tag:\t{self.security_tag}\n" \
-               f"Parent:\t\t\t{self.parent}\n\n"
+        return f"""
+            Entity:         {self.entity_type}
+            Entity Ref:     {self.reference}
+            Title:          {self.title}
+            Description:    {self.description}
+            Security Tag:   {self.security_tag}
+            Parent:         {self.parent}
+            Custom Type:    {self.custom_type}
+            """
 
     def __repr__(self):
         return self.__str__()
 
+    def has_metadata(self) -> bool:
+        return bool(self.metadata)
+
+    def metadata_namespaces(self) -> list:
+        return list(self.metadata.values())
+
 
 class Folder(Entity):
     """
        Class to represent the Structural Object or Folder in the Preservica data model
     """
 
-    def __init__(self, reference, title, description, security_tag, parent, metadata):
+    def __init__(self, reference: str, title: str, description: str = None, security_tag: str = None,
+                 parent: str = None, metadata: dict = None):
         super().__init__(reference, title, description, security_tag, parent, metadata)
         self.entity_type = EntityType.FOLDER
         self.path = SO_PATH
@@ -315,7 +528,8 @@ class Asset(Entity):
         Class to represent the Information Object or Asset in the Preservica data model
     """
 
-    def __init__(self, reference, title, description, security_tag, parent, metadata):
+    def __init__(self, reference: str, title: str, description: str = None, security_tag: str = None,
+                 parent: str = None, metadata: dict = None):
         super().__init__(reference, title, description, security_tag, parent, metadata)
         self.entity_type = EntityType.ASSET
         self.path = IO_PATH
@@ -327,7 +541,8 @@ class ContentObject(Entity):
        Class to represent the Content Object in the Preservica data model
     """
 
-    def __init__(self, reference, title, description, security_tag, parent, metadata):
+    def __init__(self, reference: str, title: str, description: str = None, security_tag: str = None,
+                 parent: str = None, metadata: dict = None):
         super().__init__(reference, title, description, security_tag, parent, metadata)
         self.entity_type = EntityType.CONTENT_OBJECT
         self.representation_type = None
@@ -336,7 +551,38 @@ class ContentObject(Entity):
         self.tag = "ContentObject"
 
 
-def content_api_identifier_to_type(ref):
+EntityT = TypeVar("EntityT", Folder, Asset, ContentObject, None)
+
+
+class Representation:
+    """
+        Class to represent the Representation Object in the Preservica data model
+    """
+
+    def __init__(self, asset: Asset, rep_type: str, name: str, url: str):
+        self.asset = asset
+        self.rep_type = rep_type
+        self.name = name
+        self.url = url
+
+    def __str__(self):
+        return f"Type:\t\t\t{self.rep_type}\n" \
+               f"Name:\t\t\t{self.name}\n" \
+               f"URL:\t{self.url}"
+
+    def __repr__(self):
+        return self.__str__()
+
+
+def only_assets(entity: Entity):
+    return bool(entity.entity_type is EntityType.ASSET)
+
+
+def only_folders(entity: Entity):
+    return bool(entity.entity_type is EntityType.FOLDER)
+
+
+def content_api_identifier_to_type(ref: str):
     ref = ref.replace('sdb:', '')
     parts = ref.split("|")
     return tuple((EntityType(parts[0]), parts[1]))
@@ -348,19 +594,30 @@ class Thumbnail(Enum):
     LARGE = "large"
 
 
-class EntityType(Enum):
-    ASSET = "IO"
-    FOLDER = "SO"
-    CONTENT_OBJECT = "CO"
-
-
-def sanitize(filename):
-    """Return a fairly safe version of the filename.
+class AsyncProgress(Enum):
+    """
+    Enumeration of the possible status of an asynchronous process
+    """
+    ABORTED = "ABORTED"
+    ACTIVE = "ACTIVE"
+    COMPLETED = "COMPLETED"
+    PENDING = "PENDING"
+    SUSPENDING = "SUSPENDING"
+    SUSPENDED = "SUSPENDED"
+    UNKNOWN = "UNKNOWN"
+    FAILED = "FAILED"
+    FINISHED_MIXED_OUTCOME = "FINISHED_MIXED_OUTCOME"
+    CANCELLED = "CANCELLED"
+
+
+def sanitize(filename) -> str:
+    """
+    Return a fairly safe version of the filename.
 
     We don't limit ourselves to ascii, because we want to keep municipality
-    names, etc, but we do want to get rid of anything potentially harmful,
+    names, etc., but we do want to get rid of anything potentially harmful,
     and make sure we do not exceed Windows filename length limits.
-    Hence a less safe blacklist, rather than a whitelist.
+    Hence, a less safe blacklist, rather than a whitelist.
     """
     blacklist = ["\\", "/", ":", "*", "?", "\"", "<", ">", "|", "\0"]
     reserved = [
@@ -403,16 +660,91 @@ def sanitize(filename):
 
 class AuthenticatedAPI:
     """
-    Base class for authenticated calls which need an access token
+        Base class for authenticated calls which need an access token
+        Authenticated calls include a "Preservica-Access-Token" header in the request
     """
 
-    def entity_from_string(self, xml_data):
+    def _check_if_user_has_manager_role(self):
+        """
+        Check if the current user has a least a manager role
+        :return: None
+
+        Throws RuntimeError if the user does not have required roles
+        """
+        if ('ROLE_SDB_MANAGER_USER' not in self.roles) and ('ROLE_SDB_ADMIN_USER' not in self.roles):
+            logger.error(f"The AdminAPI requires the user to have ROLE_SDB_MANAGER_USER")
+            raise RuntimeError(f"The API requires the user to have at least the ROLE_SDB_MANAGER_USER")
+
+    def _find_user_roles_(self) -> list[str]:
+        """
+            Get a list of roles for the user
+            :return list of roles:
+        """
+        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/json'}
+        request = self.session.get(f"{self.protocol}://{self.server}/api/user/details", headers=headers)
+        logger.debug(request.headers)
+        if request.status_code == requests.codes.ok:
+            json_document = str(request.content.decode('utf-8'))
+            logger.debug(json_document)
+            roles: list[str] = json.loads(json_document)['roles']
+            return roles
+        elif request.status_code == requests.codes.unauthorized:
+            self.token = self.__token__()
+            return self._find_user_roles_()
+        return []
+
+
+    def security_tags_base(self, with_permissions: bool = False) -> dict:
+        """
+             Return  security tags available for the  current user
+
+             :return: dict of security tags
+             :rtype:  dict
+         """
+
+        if (self.major_version < 7) and (self.minor_version < 4) and (self.patch_version < 1):
+            raise RuntimeError("security_tags API call is only available with a Preservica v6.3.1 system or higher")
+
+        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/xml;charset=UTF-8'}
+
+        request = self.session.get(f'{self.protocol}://{self.server}/api/security/tags', headers=headers)
+        if request.status_code == requests.codes.ok:
+            xml_response = str(request.content.decode('utf-8'))
+            logger.debug(xml_response)
+            entity_response = xml.etree.ElementTree.fromstring(xml_response)
+            security_tags = {}
+            tags = entity_response.findall(f'.//{{{self.sec_ns}}}Tag')
+            for tag in tags:
+                if with_permissions:
+                    permissions = []
+                    for p in tag.findall(f'.//{{{self.sec_ns}}}Permission'):
+                        permissions.append(p.text)
+                    security_tags[tag.attrib['name']] = permissions
+                else:
+                    security_tags[tag.attrib['name']] = tag.attrib['name']
+            return security_tags
+        if request.status_code == requests.codes.unauthorized:
+            self.token = self.__token__()
+            return self.security_tags_base()
+        else:
+            logger.error(f'security_tags failed {request.status_code}')
+            raise RuntimeError(request.status_code, "security_tags failed")
+
+    def entity_from_string(self, xml_data: str) -> dict:
+        """
+        Create a basic entity from XML data
+
+        :param xml_data:
+        :return: dict
+        """
         entity_response = xml.etree.ElementTree.fromstring(xml_data)
         reference = entity_response.find(f'.//{{{self.xip_ns}}}Ref')
         title = entity_response.find(f'.//{{{self.xip_ns}}}Title')
         security_tag = entity_response.find(f'.//{{{self.xip_ns}}}SecurityTag')
         description = entity_response.find(f'.//{{{self.xip_ns}}}Description')
         parent = entity_response.find(f'.//{{{self.xip_ns}}}Parent')
+        custom_type = entity_response.find(f'.//{{{self.xip_ns}}}CustomType')
+
         if hasattr(parent, 'text'):
             parent = parent.text
         else:
@@ -423,14 +755,48 @@ class AuthenticatedAPI:
         for fragment in fragments:
             metadata[fragment.text] = fragment.attrib['schema']
 
-        return {'reference': reference.text, 'title': title.text if hasattr(title, 'text') else None,
-                'description': description.text if hasattr(description, 'text') else None,
-                'security_tag': security_tag.text, 'parent': parent, 'metadata': metadata}
+        entity_dict = {'reference': reference.text, 'title': title.text if hasattr(title, 'text') else None,
+                       'description': description.text if hasattr(description, 'text') else None,
+                       'security_tag': security_tag.text, 'parent': parent, 'metadata': metadata}
+
+        if hasattr(custom_type, 'text'):
+            entity_dict['CustomType'] = custom_type.text
+
+        return entity_dict
+
+    def edition(self) -> str:
+        """
+        Return the edition of this tenancy
+        """
+        if self.major_version < 8 and self.minor_version < 3:
+            raise RuntimeError("Entitlement API is only available when connected to a v7.3 System")
+
+        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/json'}
+
+        response = self.session.get(f'{self.protocol}://{self.server}/api/entitlement/edition', headers=headers)
+
+        if response.status_code == requests.codes.ok:
+            return response.json()['edition']
+        elif response.status_code == requests.codes.unauthorized:
+            self.token = self.__token__()
+            return self.edition()
+        else:
+            exception = HTTPException("", response.status_code, response.url,
+                                      "edition", response.content.decode('utf-8'))
+            logger.error(exception)
+            raise exception
 
     def __version_namespace__(self):
         """
         Generate version specific namespaces from the server version
         """
+        if self.major_version > 6:
+            self.xip_ns = f"{NS_XIP_ROOT}v{self.major_version}.{self.minor_version}"
+            self.entity_ns = f"{NS_ENTITY_ROOT}v{self.major_version}.{self.minor_version}"
+            self.rm_ns = f"{NS_RM_ROOT}v{6}.{2}"
+            self.sec_ns = f"{NS_SEC_ROOT}/v{self.major_version}.{self.minor_version}"
+            self.admin_ns = f"{NS_ADMIN}/v{self.major_version}.{self.minor_version}"
+
         if self.major_version == 6:
             if self.minor_version < 2:
                 self.xip_ns = NS_XIP_V6
@@ -439,13 +805,18 @@ class AuthenticatedAPI:
                 self.xip_ns = f"{NS_XIP_ROOT}v{self.major_version}.{self.minor_version}"
                 self.entity_ns = f"{NS_ENTITY_ROOT}v{self.major_version}.{self.minor_version}"
                 self.rm_ns = f"{NS_RM_ROOT}v{self.major_version}.{2}"
+                self.sec_ns = f"{NS_SEC_ROOT}/v{self.major_version}.{self.minor_version}"
+                self.admin_ns = f"{NS_ADMIN}/v{self.major_version}.{self.minor_version}"
+
+        xml.etree.ElementTree.register_namespace("xip", f"{self.xip_ns}")
 
     def __version_number__(self):
         """
         Determine the version number of the server
         """
         headers = {HEADER_TOKEN: self.token}
-        request = self.session.get(f'https://{self.server}/api/entity/versiondetails/version', headers=headers)
+        request = self.session.get(f'{self.protocol}://{self.server}/api/entity/versiondetails/version',
+                                   headers=headers)
         if request.status_code == requests.codes.ok:
             xml_ = str(request.content.decode('utf-8'))
             version = xml_[xml_.find("<CurrentVersion>") + len("<CurrentVersion>"):xml_.find("</CurrentVersion>")]
@@ -453,6 +824,7 @@ class AuthenticatedAPI:
             self.major_version = int(version_numbers[0])
             self.minor_version = int(version_numbers[1])
             self.patch_version = int(version_numbers[2])
+
             return version
         elif request.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
@@ -461,24 +833,31 @@ class AuthenticatedAPI:
             logger.error(f"version number failed with http response {request.status_code}")
             logger.error(str(request.content))
             RuntimeError(request.status_code, "version number failed")
+            return None
+
+
 
     def __str__(self):
-        return f"pyPreservica version: {pyPreservica.__version__}  (Preservica 6.2 Compatible) " \
-               f"Connected to: {self.server} Preservica version: {self.version} as {self.username}"
+        return f"pyPreservica version: {pyPreservica.__version__}  (Preservica 8.0 Compatible) " \
+               f"Connected to: {self.server} Preservica version: {self.version} as {self.username} " \
+               f"in tenancy {self.tenant}"
 
     def __repr__(self):
         return self.__str__()
 
     def save_config(self):
-        config = configparser.ConfigParser()
+        config = configparser.RawConfigParser(interpolation=None)
         config['credentials'] = {'username': self.username, 'password': self.password, 'tenant': self.tenant,
                                  'server': self.server}
-        with open('credentials.properties', 'wt') as configfile:
+        if self.two_fa_secret_key is not None:
+            config['credentials']['twoFactorToken'] = self.two_fa_secret_key
+
+        with open('credentials.properties', 'wt', encoding="utf-8") as configfile:
             config.write(configfile)
 
-    def manager_token(self, username, password):
+    def manager_token(self, username: str, password: str) -> str:
         data = {'username': username, 'password': password, 'tenant': self.tenant}
-        response = self.session.post(f'https://{self.server}/api/accesstoken/login', data=data)
+        response = self.session.post(f'{self.protocol}://{self.server}/api/accesstoken/login', data=data)
         if response.status_code == requests.codes.ok:
             return response.json()['token']
         else:
@@ -486,19 +865,58 @@ class AuthenticatedAPI:
             logger.error(msg)
             logger.error(response.status_code)
             logger.error(str(response.content))
-            RuntimeError(response.status_code, "Could not generate valid manager approval password")
+            RuntimeError(response.status_code, "Could not generate valid manager approval token")
 
-    def __token__(self):
+    def __token__(self) -> str:
+        """
+            Generate am API token to use to authenticate calls
+            :return: API Token
+        """
         logger.debug("Token Expired Requesting New Token")
         if self.shared_secret is False:
-            if self.tenant == "%":
-                data = {'username': self.username, 'password': self.password}
+            if self.tenant is None:
+                data = {'username': self.username, 'password': self.password, 'includeUserDetails': 'true'}
             else:
                 data = {'username': self.username, 'password': self.password, 'tenant': self.tenant}
-            response = self.session.post(f'https://{self.server}/api/accesstoken/login', data=data)
+            response = self.session.post(f'{self.protocol}://{self.server}/api/accesstoken/login', data=data)
             if response.status_code == requests.codes.ok:
+                if self.tenant is None:
+                    self.tenant = response.json()['tenant']
                 return response.json()['token']
             else:
+                if 'message' in response.json():
+                    if response.json()['message'] == "needs.2fa":
+                        logger.debug("2FA Found")
+                        if self.tenant is None:
+                            self.tenant = response.json()['tenant']
+                        if self.two_fa_secret_key:
+                            logger.debug("Found Two Factor Token")
+                            totp = pyotp.TOTP(self.two_fa_secret_key)
+                            data = {'username': self.username,
+                                    'continuationToken': response.json()['continuationToken'],
+                                    'tenant': self.tenant, 'twoFactorToken': totp.now()}
+
+                            header = {'Content-Type': 'application/x-www-form-urlencoded'}
+                            response_2fa = self.session.post(
+                                f'{self.protocol}://{self.server}/api/accesstoken/complete-2fa',
+                                data=data, headers=header)
+                            if response_2fa.status_code == requests.codes.ok:
+                                return response_2fa.json()['token']
+                            else:
+                                msg = "Failed to create a 2FA authentication token. Check your credentials are correct"
+                                logger.error(msg)
+                                logger.error(str(response_2fa.content))
+                                raise RuntimeError(response_2fa.status_code, msg)
+                        else:
+                            msg = "2FA twoFactorToken required to authenticate against this account using 2FA"
+                            logger.error(msg)
+                            logger.error(str(response.content))
+                            raise RuntimeError(response.status_code, msg)
+                    if response.json()['message'] == "needs.2fa.setup":
+                        msg = "2FA is activated but not yet set up"
+                        logger.error(msg)
+                        logger.error(str(response.content))
+                        raise RuntimeError(response.status_code, msg)
                 msg = "Failed to create a password based authentication token. Check your credentials are correct"
                 logger.error(msg)
                 logger.error(str(response.content))
@@ -511,7 +929,7 @@ class AuthenticatedAPI:
             sha1 = hashlib.sha1()
             sha1.update(to_hash.encode(encoding='utf-8'))
             data = {"username": self.username, "tenant": self.tenant, "timestamp": timestamp, "hash": sha1.hexdigest()}
-            response = self.session.post(f'https://{self.server}/{endpoint}', data=data)
+            response = self.session.post(f'{self.protocol}://{self.server}/{endpoint}', data=data)
             if response.status_code == requests.codes.ok:
                 return response.json()['token']
             else:
@@ -519,12 +937,40 @@ class AuthenticatedAPI:
                 logger.error(msg)
                 raise RuntimeError(response.status_code, msg)
 
-    def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False):
-        config = configparser.ConfigParser()
-        config.read('credentials.properties')
-        self.session = requests.Session()
-        self.shared_secret = bool(use_shared_secret)
+    def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
+                 use_shared_secret: bool = False, two_fa_secret_key: str = None,
+                 protocol: str = "https", request_hook=None, credentials_path: str = 'credentials.properties'):
+
+        config = configparser.ConfigParser(interpolation=configparser.Interpolation())
+        config.read(os.path.relpath(credentials_path), encoding='utf-8')
+        self.session: Session = requests.Session()
+
+        if request_hook is not None:
+            self.session.hooks['response'].append(request_hook)
 
+        retries = Retry(
+            total=3,
+            backoff_factor=0.1,
+            status_forcelist=[502, 503, 504],
+            allowed_methods=Retry.DEFAULT_ALLOWED_METHODS
+        )
+
+        self.shared_secret: bool = bool(use_shared_secret)
+        self.protocol = protocol
+        self.two_fa_secret_key = two_fa_secret_key
+
+        self.session.mount(f'{self.protocol}://', HTTPAdapter(max_retries=retries))
+
+        self.session.request = functools.partial(self.session.request, timeout=TIME_OUT)
+
+        if not two_fa_secret_key:
+            two_fa_secret_key = os.environ.get('PRESERVICA_2FA_TOKEN')
+            if two_fa_secret_key is None:
+                try:
+                    two_fa_secret_key = config['credentials']['twoFactorToken']
+                except KeyError:
+                    pass
+        self.two_fa_secret_key = two_fa_secret_key
         if not username:
             username = os.environ.get('PRESERVICA_USERNAME')
             if username is None:
@@ -562,10 +1008,8 @@ class AuthenticatedAPI:
                     pass
         if not tenant:
             msg = "No valid tenant found in method arguments, environment variables or credentials.properties file"
-            logger.error(msg)
-            raise RuntimeError(msg)
-        else:
-            self.tenant = tenant
+            logger.debug(msg)
+        self.tenant = tenant
 
         if not server:
             server = os.environ.get('PRESERVICA_SERVER')
@@ -584,7 +1028,22 @@ class AuthenticatedAPI:
         self.token = self.__token__()
         self.version = self.__version_number__()
         self.__version_namespace__()
+        self.roles = self._find_user_roles_()
+
+        self.session.headers.update({'User-Agent': f'pyPreservica SDK/({pyPreservica.__version__}) '
+                                                   f' ({platform.platform()}/{os.name}/{sys.platform})'})
 
-        logger.debug(str(self))
         logger.debug(self.xip_ns)
         logger.debug(self.entity_ns)
+
+def parse_date_to_iso(date):
+    try:
+        date = datetime.datetime.fromisoformat(date.replace('Z','+0000'))
+        if date.tzinfo is None or date.tzinfo.utcoffset(date) is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        date = date.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
+    except ValueError:
+        date = dateutil.parser.parse(date)
+        if date.tzinfo is None or date.tzinfo.utcoffset(date) is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        date = date.strftime('%Y-%m-%dT%H:%M:%S.%f%z')