pyPreservica 2.0.3__py3-none-any.whl → 3.3.3__py3-none-any.whl

pyPreservica/common.py

@@ -7,12 +7,13 @@ author:     James Carr
 licence:    Apache License 2.0
 
 """
-
 import configparser
+import functools
 import hashlib
 import json
 import logging
 import os
+import platform
 import re
 import sys
 import threading
@@ -22,15 +23,18 @@ import xml.etree.ElementTree
 from enum import Enum
 from pathlib import Path
 import pyotp
-
+from requests import Session
+from urllib3.util import Retry
 import requests
+from requests.adapters import HTTPAdapter
+from typing import TypeVar
+from datetime import datetime
+import dateutil
 
 import pyPreservica
 
 logger = logging.getLogger(__name__)
 
-CHUNK_SIZE = 1024 * 2
-
 NS_XIP_ROOT = "http://preservica.com/XIP/"
 NS_ENTITY_ROOT = "http://preservica.com/EntityAPI/"
 NS_RM_ROOT = "http://preservica.com/RetentionManagement/"
@@ -50,6 +54,8 @@ SO_PATH = "structural-objects"
 CO_PATH = "content-objects"
 
 HASH_BLOCK_SIZE = 65536
+TIME_OUT = 62
+CHUNK_SIZE = 1024 * 4
 
 
 class FileHash:
@@ -76,9 +82,9 @@ class FileHash:
 
 def identifiersToDict(identifiers: set) -> dict:
     """
-    Convert a set of tuples to a dict
-    :param identifiers:
-    :return:
+        Convert a set of tuples to a dict
+        :param identifiers:
+        :return:
     """
     result = {}
     for identifier_tuple in identifiers:
@@ -124,7 +130,7 @@ def _make_stored_zipfile(base_name, base_dir, owner, group, verbose=0, dry_run=0
 
     if logger is not None:
         logger.info("creating '%s' and adding '%s' to it",
-                     zip_filename, base_dir)
+                    zip_filename, base_dir)
 
     if not dry_run:
         with zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_STORED) as zf:
@@ -402,21 +408,45 @@ class Bitstream:
         self.length = int(length)
         self.fixity = fixity
         self.content_url = content_url
+        self.bs_index = None
+        self.gen_index = None
+        self.co_ref = None
 
     def __str__(self):
-        return f"Filename:\t\t\t{self.filename}\n" \
-               f"FileSize:\t\t\t{self.length}\n" \
-               f"Content:\t{self.content_url}\n" \
-               f"Fixity:\t{self.fixity}"
+        return f"""
+           Filename:       {self.filename}
+           File Length:    {self.length}
+           Fixity:         {self.fixity}
+        """
 
     def __repr__(self):
         return self.__str__()
 
 
+class ExternIdentifier:
+    """
+        Class to represent the External Identifier Object in the Preservica data model
+    """
+
+    def __init__(self, identifier_type: str, identifier_value: str):
+        self.type = identifier_type
+        self.value = identifier_value
+        self.id = None
+
+    def __str__(self):
+        return f"""
+           Identifier:          {self.id}
+           Identifier Type:     {self.type}
+           Identifier Value:    {self.value}
+        """
+
+    def __repr__(self):
+        return self.__str__()
+
 class Generation:
     """
          Class to represent the Generation Object in the Preservica data model
-     """
+    """
 
     def __init__(self, original: bool, active: bool, format_group: str, effective_date: str, bitstreams: list):
         self.original = bool(original)
@@ -425,11 +455,18 @@ class Generation:
         self.format_group = format_group
         self.effective_date = effective_date
         self.bitstreams = bitstreams
+        self.properties = list()
+        self.formats = list()
 
     def __str__(self):
-        return f"Active:\t\t\t{self.active}\n" \
-               f"Original:\t\t\t{self.original}\n" \
-               f"Format_group:\t{self.format_group}"
+        return f"""
+           Active:         {self.active}
+           Original:       {self.original}
+           Format Group:   {self.format_group}
+           Effective Date: {self.effective_date}
+           Formats:        {self.formats}
+           Properties:     {self.properties}
+           """
 
     def __repr__(self):
         return self.__str__()
@@ -453,27 +490,23 @@ class Entity:
         self.custom_type = None
 
     def __str__(self):
-        if self.custom_type is None:
-            return f"Ref:\t\t\t{self.reference}\n" \
-                   f"Title:\t\t\t{self.title}\n" \
-                   f"Description:\t{self.description}\n" \
-                   f"Security Tag:\t{self.security_tag}\n" \
-                   f"Parent:\t\t\t{self.parent}\n\n"
-        else:
-            return f"Ref:\t\t\t{self.reference}\n" \
-                   f"Title:\t\t\t{self.title}\n" \
-                   f"Description:\t{self.description}\n" \
-                   f"Security Tag:\t{self.security_tag}\n" \
-                   f"Parent:\t\t\t{self.parent}\n" \
-                   f"Type:\t\t\t{self.custom_type}\n\n"
+        return f"""
+            Entity:         {self.entity_type}
+            Entity Ref:     {self.reference}
+            Title:          {self.title}
+            Description:    {self.description}
+            Security Tag:   {self.security_tag}
+            Parent:         {self.parent}
+            Custom Type:    {self.custom_type}
+            """
 
     def __repr__(self):
         return self.__str__()
 
-    def has_metadata(self):
+    def has_metadata(self) -> bool:
         return bool(self.metadata)
 
-    def metadata_namespaces(self):
+    def metadata_namespaces(self) -> list:
         return list(self.metadata.values())
 
 
@@ -518,6 +551,9 @@ class ContentObject(Entity):
         self.tag = "ContentObject"
 
 
+EntityT = TypeVar("EntityT", Folder, Asset, ContentObject, None)
+
+
 class Representation:
     """
         Class to represent the Representation Object in the Preservica data model
@@ -558,14 +594,30 @@ class Thumbnail(Enum):
     LARGE = "large"
 
 
+class AsyncProgress(Enum):
+    """
+    Enumeration of the possible status of an asynchronous process
+    """
+    ABORTED = "ABORTED"
+    ACTIVE = "ACTIVE"
+    COMPLETED = "COMPLETED"
+    PENDING = "PENDING"
+    SUSPENDING = "SUSPENDING"
+    SUSPENDED = "SUSPENDED"
+    UNKNOWN = "UNKNOWN"
+    FAILED = "FAILED"
+    FINISHED_MIXED_OUTCOME = "FINISHED_MIXED_OUTCOME"
+    CANCELLED = "CANCELLED"
+
+
 def sanitize(filename) -> str:
     """
     Return a fairly safe version of the filename.
 
     We don't limit ourselves to ascii, because we want to keep municipality
-    names, etc, but we do want to get rid of anything potentially harmful,
+    names, etc., but we do want to get rid of anything potentially harmful,
     and make sure we do not exceed Windows filename length limits.
-    Hence a less safe blacklist, rather than a whitelist.
+    Hence, a less safe blacklist, rather than a whitelist.
     """
     blacklist = ["\\", "/", ":", "*", "?", "\"", "<", ">", "|", "\0"]
     reserved = [
@@ -608,27 +660,39 @@ def sanitize(filename) -> str:
 
 class AuthenticatedAPI:
     """
-    Base class for authenticated calls which need access token
+        Base class for authenticated calls which need an access token
+        Authenticated calls include a "Preservica-Access-Token" header in the request
     """
 
     def _check_if_user_has_manager_role(self):
+        """
+        Check if the current user has a least a manager role
+        :return: None
+
+        Throws RuntimeError if the user does not have required roles
+        """
         if ('ROLE_SDB_MANAGER_USER' not in self.roles) and ('ROLE_SDB_ADMIN_USER' not in self.roles):
             logger.error(f"The AdminAPI requires the user to have ROLE_SDB_MANAGER_USER")
-            raise RuntimeError(f"The AdminAPI requires the user to have ROLE_SDB_MANAGER_USER")
+            raise RuntimeError(f"The API requires the user to have at least the ROLE_SDB_MANAGER_USER")
 
-    def _find_user_roles_(self) -> list:
+    def _find_user_roles_(self) -> list[str]:
         """
-        Get a list of roles for the user
-        :return list of roles:
+            Get a list of roles for the user
+            :return list of roles:
         """
-        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/xml;charset=UTF-8'}
+        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/json'}
         request = self.session.get(f"{self.protocol}://{self.server}/api/user/details", headers=headers)
+        logger.debug(request.headers)
         if request.status_code == requests.codes.ok:
-            roles = json.loads(str(request.content.decode('utf-8')))['roles']
+            json_document = str(request.content.decode('utf-8'))
+            logger.debug(json_document)
+            roles: list[str] = json.loads(json_document)['roles']
             return roles
         elif request.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
             return self._find_user_roles_()
+        return []
+
 
     def security_tags_base(self, with_permissions: bool = False) -> dict:
         """
@@ -651,10 +715,10 @@ class AuthenticatedAPI:
             security_tags = {}
             tags = entity_response.findall(f'.//{{{self.sec_ns}}}Tag')
             for tag in tags:
-                permissions = []
-                for p in tag.findall(f'.//{{{self.sec_ns}}}Permission'):
-                    permissions.append(p.text)
                 if with_permissions:
+                    permissions = []
+                    for p in tag.findall(f'.//{{{self.sec_ns}}}Permission'):
+                        permissions.append(p.text)
                     security_tags[tag.attrib['name']] = permissions
                 else:
                     security_tags[tag.attrib['name']] = tag.attrib['name']
@@ -667,6 +731,12 @@ class AuthenticatedAPI:
             raise RuntimeError(request.status_code, "security_tags failed")
 
     def entity_from_string(self, xml_data: str) -> dict:
+        """
+        Create a basic entity from XML data
+
+        :param xml_data:
+        :return: dict
+        """
         entity_response = xml.etree.ElementTree.fromstring(xml_data)
         reference = entity_response.find(f'.//{{{self.xip_ns}}}Ref')
         title = entity_response.find(f'.//{{{self.xip_ns}}}Title')
@@ -694,10 +764,39 @@ class AuthenticatedAPI:
 
         return entity_dict
 
+    def edition(self) -> str:
+        """
+        Return the edition of this tenancy
+        """
+        if self.major_version < 8 and self.minor_version < 3:
+            raise RuntimeError("Entitlement API is only available when connected to a v7.3 System")
+
+        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/json'}
+
+        response = self.session.get(f'{self.protocol}://{self.server}/api/entitlement/edition', headers=headers)
+
+        if response.status_code == requests.codes.ok:
+            return response.json()['edition']
+        elif response.status_code == requests.codes.unauthorized:
+            self.token = self.__token__()
+            return self.edition()
+        else:
+            exception = HTTPException("", response.status_code, response.url,
+                                      "edition", response.content.decode('utf-8'))
+            logger.error(exception)
+            raise exception
+
     def __version_namespace__(self):
         """
         Generate version specific namespaces from the server version
         """
+        if self.major_version > 6:
+            self.xip_ns = f"{NS_XIP_ROOT}v{self.major_version}.{self.minor_version}"
+            self.entity_ns = f"{NS_ENTITY_ROOT}v{self.major_version}.{self.minor_version}"
+            self.rm_ns = f"{NS_RM_ROOT}v{6}.{2}"
+            self.sec_ns = f"{NS_SEC_ROOT}/v{self.major_version}.{self.minor_version}"
+            self.admin_ns = f"{NS_ADMIN}/v{self.major_version}.{self.minor_version}"
+
         if self.major_version == 6:
             if self.minor_version < 2:
                 self.xip_ns = NS_XIP_V6
@@ -709,6 +808,8 @@ class AuthenticatedAPI:
                 self.sec_ns = f"{NS_SEC_ROOT}/v{self.major_version}.{self.minor_version}"
                 self.admin_ns = f"{NS_ADMIN}/v{self.major_version}.{self.minor_version}"
 
+        xml.etree.ElementTree.register_namespace("xip", f"{self.xip_ns}")
+
     def __version_number__(self):
         """
         Determine the version number of the server
@@ -723,6 +824,7 @@ class AuthenticatedAPI:
             self.major_version = int(version_numbers[0])
             self.minor_version = int(version_numbers[1])
             self.patch_version = int(version_numbers[2])
+
             return version
         elif request.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
@@ -731,9 +833,12 @@ class AuthenticatedAPI:
             logger.error(f"version number failed with http response {request.status_code}")
             logger.error(str(request.content))
             RuntimeError(request.status_code, "version number failed")
+            return None
+
+
 
     def __str__(self):
-        return f"pyPreservica version: {pyPreservica.__version__}  (Preservica 6.9 Compatible) " \
+        return f"pyPreservica version: {pyPreservica.__version__}  (Preservica 8.0 Compatible) " \
                f"Connected to: {self.server} Preservica version: {self.version} as {self.username} " \
                f"in tenancy {self.tenant}"
 
@@ -750,7 +855,7 @@ class AuthenticatedAPI:
         with open('credentials.properties', 'wt', encoding="utf-8") as configfile:
             config.write(configfile)
 
-    def manager_token(self, username: str, password: str):
+    def manager_token(self, username: str, password: str) -> str:
         data = {'username': username, 'password': password, 'tenant': self.tenant}
         response = self.session.post(f'{self.protocol}://{self.server}/api/accesstoken/login', data=data)
         if response.status_code == requests.codes.ok:
@@ -760,9 +865,13 @@ class AuthenticatedAPI:
             logger.error(msg)
             logger.error(response.status_code)
             logger.error(str(response.content))
-            RuntimeError(response.status_code, "Could not generate valid manager approval password")
+            RuntimeError(response.status_code, "Could not generate valid manager approval token")
 
-    def __token__(self):
+    def __token__(self) -> str:
+        """
+            Generate am API token to use to authenticate calls
+            :return: API Token
+        """
         logger.debug("Token Expired Requesting New Token")
         if self.shared_secret is False:
             if self.tenant is None:
@@ -781,20 +890,23 @@ class AuthenticatedAPI:
                         if self.tenant is None:
                             self.tenant = response.json()['tenant']
                         if self.two_fa_secret_key:
+                            logger.debug("Found Two Factor Token")
                             totp = pyotp.TOTP(self.two_fa_secret_key)
                             data = {'username': self.username,
                                     'continuationToken': response.json()['continuationToken'],
                                     'tenant': self.tenant, 'twoFactorToken': totp.now()}
+
+                            header = {'Content-Type': 'application/x-www-form-urlencoded'}
                             response_2fa = self.session.post(
                                 f'{self.protocol}://{self.server}/api/accesstoken/complete-2fa',
-                                data=data)
+                                data=data, headers=header)
                             if response_2fa.status_code == requests.codes.ok:
                                 return response_2fa.json()['token']
                             else:
                                 msg = "Failed to create a 2FA authentication token. Check your credentials are correct"
                                 logger.error(msg)
-                                logger.error(str(response.content))
-                                raise RuntimeError(response.status_code, msg)
+                                logger.error(str(response_2fa.content))
+                                raise RuntimeError(response_2fa.status_code, msg)
                         else:
                             msg = "2FA twoFactorToken required to authenticate against this account using 2FA"
                             logger.error(msg)
@@ -826,13 +938,30 @@ class AuthenticatedAPI:
                 raise RuntimeError(response.status_code, msg)
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
-                 use_shared_secret: bool = False, two_fa_secret_key: str = None, protocol: str = "https"):
+                 use_shared_secret: bool = False, two_fa_secret_key: str = None,
+                 protocol: str = "https", request_hook=None, credentials_path: str = 'credentials.properties'):
 
         config = configparser.ConfigParser(interpolation=configparser.Interpolation())
-        config.read('credentials.properties', encoding='utf-8')
-        self.session = requests.Session()
-        self.shared_secret = bool(use_shared_secret)
+        config.read(os.path.relpath(credentials_path), encoding='utf-8')
+        self.session: Session = requests.Session()
+
+        if request_hook is not None:
+            self.session.hooks['response'].append(request_hook)
+
+        retries = Retry(
+            total=3,
+            backoff_factor=0.1,
+            status_forcelist=[502, 503, 504],
+            allowed_methods=Retry.DEFAULT_ALLOWED_METHODS
+        )
+
+        self.shared_secret: bool = bool(use_shared_secret)
         self.protocol = protocol
+        self.two_fa_secret_key = two_fa_secret_key
+
+        self.session.mount(f'{self.protocol}://', HTTPAdapter(max_retries=retries))
+
+        self.session.request = functools.partial(self.session.request, timeout=TIME_OUT)
 
         if not two_fa_secret_key:
             two_fa_secret_key = os.environ.get('PRESERVICA_2FA_TOKEN')
@@ -901,5 +1030,20 @@ class AuthenticatedAPI:
         self.__version_namespace__()
         self.roles = self._find_user_roles_()
 
+        self.session.headers.update({'User-Agent': f'pyPreservica SDK/({pyPreservica.__version__}) '
+                                                   f' ({platform.platform()}/{os.name}/{sys.platform})'})
+
         logger.debug(self.xip_ns)
         logger.debug(self.entity_ns)
+
+def parse_date_to_iso(date):
+    try:
+        date = datetime.datetime.fromisoformat(date.replace('Z','+0000'))
+        if date.tzinfo is None or date.tzinfo.utcoffset(date) is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        date = date.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
+    except ValueError:
+        date = dateutil.parser.parse(date)
+        if date.tzinfo is None or date.tzinfo.utcoffset(date) is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        date = date.strftime('%Y-%m-%dT%H:%M:%S.%f%z')