pyPreservica 0.9.9__py3-none-any.whl → 3.3.4__py3-none-any.whl

pyPreservica/contentAPI.py

@@ -10,14 +10,43 @@ licence:    Apache License 2.0
 """
 
 import csv
+from io import BytesIO
+from typing import Generator, Callable, Optional, Union
 from pyPreservica.common import *
 
 logger = logging.getLogger(__name__)
 
+class SortOrder(Enum):
+    asc = 1
+    desc = 2
+
+class Field:
+    name: str
+    value: Optional[str]
+    operator: Optional[str]
+    sort_order: Optional[SortOrder]
+
+    def __init__(self, name: str, value: str, operator: Optional[str]=None, sort_order: Optional[SortOrder]=None):
+        self.name = name
+        self.value = value
+        self.operator = operator
+        self.sort_order = sort_order
+
+
+
 class ContentAPI(AuthenticatedAPI):
+    """
+        The ContentAPI class provides the search interface to the Preservica repository.
+
+    """
+
 
-    def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False):
-        super().__init__(username, password, tenant, server, use_shared_secret)
+    def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
+                 use_shared_secret: bool = False, two_fa_secret_key: str = None,
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
+
+        super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
+                         protocol, request_hook, credentials_path)
         self.callback = None
 
     class SearchResult:
@@ -31,10 +60,30 @@ class ContentAPI(AuthenticatedAPI):
     def search_callback(self, fn):
         self.callback = fn
 
-    def object_details(self, entity_type, reference):
+    def user_security_tags(self, with_permissions: bool = False):
+        """
+             Return available security tags
+
+             :return: dict of security tags
+             :rtype:  dict
+         """
+
+        return self.security_tags_base(with_permissions=with_permissions)
+
+    def object_details(self, entity_type, reference: str) -> dict:
+        """
+
+        :param entity_type:
+        :param reference:
+        :return: Dictionary of object attributes
+        """
         headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/json'}
-        params = {'id': f'sdb:{entity_type.value}|{reference}'}
-        request = requests.get(f'https://{self.server}/api/content/object-details', params=params, headers=headers)
+        if type(entity_type) == EntityType:
+            params = {'id': f'sdb:{entity_type.value}|{reference}'}
+        else:
+            params = {'id': f'sdb:{entity_type}|{reference}'}
+        request = self.session.get(f'{self.protocol}://{self.server}/api/content/object-details', params=params,
+                                   headers=headers)
         if request.status_code == requests.codes.ok:
             return request.json()["value"]
         elif request.status_code == requests.codes.not_found:
@@ -47,11 +96,34 @@ class ContentAPI(AuthenticatedAPI):
             logger.error(f"object_details failed with error code: {request.status_code}")
             raise RuntimeError(request.status_code, f"object_details failed with error code: {request.status_code}")
 
+
+    def download_bytes(self, reference):
+        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/octet-stream'}
+        params = {'id': f'sdb:IO|{reference}'}
+        with self.session.get(f'{self.protocol}://{self.server}/api/content/download', params=params, headers=headers,
+                              stream=True) as req:
+            if req.status_code == requests.codes.ok:
+                file_bytes = BytesIO()
+                for chunk in req.iter_content(chunk_size=CHUNK_SIZE):
+                    file_bytes.write(chunk)
+                file_bytes.seek(0)
+                return file_bytes
+            elif req.status_code == requests.codes.unauthorized:
+                self.token = self.__token__()
+                return self.download_bytes(reference)
+            elif req.status_code == requests.codes.not_found:
+                logger.error(f"The requested asset reference is not found in the repository: {reference}")
+                raise RuntimeError(reference, "The requested reference is not found in the repository")
+            else:
+                logger.error(f"download failed with error code: {req.status_code}")
+                raise RuntimeError(req.status_code, f"download failed with error code: {req.status_code}")
+
+
     def download(self, reference, filename):
         headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/octet-stream'}
         params = {'id': f'sdb:IO|{reference}'}
-        with requests.get(f'https://{self.server}/api/content/download', params=params, headers=headers,
-                          stream=True) as req:
+        with self.session.get(f'{self.protocol}://{self.server}/api/content/download', params=params, headers=headers,
+                              stream=True) as req:
             if req.status_code == requests.codes.ok:
                 with open(filename, 'wb') as file:
                     for chunk in req.iter_content(chunk_size=CHUNK_SIZE):
@@ -69,27 +141,43 @@ class ContentAPI(AuthenticatedAPI):
                 logger.error(f"download failed with error code: {req.status_code}")
                 raise RuntimeError(req.status_code, f"download failed with error code: {req.status_code}")
 
+    def thumbnail_bytes(self, entity_type, reference: str, size: Thumbnail = Thumbnail.LARGE) -> Union[BytesIO, None]:
+        headers = {HEADER_TOKEN: self.token, 'accept': 'image/png'}
+        params = {'id': f'sdb:{entity_type}|{reference}', 'size': f'{size.value}'}
+        with self.session.get(f'{self.protocol}://{self.server}/api/content/thumbnail', params=params, headers=headers, stream=True) as req:
+            if req.status_code == requests.codes.ok:
+                file_bytes = BytesIO()
+                for chunk in req.iter_content(chunk_size=CHUNK_SIZE):
+                    file_bytes.write(chunk)
+                file_bytes.seek(0)
+                return file_bytes
+            elif req.status_code == requests.codes.unauthorized:
+                self.token = self.__token__()
+                return self.thumbnail_bytes(entity_type, reference, size)
+            elif req.status_code == requests.codes.not_found:
+                logger.error(req.content.decode("utf-8"))
+                logger.error(f"The requested reference is not found in the repository: {reference}")
+                raise RuntimeError(reference, "The requested reference is not found in the repository")
+            else:
+                logger.error(f"thumbnail failed with error code: {req.status_code}")
+                raise RuntimeError(req.status_code, f"thumbnail failed with error code: {req.status_code}")
+
     def thumbnail(self, entity_type, reference, filename, size=Thumbnail.LARGE):
-        headers = {HEADER_TOKEN: self.token, 'Content-Type': 'application/octet-stream'}
-        if entity_type == "IO":
-            params = {'id': f'sdb:IO|{reference}', 'size': f'{size.value}'}
-        elif entity_type == "SO":
-            params = {'id': f'sdb:SO|{reference}', 'size': f'{size.value}'}
-        else:
-            print(f"entity_type must be a folder or asset, IO or SO")
-            raise SystemExit
-        with requests.get(f'https://{self.server}/api/content/thumbnail', params=params, headers=headers) as req:
+        headers = {HEADER_TOKEN: self.token, 'accept': 'image/png'}
+        params = {'id': f'sdb:{entity_type}|{reference}', 'size': f'{size.value}'}
+        with self.session.get(f'{self.protocol}://{self.server}/api/content/thumbnail', params=params, headers=headers,
+                              stream=True) as req:
             if req.status_code == requests.codes.ok:
                 with open(filename, 'wb') as file:
                     for chunk in req.iter_content(chunk_size=CHUNK_SIZE):
                         file.write(chunk)
                         file.flush()
-                file.close()
                 return filename
             elif req.status_code == requests.codes.unauthorized:
                 self.token = self.__token__()
                 return self.thumbnail(entity_type, reference, filename, size)
             elif req.status_code == requests.codes.not_found:
+                logger.error(req.content.decode("utf-8"))
                 logger.error(f"The requested reference is not found in the repository: {reference}")
                 raise RuntimeError(reference, "The requested reference is not found in the repository")
             else:
@@ -98,12 +186,12 @@ class ContentAPI(AuthenticatedAPI):
 
     def indexed_fields(self):
         headers = {HEADER_TOKEN: self.token}
-        results = requests.get(f'https://{self.server}/api/content/indexed-fields', headers=headers)
+        results = self.session.get(f'{self.protocol}://{self.server}/api/content/indexed-fields', headers=headers)
         if results.status_code == requests.codes.ok:
-            fields = list()
+            fields = {}
             for ob in results.json()["value"]:
                 field = f'{ob["shortName"]}.{ob["index"]}'
-                fields.append(field)
+                fields[field] = ob["uri"]
             return fields
         elif results.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
@@ -112,52 +200,52 @@ class ContentAPI(AuthenticatedAPI):
             logger.error(f"indexed_fields failed with error code: {results.status_code}")
             raise RuntimeError(results.status_code, f"indexed_fields failed with error code: {results.status_code}")
 
-    def simple_search_csv(self, query: str = "%", csv_file="search.csv", *args):
-        page_size = 50
-        if len(args) == 0:
+    def simple_search_csv(self, query: str = "%", page_size: int = 50, csv_file="search.csv",
+                          list_indexes: list = None):
+        if list_indexes is None or len(list_indexes) == 0:
             metadata_fields = ["xip.reference", "xip.title", "xip.description", "xip.document_type",
                                "xip.parent_ref", "xip.security_descriptor"]
         else:
-            metadata_fields = list(*args)
+            metadata_fields = list(list_indexes)
         if "xip.reference" not in metadata_fields:
             metadata_fields.insert(0, "xip.reference")
         with open(csv_file, newline='', mode="wt", encoding="utf-8") as csv_file:
             writer = csv.DictWriter(csv_file, fieldnames=metadata_fields)
             writer.writeheader()
-            writer.writerows(self.simple_search_list(query, page_size, *args))
+            writer.writerows(self.simple_search_list(query, page_size, metadata_fields))
 
-    def simple_search_list(self, query: str = "%", page_size: int = 10, *args):
-        search_result = self.simple_search(query, 0, page_size, *args)
+    def simple_search_list(self, query: str = "%", page_size: int = 50, list_indexes: list = None):
+        search_result = self._simple_search(query, 0, page_size, list_indexes)
         for e in search_result.results_list:
             yield e
         found = len(search_result.results_list)
         while search_result.hits > found:
-            search_result = self.simple_search(query, found, page_size, *args)
+            search_result = self._simple_search(query, found, page_size, list_indexes)
             for e in search_result.results_list:
                 yield e
             found = found + len(search_result.results_list)
 
-    def simple_search(self, query: str = "%", start_index: int = 0, page_size: int = 10, *args):
+    def _simple_search(self, query: str = "%", start_index: int = 0, page_size: int = 10, list_indexes: list = None):
         start_from = str(start_index)
         headers = {'Content-Type': 'application/x-www-form-urlencoded', HEADER_TOKEN: self.token}
         query_term = ('{ "q":  "%s" }' % query)
-        if len(args) == 0:
+        if list_indexes is None or len(list_indexes) == 0:
             metadata_fields = "xip.title,xip.description,xip.document_type,xip.parent_ref,xip.security_descriptor"
         else:
-            metadata_fields = ','.join(*args)
+            metadata_fields = ','.join(list_indexes)
         payload = {'start': start_from, 'max': str(page_size), 'metadata': metadata_fields, 'q': query_term}
-        results = requests.post(f'https://{self.server}/api/content/search', data=payload,  headers=headers)
-        results_list = list()
+        results = self.session.post(f'{self.protocol}://{self.server}/api/content/search', data=payload,
+                                    headers=headers)
+        results_list = []
         if results.status_code == requests.codes.ok:
-            json = results.json()
-            metadata = json['value']['metadata']
-            refs = list(json['value']['objectIds'])
+            json_doc = results.json()
+            metadata = json_doc['value']['metadata']
+            refs = list(json_doc['value']['objectIds'])
             refs = list(map(lambda x: content_api_identifier_to_type(x), refs))
-            hits = int(json['value']['totalHits'])
+            hits = int(json_doc['value']['totalHits'])
 
             for m_row, r_row in zip(metadata, refs):
-                results_map = dict()
-                results_map['xip.reference'] = r_row[1]
+                results_map = {'xip.reference': r_row[1]}
                 for li in m_row:
                     results_map[li['name']] = li['value']
                 results_list.append(results_map)
@@ -171,65 +259,225 @@ class ContentAPI(AuthenticatedAPI):
             return search_results
         elif results.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
-            return self.simple_search(query, start_index, page_size, *args)
+            return self._simple_search(query, start_index, page_size, list_indexes)
         else:
             logger.error(f"search failed with error code: {results.status_code}")
             raise RuntimeError(results.status_code, f"simple_search failed with error code: {results.status_code}")
 
-    def search_index_filter_csv(self, query: str = "%", csv_file="search.csv", map_fields=None):
-        page_size = 50
-        if map_fields is None:
-            map_fields = dict()
-        if "xip.reference" not in map_fields:
-            map_fields["xip.reference"] = ""
+    def search_index_filter_csv(self, query: str = "%", csv_file="search.csv", page_size: int = 50,
+                                filter_values: dict = None,
+                                sort_values: dict = None):
+        if filter_values is None:
+            filter_values = {}
+        if "xip.reference" not in filter_values:
+            filter_values["xip.reference"] = ""
 
-        header_fields = list(map_fields.keys())
+        header_fields = list(filter_values.keys())
         index = header_fields.index("xip.reference")
         header_fields.insert(0, header_fields.pop(index))
         with open(csv_file, newline='', mode="wt", encoding="utf-8") as csv_file:
             writer = csv.DictWriter(csv_file, fieldnames=header_fields)
             writer.writeheader()
-            writer.writerows(self.search_index_filter_list(query, page_size, map_fields))
+            writer.writerows(self.search_index_filter_list(query, page_size, filter_values, sort_values))
+
+    def search_fields(self, query: str = "%",  fields: list[Field]=None,  page_size: int = 25) -> Generator:
+        """
+        Run a search query with multiple fields
 
-    def search_index_filter_list(self, query: str = "%", page_size: int = 10, map_fields=None):
-        search_result = self.search_index_filter(query, 0, page_size, map_fields)
+        :param query: The main search query.
+        :param fields:  List of search fields
+        :param page_size:  The default search page size
+        :return: search result
+        """
+
+        if self.major_version < 7 and self.minor_version < 5:
+            raise RuntimeError("search_fields API call is not available when connected to a v7.5 System")
+
+        search_result = self._search_fields(query=query, fields=fields, start_index=0, page_size=page_size)
         for e in search_result.results_list:
             yield e
         found = len(search_result.results_list)
         while search_result.hits > found:
-            search_result = self.search_index_filter(query, found, page_size, map_fields)
+            search_result = self._search_fields(query=query, fields=fields, start_index=found, page_size=page_size)
             for e in search_result.results_list:
                 yield e
             found = found + len(search_result.results_list)
 
-    def search_index_filter(self, query: str = "%", start_index: int = 0, page_size: int = 10, map_fields=None):
+    def _search_fields(self, query: str = "%", fields: list[Field]=None, start_index: int = 0, page_size: int = 25):
+
         start_from = str(start_index)
         headers = {'Content-Type': 'application/x-www-form-urlencoded', HEADER_TOKEN: self.token}
 
-        field_list = list()
-        for key, value in map_fields.items():
+        if fields is None:
+            fields = []
+
+        field_list = []
+        sort_list = []
+        metadata_elements = []
+        for field in fields:
+            metadata_elements.append(field.name)
+            if field.value is None or field.value == "":
+                field_list.append('{' f' "name": "{field.name}", "values": [] ' + '}')
+            elif field.operator == "NOT":
+                field_list.append('{' f' "name": "{field.name}", "values": ["{field.value}"], "operator": "NOT" ' + '}')
+            else:
+                field_list.append('{' f' "name": "{field.name}", "values": ["{field.value}"] ' + '}')
+
+            if field.sort_order is not None:
+                sort_list.append(f'{{"sortFields": ["{field.name}"], "sortOrder": "{field.sort_order.name}"}}')
+
+
+        filter_terms = ','.join(field_list)
+
+        if len(sort_list) == 0:
+            query_term = ('{ "q":  "%s",  "fields":  [ %s ] }' % (query, filter_terms))
+        else:
+            sort_terms = ','.join(sort_list)
+            query_term = ('{ "q":  "%s",  "fields":  [ %s ],  "sort": [ %s ]}' % (query, filter_terms, sort_terms))
+
+        if len(metadata_elements) == 0:
+            metadata_elements.append("xip.title")
+
+
+        payload = {'start': start_from, 'max': str(page_size), 'metadata': list(metadata_elements), 'q': query_term}
+        logger.debug(payload)
+        results = self.session.post(f'{self.protocol}://{self.server}/api/content/search', data=payload,
+                                    headers=headers)
+        results_list = []
+        if results.status_code == requests.codes.ok:
+            json_doc = results.json()
+            metadata = json_doc['value']['metadata']
+            refs = list(json_doc['value']['objectIds'])
+            refs = list(map(lambda x: content_api_identifier_to_type(x), refs))
+            hits = int(json_doc['value']['totalHits'])
+
+            for m_row, r_row in zip(metadata, refs):
+                results_map = {'xip.reference': r_row[1]}
+                for li in m_row:
+                    results_map[li['name']] = li['value']
+                results_list.append(results_map)
+            next_start = start_index + page_size
+
+            if self.callback is not None:
+                value = str(f'{len(results_list) + start_index}:{hits}')
+                self.callback(value)
+
+            search_results = self.SearchResult(metadata, refs, hits, results_list, next_start)
+            return search_results
+        elif results.status_code == requests.codes.unauthorized:
+            self.token = self.__token__()
+            return self._search_fields(query, fields, start_index, page_size)
+        else:
+            logger.error(f"search failed with error code: {results.status_code}")
+            raise RuntimeError(results.status_code, f"search_index_filter failed")
+
+    def search_index_filter_list(self, query: str = "%", page_size: int = 25, filter_values: dict = None,
+                                 sort_values: dict = None) -> Generator:
+        """
+        Run a search query with optional filters
+
+        :param query: The main search query.
+        :param page_size:  The default search page size
+        :param filter_values:  Dictionary of index names and values
+        :param sort_values:    Dictionary of sort index names and values
+        :return: search result
+        """
+        search_result = self._search_index_filter(query, 0, page_size, filter_values, sort_values)
+        for e in search_result.results_list:
+            yield e
+        found = len(search_result.results_list)
+        while search_result.hits > found:
+            search_result = self._search_index_filter(query, found, page_size, filter_values, sort_values)
+            for e in search_result.results_list:
+                yield e
+            found = found + len(search_result.results_list)
+
+    def search_index_filter_hits(self, query: str = "%", filter_values: dict = None) -> int:
+        """
+        Run a search query with filters and return the number of hits only
+
+        :param query: The main search query.
+        :param filter_values:  Dictionary of index names and values
+        :return: Number of search results
+        """
+        start_from = str(0)
+        headers = {'Content-Type': 'application/x-www-form-urlencoded', HEADER_TOKEN: self.token}
+
+        field_list = []
+        for key, value in filter_values.items():
             if value == "":
                 field_list.append('{' f' "name": "{key}", "values": [] ' + '}')
             else:
-                field_list.append('{' f' "name": "{key}", "values": ["{value}"] ' + '}')
+                if isinstance(value, str):
+                    field_list.append('{' f' "name": "{key}", "values": ["{value}"] ' + '}')
+                if isinstance(value, list):
+                    v = f' {",".join(f'"{w}"' for w in value)} '
+                    field_list.append('{' f' "name": "{key}", "values":[ {v} ]' '}')
 
         filter_terms = ','.join(field_list)
 
         query_term = ('{ "q":  "%s",  "fields":  [ %s ] }' % (query, filter_terms))
 
-        payload = {'start': start_from, 'max': str(page_size), 'metadata': list(map_fields.keys()), 'q': query_term}
-        results = requests.post(f'https://{self.server}/api/content/search', data=payload, headers=headers)
-        results_list = list()
+        payload = {'start': start_from, 'max': str(10), 'metadata': list(filter_values.keys()), 'q': query_term}
+        results = self.session.post(f'{self.protocol}://{self.server}/api/content/search', data=payload,
+                                    headers=headers)
         if results.status_code == requests.codes.ok:
-            json = results.json()
-            metadata = json['value']['metadata']
-            refs = list(json['value']['objectIds'])
+            json_doc = results.json()
+            return int(json_doc['value']['totalHits'])
+        elif results.status_code == requests.codes.unauthorized:
+            self.token = self.__token__()
+            return self.search_index_filter_hits(query, filter_values)
+        else:
+            logger.error(f"search failed with error code: {results.status_code}")
+            raise RuntimeError(results.status_code, f"_search_index_filter_hits failed")
+
+    def _search_index_filter(self, query: str = "%", start_index: int = 0, page_size: int = 25,
+                             filter_values: dict = None, sort_values: dict = None):
+        start_from = str(start_index)
+        headers = {'Content-Type': 'application/x-www-form-urlencoded', HEADER_TOKEN: self.token}
+
+        if filter_values is None:
+            filter_values = {}
+
+        field_list = []
+        for key, value in filter_values.items():
+            if value == "":
+                field_list.append('{' f' "name": "{key}", "values": [] ' + '}')
+            else:
+                if isinstance(value, str):
+                    field_list.append('{' f' "name": "{key}", "values": ["{value}"] ' + '}')
+                if isinstance(value, list):
+                    v = f' {",".join(f'"{w}"' for w in value)} '
+                    field_list.append('{' f' "name": "{key}", "values":[ {v} ]' '}')
+
+        filter_terms = ','.join(field_list)
+
+        if sort_values is None:
+            query_term = ('{ "q":  "%s",  "fields":  [ %s ] }' % (query, filter_terms))
+        else:
+            sort_list = []
+            for key, value in sort_values.items():
+                direction = "asc"
+                if str(value).lower().startswith("d"):
+                    direction = "desc"
+                sort_list.append(f'{{"sortFields": ["{key}"], "sortOrder": "{direction}"}}')
+            sort_terms = ','.join(sort_list)
+            query_term = ('{ "q":  "%s",  "fields":  [ %s ],  "sort": [ %s ]}' % (query, filter_terms, sort_terms))
+
+        payload = {'start': start_from, 'max': str(page_size), 'metadata': list(filter_values.keys()), 'q': query_term}
+        logger.debug(payload)
+        results = self.session.post(f'{self.protocol}://{self.server}/api/content/search', data=payload,
+                                    headers=headers)
+        results_list = []
+        if results.status_code == requests.codes.ok:
+            json_doc = results.json()
+            metadata = json_doc['value']['metadata']
+            refs = list(json_doc['value']['objectIds'])
             refs = list(map(lambda x: content_api_identifier_to_type(x), refs))
-            hits = int(json['value']['totalHits'])
+            hits = int(json_doc['value']['totalHits'])
 
             for m_row, r_row in zip(metadata, refs):
-                results_map = dict()
-                results_map['xip.reference'] = r_row[1]
+                results_map = {'xip.reference': r_row[1]}
                 for li in m_row:
                     results_map[li['name']] = li['value']
                 results_list.append(results_map)
@@ -243,7 +491,25 @@ class ContentAPI(AuthenticatedAPI):
             return search_results
         elif results.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
-            return self.search_index_filter(query, start_index, page_size, map_fields)
+            return self._search_index_filter(query, start_index, page_size, filter_values)
         else:
             logger.error(f"search failed with error code: {results.status_code}")
             raise RuntimeError(results.status_code, f"search_index_filter failed")
+
+    class ReportProgressCallBack:
+        def __init__(self):
+            self.current = 0
+            self.total = 0
+            self._lock = threading.Lock()
+
+        def __call__(self, value):
+            with self._lock:
+                values = value.split(":")
+                self.total = int(values[1])
+                self.current = int(values[0])
+                if self.total == 0:
+                    percentage = 100.0
+                else:
+                    percentage = (self.current / self.total) * 100
+                sys.stdout.write("\rProcessing Hits %s from %s  (%.2f%%)" % (self.current, self.total, percentage))
+                sys.stdout.flush()