pyPreservica 3.0.0__tar.gz → 3.0.2__tar.gz

{pypreservica-3.0.0 → pypreservica-3.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pyPreservica
-Version: 3.0.0
+Version: 3.0.2
 Summary: Python library for the Preservica API
 Home-page: https://pypreservica.readthedocs.io/
 Author: James Carr
@@ -30,6 +30,17 @@ Requires-Dist: s3transfer
 Requires-Dist: azure-storage-blob
 Requires-Dist: tqdm
 Requires-Dist: pyotp
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: summary
 
 
 # pyPreservica

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/__init__.py

@@ -23,6 +23,6 @@ from .mdformsAPI import MetadataGroupsAPI, Group, GroupField, GroupFieldType
 __author__ = "James Carr (drjamescarr@gmail.com)"
 
 # Version of the pyPreservica package
-__version__ = "3.0.0"
+__version__ = "3.0.2"
 
 __license__ = "Apache License Version 2.0"

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/common.py

@@ -405,6 +405,9 @@ class Bitstream:
         self.length = int(length)
         self.fixity = fixity
         self.content_url = content_url
+        self.bs_index = None
+        self.gen_index = None
+        self.co_ref = None
 
     def __str__(self):
         return f"""
@@ -880,10 +883,10 @@ class AuthenticatedAPI:
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook=None):
+                 protocol: str = "https", request_hook=None, credentials_path: str = 'credentials.properties'):
 
         config = configparser.ConfigParser(interpolation=configparser.Interpolation())
-        config.read('credentials.properties', encoding='utf-8')
+        config.read(os.path.relpath(credentials_path), encoding='utf-8')
         self.session: Session = requests.Session()
 
         if request_hook is not None:

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/contentAPI.py

@@ -35,11 +35,12 @@ class Field:
 
 class ContentAPI(AuthenticatedAPI):
 
-    def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False,
-                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None):
+    def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
+                 use_shared_secret: bool = False, two_fa_secret_key: str = None,
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
         self.callback = None
 
     class SearchResult:

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/entityAPI.py

@@ -8,7 +8,7 @@ author:     James Carr
 licence:    Apache License 2.0
 
 """
-import hashlib
+
 import os.path
 import uuid
 import xml.etree.ElementTree
@@ -35,10 +35,10 @@ class EntityAPI(AuthenticatedAPI):
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
 
         xml.etree.ElementTree.register_namespace("oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/")
         xml.etree.ElementTree.register_namespace("ead", "urn:isbn:1-931666-22-9")
@@ -118,6 +118,38 @@ class EntityAPI(AuthenticatedAPI):
                 logger.error(exception)
                 raise exception
 
+    def bitstream_location(self, bitstream: Bitstream):
+        """"
+        Retrieves information about a bitstreams storage locations
+        """
+        if not isinstance(bitstream, Bitstream):
+            logger.error("bitstream argument is not a Bitstream object")
+            raise RuntimeError("bitstream argument is not a Bitstream object")
+
+        storage_locations = []
+
+        url: str = f'{self.protocol}://{self.server}/api/entity/content-objects/{bitstream.co_ref}/generations/{bitstream.gen_index}/bitstreams/{bitstream.bs_index}/storage-locations'
+
+        with self.session.get(url, headers={HEADER_TOKEN: self.token}, stream=True) as request:
+            if request.status_code == requests.codes.unauthorized:
+                self.token = self.__token__()
+                return self.bitstream_location(bitstream)
+            elif request.status_code == requests.codes.ok:
+                xml_response = str(request.content.decode('utf-8'))
+                entity_response = xml.etree.ElementTree.fromstring(xml_response)
+                logger.debug(xml_response)
+                locations = entity_response.find(f'.//{{{self.entity_ns}}}StorageLocation')
+                for adapter in locations:
+                    storage_locations.append(adapter.attrib['name'])
+            else:
+                exception = HTTPException(bitstream.filename, request.status_code, request.url, "bitstream_location",
+                                          request.content.decode('utf-8'))
+                logger.error(exception)
+                raise exception
+
+        return storage_locations
+
+
     def bitstream_content(self, bitstream: Bitstream, filename: str, chunk_size: int = CHUNK_SIZE) -> Union[int, None]:
         """
         Download a file represented as a Bitstream to a local filename
@@ -1474,7 +1506,7 @@ class EntityAPI(AuthenticatedAPI):
             logger.error(exception)
             raise exception
 
-    def generation(self, url: str) -> Generation:
+    def generation(self, url: str, content_ref: str = None) -> Generation:
         """
          Retrieve a list of generation objects
 
@@ -1524,7 +1556,11 @@ class EntityAPI(AuthenticatedAPI):
             bitstreams = entity_response.findall(f'./{{{self.entity_ns}}}Bitstreams/{{{self.entity_ns}}}Bitstream')
             bitstream_list = []
             for bit in bitstreams:
-                bitstream_list.append(self.bitstream(bit.text))
+                bs: Bitstream = self.bitstream(bit.text)
+                bs.gen_index = index
+                if content_ref is not None:
+                    bs.co_ref = content_ref
+                bitstream_list.append(bs)
             generation = Generation(strtobool(ge.attrib['original']), strtobool(ge.attrib['active']),
                                     format_group.text if hasattr(format_group, 'text') else None,
                                     effective_date.text if hasattr(effective_date, 'text') else None,
@@ -1741,7 +1777,7 @@ class EntityAPI(AuthenticatedAPI):
             result = []
             for g in generations:
                 if hasattr(g, 'text'):
-                    generation = self.generation(g.text)
+                    generation = self.generation(g.text, content_object.reference)
                     generation.asset = content_object.asset
                     generation.content_object = content_object
                     generation.representation_type = content_object.representation_type
@@ -2281,7 +2317,7 @@ class EntityAPI(AuthenticatedAPI):
             logger.error(exception)
             raise exception
 
-    def delete_asset(self, asset: Asset, operator_comment: str, supervisor_comment: str):
+    def delete_asset(self, asset: Asset, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
         Delete an asset from the repository
 
@@ -2290,11 +2326,11 @@ class EntityAPI(AuthenticatedAPI):
         :param supervisor_comment:  The supervisor comment on the deletion
         """
         if isinstance(asset, Asset):
-            return self._delete_entity(asset, operator_comment, supervisor_comment)
+            return self._delete_entity(asset, operator_comment, supervisor_comment, credentials_path)
         else:
             raise RuntimeError("delete_asset only deletes assets")
 
-    def delete_folder(self, folder: Folder, operator_comment: str, supervisor_comment: str):
+    def delete_folder(self, folder: Folder, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
          Delete an asset from the repository
 
@@ -2304,11 +2340,11 @@ class EntityAPI(AuthenticatedAPI):
          :param supervisor_comment:  The supervisor comment on the deletion
          """
         if isinstance(folder, Folder):
-            return self._delete_entity(folder, operator_comment, supervisor_comment)
+            return self._delete_entity(folder, operator_comment, supervisor_comment, credentials_path)
         else:
             raise RuntimeError("delete_folder only deletes folders")
 
-    def _delete_entity(self, entity: Entity, operator_comment: str, supervisor_comment: str):
+    def _delete_entity(self, entity: Entity, operator_comment: str, supervisor_comment: str, credentials_path: str = "credentials.properties"):
         """
         Delete an asset from the repository
 
@@ -2319,7 +2355,7 @@ class EntityAPI(AuthenticatedAPI):
 
         # check manager password is available:
         config = configparser.ConfigParser()
-        config.read('credentials.properties', encoding='utf-8')
+        config.read(credentials_path, encoding='utf-8')
         try:
             manager_username = config['credentials']['manager.username']
             manager_password = config['credentials']['manager.password']
@@ -2373,7 +2409,7 @@ class EntityAPI(AuthenticatedAPI):
                                        headers=headers)
         elif request.status_code == requests.codes.unauthorized:
             self.token = self.__token__()
-            return self._delete_entity(entity, operator_comment, supervisor_comment)
+            return self._delete_entity(entity, operator_comment, supervisor_comment, credentials_path)
         if request.status_code == requests.codes.unprocessable:
             logger.error(request.content.decode('utf-8'))
             raise RuntimeError(request.status_code, "no active workflow context for full deletion exists in the system")
@@ -2385,4 +2421,4 @@ class EntityAPI(AuthenticatedAPI):
         exception = HTTPException(entity.reference, request.status_code, request.url,
                                   "_delete_entity", request.content.decode('utf-8'))
         logger.error(exception)
-        raise exception
+        raise exception

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/mdformsAPI.py

@@ -134,12 +134,13 @@ def _json_from_object_(group: Group) -> dict:
 
 
 class MetadataGroupsAPI(AuthenticatedAPI):
+
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
 
         xml.etree.ElementTree.register_namespace("oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/")
         xml.etree.ElementTree.register_namespace("ead", "urn:isbn:1-931666-22-9")

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/parAPI.py

@@ -23,43 +23,7 @@ def __get_contents__(document) -> AnyStr:
         return json.dumps(json.loads(document))
 
 
-class PreservationActionRegistry:
-
-    def __init__(self, server: str = None, username: str = None, password: str = None, protocol: str = 'https'):
-        self.protocol = protocol
-        self.session = requests.Session()
-        self.session.headers.update({'accept': 'application/json;charset=UTF-8'})
-        config = configparser.ConfigParser()
-        config.read('credentials.properties')
-        if not server:
-            server = os.environ.get('PRESERVICA_SERVER')
-            if server is None:
-                try:
-                    server = config['credentials']['server']
-                except KeyError:
-                    pass
-        if not server:
-            msg = "No valid server found in method arguments, environment variables or credentials.properties file"
-            logger.error(msg)
-            raise RuntimeError(msg)
-        else:
-            self.server = server
-        if not username:
-            username = os.environ.get('PRESERVICA_USERNAME')
-            if username is None:
-                try:
-                    username = config['credentials']['username']
-                except KeyError:
-                    pass
-        self.username = username
-        if not password:
-            password = os.environ.get('PRESERVICA_PASSWORD')
-            if password is None:
-                try:
-                    password = config['credentials']['password']
-                except KeyError:
-                    pass
-        self.password = password
+class PreservationActionRegistry(AuthenticatedAPI):
 
     def format_family(self, guid: str) -> str:
         return self.__guid__(guid, "format-families")

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/retentionAPI.py

@@ -58,9 +58,10 @@ class RetentionPolicy:
 class RetentionAPI(AuthenticatedAPI):
 
     def __init__(self, username=None, password=None, tenant=None, server=None, use_shared_secret=False,
-                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None):
+                 two_fa_secret_key: str = None, protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
+
         if self.major_version < 7 and self.minor_version < 2:
             raise RuntimeError("Retention API is only available when connected to a v6.2 System")
 

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/uploadAPI.py

@@ -1659,7 +1659,22 @@ class UploadAPI(AuthenticatedAPI):
 
     def crawl_filesystem(self, filesystem_path, bucket_name, preservica_parent, callback: bool = False,
                          security_tag: str = "open",
-                         delete_after_upload: bool = True, max_MB_ingested: int = -1):
+                         delete_after_upload: bool = True, max_MB_ingested: int = -1, max_workflows = 8):
+        """
+        Crawl a filesystem and upload the contents to Preservica, the filesystem structure is mirrored in Preservica
+
+
+        :param str filesystem_path: The path to the filesystem to crawl
+        :param str bucket_name: The name of the bucket to upload to, use None to send directly to Preservica
+        :param str preservica_parent: The parent folder in Preservica to upload to
+        :param bool callback: Show upload progress bar
+        :param str security_tag: The security tag to apply to the uploaded assets
+        :param bool delete_after_upload: Delete the local copy of the package after the upload has completed
+        :param int max_MB_ingested: The maximum number of MB to ingest
+        :param int max_workflows: The maximum number of workflows to run concurrently
+
+
+        """
 
         def get_parent(client, identifier, parent_reference):
             id = str(os.path.dirname(identifier))
@@ -1684,12 +1699,17 @@ class UploadAPI(AuthenticatedAPI):
                 folder = entities.pop()
             return folder
 
-        from pyPreservica import EntityAPI
+        from pyPreservica import EntityAPI, WorkflowAPI
         entity_client = EntityAPI(username=self.username, password=self.password, server=self.server,
                                   tenant=self.tenant,
                                   two_fa_secret_key=self.two_fa_secret_key, use_shared_secret=self.shared_secret,
                                   protocol=self.protocol)
 
+        workflow_client = WorkflowAPI(username=self.username, password=self.password, server=self.server,
+                                  tenant=self.tenant,
+                                  two_fa_secret_key=self.two_fa_secret_key, use_shared_secret=self.shared_secret,
+                                  protocol=self.protocol)
+
         if preservica_parent:
             parent = entity_client.folder(preservica_parent)
             logger.info(f"Folders will be created inside Preservica collection {parent.title}")
@@ -1732,6 +1752,11 @@ class UploadAPI(AuthenticatedAPI):
                 else:
                     progress_display = None
 
+                workflow_queue_length = len(list(workflow_client.workflow_instances(workflow_state="Active", workflow_type="Ingest")))
+                while workflow_queue_length > max_workflows:
+                    sleep(30)
+                    workflow_queue_length = len(list(workflow_client.workflow_instances(workflow_state="Active", workflow_type="Ingest")))
+
                 if bucket_name is None:
                     self.upload_zip_package(path_to_zip_package=package, callback=progress_display,
                                             delete_after_upload=delete_after_upload)

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica/workflowAPI.py

@@ -81,10 +81,10 @@ class WorkflowAPI(AuthenticatedAPI):
 
     def __init__(self, username: str = None, password: str = None, tenant: str = None, server: str = None,
                  use_shared_secret: bool = False, two_fa_secret_key: str = None,
-                 protocol: str = "https", request_hook: Callable = None):
+                 protocol: str = "https", request_hook: Callable = None, credentials_path: str = 'credentials.properties'):
 
         super().__init__(username, password, tenant, server, use_shared_secret, two_fa_secret_key,
-                         protocol, request_hook)
+                         protocol, request_hook, credentials_path)
         self.base_url = "sdb/rest/workflow"
 
     def get_workflow_contexts_by_type(self, workflow_type: str):

{pypreservica-3.0.0 → pypreservica-3.0.2}/pyPreservica.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: pyPreservica
-Version: 3.0.0
+Version: 3.0.2
 Summary: Python library for the Preservica API
 Home-page: https://pypreservica.readthedocs.io/
 Author: James Carr
@@ -30,6 +30,17 @@ Requires-Dist: s3transfer
 Requires-Dist: azure-storage-blob
 Requires-Dist: tqdm
 Requires-Dist: pyotp
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: summary
 
 
 # pyPreservica

{pypreservica-3.0.0 → pypreservica-3.0.2}/setup.py

@@ -13,7 +13,7 @@ PKG = "pyPreservica"
 
 # 'setup.py publish' shortcut.
 if sys.argv[-1] == 'publish':
-    os.system('python setup.py sdist bdist_wheel')
+    os.system('python -m build')
     os.system('twine upload dist/*')
     sys.exit()
 
@@ -21,7 +21,7 @@ if sys.argv[-1] == 'publish':
 # This call to setup() does all the work
 setup(
     name=PKG,
-    version="3.0.0",
+    version="3.0.2",
     description="Python library for the Preservica API",
     long_description=README,
     long_description_content_type="text/markdown",

{pypreservica-3.0.0 → pypreservica-3.0.2}/tests/test_bitstream.py

@@ -97,3 +97,11 @@ def test_get_bitstream_content(setup_data):
     assert os.path.isfile(bitstream.filename) is True
     assert Path(bitstream.filename).stat().st_size == 1942466
     os.remove(bitstream.filename)
+
+
+def test_get_bitstream_locations(setup_data):
+    client = EntityAPI()
+    asset = client.asset(ASSET_ID)
+    for bs  in client.bitstreams_for_asset(asset):
+        locations = client.bitstream_location(bs)
+        assert "Primary Adapter" in locations

{pypreservica-3.0.0 → pypreservica-3.0.2}/tests/test_users.py

@@ -1,7 +1,35 @@
+import pytest
 from pyPreservica import *
 
 
-def test_get_all_users():
+def setup():
+    client = AdminAPI()
+    users = client.all_users()
+    for u in users:
+        if u  ==  "pypreservica@gmail.com":
+            client.delete_user(u)
+
+
+
+def tear_down():
+    pass
+
+
+@pytest.fixture
+def setup_data():
+    print("\nSetting up resources...")
+
+    setup()
+
+    yield
+
+    print("\nTearing down resources...")
+
+    tear_down()
+
+
+
+def test_get_all_users(setup_data):
     client = AdminAPI()
     users = client.all_users()
     assert type(users) is list
@@ -10,7 +38,7 @@ def test_get_all_users():
     assert client.username in users
 
 
-def test_get_user():
+def test_get_user(setup_data):
     client = AdminAPI()
     user = client.user_details(client.username)
     assert type(user) is dict
@@ -19,7 +47,7 @@ def test_get_user():
     assert 'SDB_MANAGER_USER' in user['Roles']
 
 
-def test_add_user():
+def test_add_user(setup_data):
     client = AdminAPI()
     user = client.add_user("pypreservica@gmail.com", "pypreservica", ['SDB_MANAGER_USER', 'SDB_INGEST_USER'])
     assert user['UserName'] == "pypreservica@gmail.com"
@@ -30,7 +58,7 @@ def test_add_user():
     assert "pypreservica@gmail.com" not in client.all_users()
 
 
-def test_change_display_name():
+def test_change_display_name(setup_data):
     client = AdminAPI()
     user = client.add_user("pypreservica@gmail.com", "pypreservica", ['SDB_MANAGER_USER', 'SDB_INGEST_USER'])
     assert user['UserName'] == "pypreservica@gmail.com"