pyPreservica 3.0.2__py3-none-any.whl → 3.0.5__py3-none-any.whl

pyPreservica/__init__.py

@@ -23,6 +23,6 @@ from .mdformsAPI import MetadataGroupsAPI, Group, GroupField, GroupFieldType
 __author__ = "James Carr (drjamescarr@gmail.com)"
 
 # Version of the pyPreservica package
-__version__ = "3.0.2"
+__version__ = "3.0.5"
 
 __license__ = "Apache License Version 2.0"

pyPreservica/uploadAPI.py

@@ -1659,57 +1659,59 @@ class UploadAPI(AuthenticatedAPI):
 
     def crawl_filesystem(self, filesystem_path, bucket_name, preservica_parent, callback: bool = False,
                          security_tag: str = "open",
-                         delete_after_upload: bool = True, max_MB_ingested: int = -1, max_workflows = 8):
-        """
-        Crawl a filesystem and upload the contents to Preservica, the filesystem structure is mirrored in Preservica
-
-
-        :param str filesystem_path: The path to the filesystem to crawl
-        :param str bucket_name: The name of the bucket to upload to, use None to send directly to Preservica
-        :param str preservica_parent: The parent folder in Preservica to upload to
-        :param bool callback: Show upload progress bar
-        :param str security_tag: The security tag to apply to the uploaded assets
-        :param bool delete_after_upload: Delete the local copy of the package after the upload has completed
-        :param int max_MB_ingested: The maximum number of MB to ingest
-        :param int max_workflows: The maximum number of workflows to run concurrently
+                         delete_after_upload: bool = True, max_MB_ingested: int = -1):
 
+        from pyPreservica import EntityAPI
 
-        """
+        def entity_value(client: EntityAPI, identifier: str) -> Entity:
+            back_off: int = 5
+            while True:
+                try:
+                    entities = client.identifier("code", identifier)
+                    if bool(len(entities) > 0):
+                        return entities.pop()
+                    else:
+                        return None
+                except HTTPException as e:
+                    sleep(back_off)
+                    back_off = back_off * 2
+
+        def entity_exists(client: EntityAPI, identifier: str) -> bool:
+            back_off: int = 5
+            while True:
+                try:
+                    entities = client.identifier("code", identifier)
+                    return bool(len(entities) > 0)
+                except HTTPException as e:
+                    sleep(back_off)
+                    back_off = back_off * 2
 
         def get_parent(client, identifier, parent_reference):
-            id = str(os.path.dirname(identifier))
-            if not id:
-                id = identifier
-            entities = client.identifier("code", id)
-            if len(entities) > 0:
-                folder = entities.pop()
+            dirname_id: str = str(os.path.dirname(identifier))
+            if not dirname_id:
+                dirname_id = identifier
+            folder = entity_value(client, dirname_id)
+            if folder is not None:
                 folder = client.folder(folder.reference)
                 return folder.reference
             else:
                 return parent_reference
 
         def get_folder(client, name, tag, parent_reference, identifier):
-            entities = client.identifier("code", identifier)
-            if len(entities) == 0:
+            folder = entity_value(client, identifier)
+            if folder is None:
                 logger.info(f"Creating new folder with name {name}")
                 folder = client.create_folder(name, name, tag, parent_reference)
                 client.add_identifier(folder, "code", identifier)
             else:
                 logger.info(f"Found existing folder with name {name}")
-                folder = entities.pop()
             return folder
 
-        from pyPreservica import EntityAPI, WorkflowAPI
         entity_client = EntityAPI(username=self.username, password=self.password, server=self.server,
                                   tenant=self.tenant,
                                   two_fa_secret_key=self.two_fa_secret_key, use_shared_secret=self.shared_secret,
                                   protocol=self.protocol)
 
-        workflow_client = WorkflowAPI(username=self.username, password=self.password, server=self.server,
-                                  tenant=self.tenant,
-                                  two_fa_secret_key=self.two_fa_secret_key, use_shared_secret=self.shared_secret,
-                                  protocol=self.protocol)
-
         if preservica_parent:
             parent = entity_client.folder(preservica_parent)
             logger.info(f"Folders will be created inside Preservica collection {parent.title}")
@@ -1734,7 +1736,7 @@ class UploadAPI(AuthenticatedAPI):
                     files.remove(file)
                     continue
                 asset_code = os.path.join(code, file)
-                if len(entity_client.identifier("code", asset_code)) == 0:
+                if not entity_exists(entity_client, asset_code):
                     bytes_ingested = bytes_ingested + os.stat(full_path).st_size
                     logger.info(f"Adding new file: {file} to package ready for upload")
                     file_identifiers = {"code": asset_code}
@@ -1752,18 +1754,13 @@ class UploadAPI(AuthenticatedAPI):
                 else:
                     progress_display = None
 
-                workflow_queue_length = len(list(workflow_client.workflow_instances(workflow_state="Active", workflow_type="Ingest")))
-                while workflow_queue_length > max_workflows:
-                    sleep(30)
-                    workflow_queue_length = len(list(workflow_client.workflow_instances(workflow_state="Active", workflow_type="Ingest")))
-
                 if bucket_name is None:
                     self.upload_zip_package(path_to_zip_package=package, callback=progress_display,
                                             delete_after_upload=delete_after_upload)
                 else:
                     self.upload_zip_to_Source(path_to_zip_package=package, container_name=bucket_name,
-                                                  show_progress= bool(progress_display is not None),
-                                                  delete_after_upload=delete_after_upload)
+                                              show_progress=bool(progress_display is not None),
+                                              delete_after_upload=delete_after_upload)
 
                 logger.info(f"Uploaded " + "{:.1f}".format(bytes_ingested / (1024 * 1024)) + " MB")
 

{pyPreservica-3.0.2.dist-info → pyPreservica-3.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: pyPreservica
-Version: 3.0.2
+Version: 3.0.5
 Summary: Python library for the Preservica API
 Home-page: https://pypreservica.readthedocs.io/
 Author: James Carr
@@ -24,7 +24,7 @@ License-File: LICENSE.txt
 Requires-Dist: requests
 Requires-Dist: urllib3
 Requires-Dist: certifi
-Requires-Dist: boto3
+Requires-Dist: boto3<1.36.0
 Requires-Dist: botocore
 Requires-Dist: s3transfer
 Requires-Dist: azure-storage-blob

{pyPreservica-3.0.2.dist-info → pyPreservica-3.0.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-pyPreservica/__init__.py,sha256=HkIx9xYPB52tEV5TP_tH6MFW57o7yDtmSOU9DXOPW8o,1177
+pyPreservica/__init__.py,sha256=d5_ol6jTBuUxdJrWZUCoaS7jqw4C-o4uX94c5FrF7Ag,1177
 pyPreservica/adminAPI.py,sha256=511bc5KtrCAXbDyBk39dmDnxUVDaOu6xaiyu0jYhxa4,37781
 pyPreservica/authorityAPI.py,sha256=Eule8g6LXr8c8SFcJgpRah4lH1FgevUItO5HhHDEaZE,9172
 pyPreservica/common.py,sha256=nZeUObfypPXFauag5yYXEutvRC72sNXwVCN0wAFnssg,37796
@@ -9,11 +9,11 @@ pyPreservica/monitorAPI.py,sha256=HD-PUPdSI9wGAa07e2_2_-FLINH8PoWUwpFogz7F-j4,62
 pyPreservica/opex.py,sha256=ccra1S4ojUXS3PlbU8WfxajOkJrwG4OykBnNrYP_jus,4875
 pyPreservica/parAPI.py,sha256=f0ZUxLd0U-BW6kBx5K7W2Pv7NjG3MkTNydmxQ3U1ZVE,9296
 pyPreservica/retentionAPI.py,sha256=F6okFSyqtnLhfMbcyChd_5V-D_PAxNLwyx8XohH2DEM,24840
-pyPreservica/uploadAPI.py,sha256=_ldYoDWD6cOIlMXM98KTF2Xnc7GaZ8SKZNHPzbJyfQI,100105
+pyPreservica/uploadAPI.py,sha256=j7BnJ3tJZhXOIrAyvalAoAXWPgpQLfNVfoz89kO7D_Q,99367
 pyPreservica/webHooksAPI.py,sha256=_K3KUOsmwYf8qMa-mD47sAmNUW7Pzb9oKVpS0VoSbC0,6827
 pyPreservica/workflowAPI.py,sha256=OcOiiUdrQerbPllrkj1lWpmuW0jTuyyV0urwPSYcd_U,17561
-pyPreservica-3.0.2.dist-info/LICENSE.txt,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-pyPreservica-3.0.2.dist-info/METADATA,sha256=aPMxioiNMabjITQ0OR-5cSUsk6VgNqSwfsg1QizXVNM,3018
-pyPreservica-3.0.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-pyPreservica-3.0.2.dist-info/top_level.txt,sha256=iIBh6NAznYQHOV8mv_y_kGKSDITek9rANyFDwJsbU-c,13
-pyPreservica-3.0.2.dist-info/RECORD,,
+pyPreservica-3.0.5.dist-info/LICENSE.txt,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+pyPreservica-3.0.5.dist-info/METADATA,sha256=HSictV4AVTAsXV0nXT9PAOjyaAb-EQ4hUSIGPDBWCKc,3025
+pyPreservica-3.0.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+pyPreservica-3.0.5.dist-info/top_level.txt,sha256=iIBh6NAznYQHOV8mv_y_kGKSDITek9rANyFDwJsbU-c,13
+pyPreservica-3.0.5.dist-info/RECORD,,