assemblyline 4.7.5.dev15__tar.gz → 4.7.5.dev16__tar.gz

{assemblyline-4.7.5.dev15/assemblyline.egg-info → assemblyline-4.7.5.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev15
+Version: 4.7.5.dev16
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.5.dev16/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.5.dev16

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16}/assemblyline/filestore/__init__.py

@@ -143,7 +143,7 @@ def create_transport(url, connection_attempts=None):
         valid_bool_keys = ['allow_directory_access', 'use_default_credentials', 'initalize_container', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
-        t = TransportAzure(base=base, host=host, connection_attempts=connection_attempts, **extras)
+        t = TransportAzure(base=base, host=host, connection_attempts=connection_attempts, port=port, **extras)
 
     else:
         raise FileStoreException("Unknown transport: %s" % scheme)
@@ -200,6 +200,9 @@ class FileStore(object):
                     trace = get_stacktrace_info(ex)
                     self.log.info('Transport problem: %s', trace)
 
+    def delete_batch_chunk_size(self) -> int:
+        return min(transport.delete_batch_chunk_size() for transport in self.transports)
+
     @elasticapm.capture_span(span_type='filestore')
     def delete_batch(self, file_list: list[str], location='all'):
         # Don't even call into the transports if there are no files given
@@ -211,12 +214,7 @@ class FileStore(object):
                 if t.read_only:
                     # Don't attempt to delete from read only transports
                     continue
-
-                try:
-                    t.delete_batch(file_list)
-                except Exception as ex:
-                    trace = get_stacktrace_info(ex)
-                    self.log.info('Transport problem: %s', trace)
+                t.delete_batch(file_list)
 
     @elasticapm.capture_span(span_type='filestore')
     def download(self, src_path: str, dest_path: str, location='any'):

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16}/assemblyline/filestore/transport/azure.py

@@ -37,7 +37,7 @@ class TransportAzure(Transport):
 
     def __init__(self, base=None, access_key=None, tenant_id=None, client_id=None, client_secret=None,
                  host=None, connection_attempts=None, allow_directory_access=False, use_default_credentials=False,
-                 initalize_container=True, read_only=False):
+                 initalize_container=True, read_only=False, port=None):
         self.log = logging.getLogger('assemblyline.transport.azure')
         self.read_only = False
         self.connection_attempts: Optional[int] = connection_attempts
@@ -46,6 +46,8 @@ class TransportAzure(Transport):
         # Get URL
         self.host = host
         self.endpoint_url = f"https://{self.host}"
+        if port:
+            self.endpoint_url += ":" + str(port)
 
         # Get container and base_path
         parts = base.strip("/").split("/", 1)
@@ -59,7 +61,7 @@ class TransportAzure(Transport):
         if use_default_credentials:
             if (tenant_id and client_id) and (not client_secret):
                 self.credential = WorkloadIdentityCredential(tenant_id=tenant_id,
-                                                            client_id=client_id)
+                                                             client_id=client_id)
             else:
                 # Service accounts will by default create the enviromental variables, and use them as params
                 self.credential = DefaultAzureCredential()
@@ -146,12 +148,16 @@ class TransportAzure(Transport):
             if not isinstance(error.cause, ResourceNotFoundError):
                 raise
 
+    def delete_batch_chunk_size(self) -> int:
+        return 256
+
     def delete_batch(self, file_list: Iterable[str]):
         """Deletes a batch of files."""
-        container_client = self.service_client.get_container_client(self.blob_container)
         keys = [self.normalize(path) for path in file_list]
         for key_chunk in chunk.chunk(keys, 256):
-            responses: Iterable[HttpResponse] = self.with_retries(container_client.delete_blobs(*key_chunk))
+            responses: Iterable[HttpResponse] = self.with_retries(
+                self.container_client.delete_blobs, *key_chunk, raise_on_any_failure=False
+            )
             for response in responses:
                 if response.status_code == 404:
                     continue

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16}/assemblyline/filestore/transport/base.py

@@ -46,6 +46,9 @@ class Transport(object):
         """
         raise TransportException("Not Implemented")
 
+    def delete_batch_chunk_size(self) -> int:
+        return 1
+
     def delete_batch(self, file_list: Iterable[str]):
         """
         Deletes a batch of files

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16}/assemblyline/filestore/transport/ftp.py

@@ -154,7 +154,12 @@ class TransportFTP(Transport):
     @reconnect_retry_on_fail
     def delete(self, path):
         path = self.normalize(path)
-        self.ftp.delete(path)
+        try:
+            self.ftp.delete(path)
+        except ftplib.error_perm as e:
+            # If the file doesnt exist we get a 550.
+            if not str(e).startswith('550'):
+                raise
 
     @reconnect_retry_on_fail
     def exists(self, path) -> bool:

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16}/assemblyline/filestore/transport/s3.py

@@ -134,7 +134,7 @@ class TransportS3(Transport):
                 return ret_val
 
             except (EndpointConnectionError, ConnectionClosedError):
-                self.log.warning(f"No connection to S3 transport {self.endpoint_url}, retrying...")
+                self.log.warning("No connection to S3 transport %s, retrying...", self.endpoint_url)
                 retries += 1
         raise ConnectionError(f"Couldn't connect to the requested S3 endpoint {self.endpoint_url} inside retry limit")
 
@@ -142,6 +142,12 @@ class TransportS3(Transport):
         key = self.normalize(path)
         self.with_retries(self.client.delete_object, Bucket=self.bucket, Key=key)
 
+    def delete_batch_chunk_size(self) -> int:
+        if self.use_batch_delete:
+            return 1000
+        else:
+            return 1
+
     def delete_batch(self, file_list: Iterable[str]):
         """Deletes a batch of files."""
         if not self.use_batch_delete:

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16}/assemblyline/filestore/transport/sftp.py

@@ -117,8 +117,11 @@ class TransportSFTP(Transport):
 
     @reconnect_retry_on_fail
     def delete(self, path):
-        path = self.normalize(path)
-        self.sftp.remove(path)
+        try:
+            path = self.normalize(path)
+            self.sftp.remove(path)
+        except FileNotFoundError:
+            pass
 
     @reconnect_retry_on_fail
     def exists(self, path):

{assemblyline-4.7.5.dev15 → assemblyline-4.7.5.dev16/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev15
+Version: 4.7.5.dev16
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.5.dev15/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.5.dev15