assemblyline 4.7.4.dev7__tar.gz → 4.7.4.dev9__tar.gz

{assemblyline-4.7.4.dev7/assemblyline.egg-info → assemblyline-4.7.4.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.4.dev7
+Version: 4.7.4.dev9
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.4.dev9/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.4.dev9

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/common/forge.py

@@ -9,10 +9,11 @@ from typing import TYPE_CHECKING, Optional
 
 import elasticapm
 import yaml
+from hauntedhouse import Client
+
 from assemblyline.common.constants import service_queue_name
 from assemblyline.common.dict_utils import recursive_update
 from assemblyline.common.importing import load_module_by_path
-from hauntedhouse import Client
 
 if TYPE_CHECKING:
     from assemblyline.odm.models.config import Config

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/datastore/collection.py

@@ -1224,7 +1224,7 @@ class ESCollection(Generic[ModelType]):
         """
         if self.model_class:
             fields = self.model_class.flat_fields(show_compound=True)
-            if 'classification in fields':
+            if 'classification' in fields:
                 fields.update({"__access_lvl__": Integer(),
                                "__access_req__": List(Keyword()),
                                "__access_grp1__": List(Keyword()),

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/__init__.py

@@ -2,10 +2,12 @@ from __future__ import annotations
 
 import json
 import logging
+import time
 from typing import TYPE_CHECKING, AnyStr, Optional, Tuple
 from urllib.parse import parse_qs, unquote, urlparse
 
 import elasticapm
+
 from assemblyline.common.exceptions import get_stacktrace_info
 from assemblyline.filestore.transport.azure import TransportAzure
 from assemblyline.filestore.transport.base import TransportException
@@ -77,7 +79,7 @@ def create_transport(url, connection_attempts=None):
         sftp: private_key (string), private_key_pass (string), validate_host (bool)
         s3: aws_region (string), s3_bucket (string), use_ssl (bool), verify (bool)
         file: normalize (bool)
-        azure: access_key (string), tenant_id (string), client_id (string), client_secret (string), 
+        azure: access_key (string), tenant_id (string), client_id (string), client_secret (string),
                allow_directory_access (bool), use_default_credentials (bool), initalize_container (bool)
 
     """
@@ -97,7 +99,7 @@ def create_transport(url, connection_attempts=None):
 
     scheme = parsed.scheme.lower()
     if scheme == 'ftp' or scheme == 'ftps':
-        valid_bool_keys = ['use_tls']
+        valid_bool_keys = ['use_tls', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_bool_keys=valid_bool_keys)
         if scheme == 'ftps':
             extras['use_tls'] = True
@@ -106,27 +108,27 @@ def create_transport(url, connection_attempts=None):
 
     elif scheme == "sftp":
         valid_str_keys = ['private_key', 'private_key_pass']
-        valid_bool_keys = ['validate_host']
+        valid_bool_keys = ['validate_host', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
         t = TransportSFTP(base=base, host=host, password=password, user=user, port=port, **extras)
 
     elif scheme == 'http' or scheme == 'https':
         valid_str_keys = ['pki']
-        valid_bool_keys = ['verify']
+        valid_bool_keys = ['verify', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
         t = TransportHTTP(scheme=scheme, base=base, host=host, password=password, user=user, port=port, **extras)
 
     elif scheme == 'file':
-        valid_bool_keys = ['normalize']
+        valid_bool_keys = ['normalize', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_bool_keys=valid_bool_keys)
 
         t = TransportLocal(base=base, **extras)
 
     elif scheme == 's3':
         valid_str_keys = ['aws_region', 's3_bucket']
-        valid_bool_keys = ['use_ssl', 'verify', 'boto_defaults']
+        valid_bool_keys = ['use_ssl', 'verify', 'boto_defaults', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
         # If user/password not specified, access might be dictated by IAM roles
@@ -138,7 +140,7 @@ def create_transport(url, connection_attempts=None):
 
     elif scheme == 'azure':
         valid_str_keys = ['access_key', 'tenant_id', 'client_id', 'client_secret']
-        valid_bool_keys = ['allow_directory_access', 'use_default_credentials', 'initalize_container']
+        valid_bool_keys = ['allow_directory_access', 'use_default_credentials', 'initalize_container', 'read_only']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
         t = TransportAzure(base=base, host=host, connection_attempts=connection_attempts, **extras)
@@ -150,6 +152,8 @@ def create_transport(url, connection_attempts=None):
 
 
 class FileStore(object):
+    SLOW_OP_THRESHOLD = 5.0  # Log warning when a single filestore operation exceeds this (seconds)
+
     def __init__(self, *transport_urls, connection_attempts=None):
         self.log = logging.getLogger('assemblyline.transport')
         self.transport_urls = transport_urls
@@ -168,7 +172,11 @@ class FileStore(object):
         self.close()
 
     def __str__(self):
-        return ', '.join(str(t) for t in self.transports)
+        out = ', '.join(str(t) for t in self.transports)
+        read_only_transports = [str(t) for t in self.transports if t.read_only]
+        if read_only_transports:
+            out += " | read-only: {}".format(', '.join(read_only_transports))
+        return out
 
     def close(self):
         for t in self.transports:
@@ -182,6 +190,10 @@ class FileStore(object):
     def delete(self, path: str, location='all'):
         with elasticapm.capture_span(name='delete', span_type='filestore', labels={'path': path}):
             for t in self.slice(location):
+                if t.read_only:
+                    # Don't attempt to delete from read only transports
+                    continue
+
                 try:
                     t.delete(path)
                 except Exception as ex:
@@ -193,6 +205,7 @@ class FileStore(object):
         successful = False
         transports = []
         download_errors = []
+        start = time.monotonic()
         for t in self.slice(location):
             try:
                 t.download(src_path, dest_path)
@@ -202,6 +215,11 @@ class FileStore(object):
             except Exception as ex:
                 download_errors.append((str(t), str(ex)))
 
+        elapsed = time.monotonic() - start
+        if elapsed > self.SLOW_OP_THRESHOLD:
+            self.log.warning("Slow filestore download: %s took %.2fs (transports tried: %d)",
+                             src_path, elapsed, len(download_errors) + len(transports))
+
         if not successful:
             raise FileStoreException('No transport succeeded => %s' % json.dumps(download_errors))
         return transports
@@ -240,6 +258,10 @@ class FileStore(object):
     def put(self, dst_path: str, content: AnyStr, location='all', force=False) -> list[Transport]:
         transports = []
         for t in self.slice(location):
+            if t.read_only:
+                # Skip saving files to read-only transports
+                continue
+
             if force or not t.exists(dst_path):
                 transports.append(t)
                 t.put(dst_path, content)
@@ -263,13 +285,22 @@ class FileStore(object):
     @elasticapm.capture_span(span_type='filestore')
     def upload(self, src_path: str, dst_path: str, location='all', force=False, verify=False) -> list[Transport]:
         transports = []
+        start = time.monotonic()
         for t in self.slice(location):
+            if t.read_only:
+                # Skip saving files to read-only transports
+                continue
+
             if force or not t.exists(dst_path):
                 transports.append(t)
                 t.upload(src_path, dst_path)
                 if verify and not t.exists(dst_path):
                     raise FileStoreException('File transfer failed. Remote file does not '
                                              'exist for %s on %s (%s)' % (dst_path, location, t))
+        elapsed = time.monotonic() - start
+        if elapsed > self.SLOW_OP_THRESHOLD:
+            self.log.warning("Slow filestore upload: %s took %.2fs across %d transport(s)",
+                             dst_path, elapsed, len(transports))
         return transports
 
     @elasticapm.capture_span(span_type='filestore')

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/azure.py

@@ -4,8 +4,6 @@ import time
 from io import BytesIO
 from typing import Iterable, Optional
 
-from assemblyline.common.exceptions import ChainAll
-from assemblyline.filestore.transport.base import Transport, TransportException
 from azure.core.exceptions import (
     ClientAuthenticationError,
     DecodeError,
@@ -17,9 +15,16 @@ from azure.core.exceptions import (
     ServiceRequestError,
     TooManyRedirectsError,
 )
-from azure.identity import ClientSecretCredential, DefaultAzureCredential, WorkloadIdentityCredential
+from azure.identity import (
+    ClientSecretCredential,
+    DefaultAzureCredential,
+    WorkloadIdentityCredential,
+)
 from azure.storage.blob import BlobServiceClient
 
+from assemblyline.common.exceptions import ChainAll
+from assemblyline.filestore.transport.base import Transport, TransportException
+
 """
 This class assumes a flat file structure in the Azure storage blob.
 """
@@ -30,7 +35,7 @@ class TransportAzure(Transport):
 
     def __init__(self, base=None, access_key=None, tenant_id=None, client_id=None, client_secret=None,
                  host=None, connection_attempts=None, allow_directory_access=False, use_default_credentials=False,
-                 initalize_container=True):
+                 initalize_container=True, read_only=False):
         self.log = logging.getLogger('assemblyline.transport.azure')
         self.read_only = False
         self.connection_attempts: Optional[int] = connection_attempts
@@ -76,13 +81,15 @@ class TransportAzure(Transport):
             except TransportException as e:
                 if not isinstance(e.cause, ResourceNotFoundError):
                     raise
-                try:
-                    self.with_retries(self.container_client.create_container)
-                except TransportException as error:
-                    if not isinstance(error.cause, ResourceNotFoundError):
-                        raise
-                    self.log.info('Failed to create container, we\'re most likely in read only mode')
-                    self.read_only = True
+                if not read_only:
+                    # Attempt to initialize the container if it doesn't exist if the transport is writable
+                    try:
+                        self.with_retries(self.container_client.create_container)
+                    except TransportException as error:
+                        if not isinstance(error.cause, ResourceNotFoundError):
+                            raise
+                        self.log.info('Failed to create container, we\'re most likely in read only mode')
+                        read_only = True
 
         def azure_normalize(path):
             # flatten path to just the basename
@@ -94,7 +101,7 @@ class TransportAzure(Transport):
             else:
                 return path
 
-        super(TransportAzure, self).__init__(normalize=azure_normalize)
+        super(TransportAzure, self).__init__(normalize=azure_normalize, read_only=read_only)
 
     def __str__(self):
         return f"azure://{self.host}/{self.blob_container}/"

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/base.py

@@ -33,8 +33,9 @@ class Transport(object):
       TransportExceptions.
     """
 
-    def __init__(self, normalize=normalize_srl_path):
+    def __init__(self, normalize=normalize_srl_path, read_only=False):
         self.normalize = normalize
+        self.read_only = read_only
 
     def close(self):
         pass

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/ftp.py

@@ -1,23 +1,27 @@
 from __future__ import annotations
+
+import errno
 import ftplib
 import logging
 import os
 import os.path
 import posixpath
-import threading
+import re
 import socket
+import threading
 import time
-import errno
 import weakref
-import re
-
 from io import BytesIO
-from typing import Optional, Union, AnyStr, Iterable
+from typing import AnyStr, Iterable, Optional, Union
 
 from assemblyline.common.exceptions import ChainAll
 from assemblyline.common.path import splitpath
 from assemblyline.common.uid import get_random_id
-from assemblyline.filestore.transport.base import Transport, TransportException, normalize_srl_path
+from assemblyline.filestore.transport.base import (
+    Transport,
+    TransportException,
+    normalize_srl_path,
+)
 
 NORMALIZED = re.compile('[a-z0-9]/[a-z0-9]/[a-z0-9]/[a-z0-9]/[a-z0-9]{64}')
 
@@ -105,7 +109,7 @@ class TransportFTP(Transport):
     FTP Transport class.
     """
 
-    def __init__(self, base=None, host=None, password=None, user=None, port=None, use_tls=None):
+    def __init__(self, base=None, host=None, password=None, user=None, port=None, use_tls=None, read_only=False):
         self.log: logging.Logger = logging.getLogger('assemblyline.transport.ftp')
         self.base: str = base
         self.ftp_objects: weakref.WeakKeyDictionary[threading.Thread, ftplib.FTP] = weakref.WeakKeyDictionary()
@@ -127,7 +131,7 @@ class TransportFTP(Transport):
             self.log.debug('ftp normalized: %s -> %s', path, s)
             return s
 
-        super(TransportFTP, self).__init__(normalize=ftp_normalize)
+        super(TransportFTP, self).__init__(normalize=ftp_normalize, read_only=read_only)
 
     @property
     def ftp(self) -> Union[ftplib.FTP, ftplib.FTP_TLS]:

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/http.py

@@ -1,10 +1,15 @@
 import logging
 import os
 import posixpath
+
 import requests
 
 from assemblyline.common.exceptions import ChainAll
-from assemblyline.filestore.transport.base import Transport, TransportException, normalize_srl_path
+from assemblyline.filestore.transport.base import (
+    Transport,
+    TransportException,
+    normalize_srl_path,
+)
 
 
 @ChainAll(TransportException)
@@ -13,7 +18,7 @@ class TransportHTTP(Transport):
     HTTP Transport class.
     """
 
-    def __init__(self, scheme='http', base=None, host=None, password=None, user=None, pki=None, port=None, verify=None):
+    def __init__(self, scheme='http', base=None, host=None, password=None, user=None, pki=None, port=None, verify=None, read_only=False):
         self.log = logging.getLogger('assemblyline.transport.http')
         self.base = base
         self.host = host
@@ -45,7 +50,7 @@ class TransportHTTP(Transport):
 
         self._session = None
 
-        super(TransportHTTP, self).__init__(normalize=http_normalize)
+        super(TransportHTTP, self).__init__(normalize=http_normalize, read_only=read_only)
 
     @property
     def session(self):

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/local.py

@@ -1,13 +1,17 @@
 import logging
 import os
-import shutil
 import re
+import shutil
 from typing import AnyStr, Iterable, Optional
 
 from assemblyline.common.exceptions import ChainAll
 from assemblyline.common.path import strip_path_inclusion
 from assemblyline.common.uid import get_random_id
-from assemblyline.filestore.transport.base import Transport, TransportException, normalize_srl_path
+from assemblyline.filestore.transport.base import (
+    Transport,
+    TransportException,
+    normalize_srl_path,
+)
 
 NORMALIZED = re.compile('[a-z0-9]/[a-z0-9]/[a-z0-9]/[a-z0-9]/[a-z0-9]{64}')
 
@@ -18,7 +22,7 @@ class TransportLocal(Transport):
     Local file system Transport class.
     """
 
-    def __init__(self, base=None, normalize=None):
+    def __init__(self, base=None, normalize=None, read_only=False):
         self.log = logging.getLogger('assemblyline.transport.local')
         self.base = base
         self.host = "localhost"
@@ -38,7 +42,7 @@ class TransportLocal(Transport):
         if not normalize:
             normalize = local_normalize
 
-        super(TransportLocal, self).__init__(normalize=normalize)
+        super(TransportLocal, self).__init__(normalize=normalize, read_only=read_only)
 
     def delete(self, path):
         normal_path = self.normalize(path)

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/s3.py

@@ -1,13 +1,16 @@
-import boto3
 import logging
 import os
 import tempfile
 import threading
-
+from io import BytesIO
 from typing import Iterable, Optional
 
-from botocore.exceptions import ClientError, EndpointConnectionError, ConnectionClosedError
-from io import BytesIO
+import boto3
+from botocore.exceptions import (
+    ClientError,
+    ConnectionClosedError,
+    EndpointConnectionError,
+)
 
 from assemblyline.common.exceptions import ChainAll
 from assemblyline.filestore.transport.base import Transport, TransportException
@@ -32,7 +35,7 @@ class TransportS3(Transport):
     DEFAULT_HOST = "s3.amazonaws.com"
 
     def __init__(self, base=None, accesskey=None, secretkey=None, aws_region=None, s3_bucket="al-storage",
-                 host=None, port=None, use_ssl=None, verify=True, connection_attempts=None, boto_defaults=False):
+                 host=None, port=None, use_ssl=None, verify=True, connection_attempts=None, boto_defaults=False, read_only=False):
         self.log = logging.getLogger('assemblyline.transport.s3')
         self.base = base
         self.bucket = s3_bucket
@@ -88,7 +91,8 @@ class TransportS3(Transport):
             else:
                 raise
 
-        if not bucket_exist:
+        if not bucket_exist and not read_only:
+            # Only initialize the bucket if the transport has been deemed writable.
             try:
                 self.with_retries(self.client.create_bucket, Bucket=self.bucket)
             except TransportException as e:
@@ -102,7 +106,7 @@ class TransportS3(Transport):
             # flatten path to just the basename
             return os.path.basename(path)
 
-        super(TransportS3, self).__init__(normalize=s3_normalize)
+        super(TransportS3, self).__init__(normalize=s3_normalize, read_only=read_only)
 
     def __str__(self):
         out = "s3://"

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9}/assemblyline/filestore/transport/sftp.py

@@ -4,7 +4,6 @@ import posixpath
 import tempfile
 import warnings
 
-
 # Stop Blowfish deprecation warning
 with warnings.catch_warnings():
     warnings.simplefilter("ignore")
@@ -12,11 +11,16 @@ with warnings.catch_warnings():
     import pysftp
 
 from io import BytesIO
+
 from paramiko import SSHException
 
 from assemblyline.common.exceptions import ChainAll
 from assemblyline.common.uid import get_random_id
-from assemblyline.filestore.transport.base import Transport, TransportException, normalize_srl_path
+from assemblyline.filestore.transport.base import (
+    Transport,
+    TransportException,
+    normalize_srl_path,
+)
 
 
 def reconnect_retry_on_fail(func):
@@ -63,7 +67,7 @@ class TransportSFTP(Transport):
     """
 
     def __init__(self, base=None, host=None, password=None, user=None, port=None, private_key=None,
-                 private_key_pass=None, validate_host=False):
+                 private_key_pass=None, validate_host=False, read_only=False):
         self.log = logging.getLogger('assemblyline.transport.sftp')
         if base == "/":
             self.base = "./"
@@ -102,7 +106,7 @@ class TransportSFTP(Transport):
             self.log.debug('sftp normalized: %s -> %s', path, s)
             return s
 
-        super(TransportSFTP, self).__init__(normalize=sftp_normalize)
+        super(TransportSFTP, self).__init__(normalize=sftp_normalize, read_only=read_only)
 
     def __str__(self):
         return 'sftp://{}@{}{}'.format(self.user, self.host, self.base)

{assemblyline-4.7.4.dev7 → assemblyline-4.7.4.dev9/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.4.dev7
+Version: 4.7.4.dev9
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.4.dev7/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.4.dev7