assemblyline 4.7.5.dev11__tar.gz → 4.7.5.dev15__tar.gz

{assemblyline-4.7.5.dev11/assemblyline.egg-info → assemblyline-4.7.5.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev11
+Version: 4.7.5.dev15
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.5.dev15/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.5.dev15

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/common/bundling.py

@@ -1,8 +1,11 @@
+import glob
 import json
 import logging
 import os
+import re
 import shutil
 import subprocess
+import tarfile
 import tempfile
 import time
 from copy import copy
@@ -10,6 +13,8 @@ from copy import copy
 from cart import is_cart, pack_stream, unpack_stream
 
 from assemblyline.common import forge
+from assemblyline.common.safe_archive import safe_extract_tar
+from assemblyline.odm.base import SHA256_REGEX
 from assemblyline.common.classification import InvalidClassification
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.uid import get_random_id
@@ -27,6 +32,7 @@ MAX_RETRY = 10
 WORK_DIR = "/tmp/bundling"
 BUNDLE_MAGIC = b'\x1f\x8b\x08'
 BUNDLE_TYPE = "archive/bundle/al"
+SHA256_RE = re.compile(SHA256_REGEX)
 
 log = logging.getLogger('assemblyline.bundling')
 
@@ -238,7 +244,11 @@ def create_bundle(sid, working_dir=WORK_DIR, use_alert=False, user_classificatio
                     json.dump(data, fp)
 
                 # Create the bundle
-                subprocess.check_call("tar czf %s *" % tgz_file, shell=True, cwd=current_working_dir)
+                files = glob.glob(os.path.join(current_working_dir, '*'))
+                subprocess.check_call(
+                    ["tar", "czf", tgz_file] + [os.path.basename(f) for f in files],
+                    cwd=current_working_dir
+                )
 
                 with open(target_file, 'wb') as oh:
                     with open(tgz_file, 'rb') as ih:
@@ -294,11 +304,11 @@ def import_bundle(
             else:
                 extracted_path = path
 
-        # Extract  the bundle
+        # Extract the bundle without allowing members to escape the workdir.
         try:
-            subprocess.check_call(["tar", "-zxf", extracted_path, "-C", current_working_dir])
-        except subprocess.CalledProcessError:
-            raise BundlingException("Bundle decompression failed. Not a valid bundle...")
+            safe_extract_tar(extracted_path, current_working_dir)
+        except (tarfile.TarError, OSError, ValueError) as e:
+            raise BundlingException(f"Bundle decompression failed. Not a valid bundle: {e}")
 
         with open(res_file, 'rb') as fh:
             data = json.load(fh)
@@ -369,6 +379,8 @@ def import_bundle(
                     # Make sure files meet minimum classification and save the files
                     with forge.get_filestore() as filestore:
                         for f, f_data in files['infos'].items():
+                            if not SHA256_RE.fullmatch(f):
+                                raise BundlingException(f"Invalid file key in bundle: {f!r}")
                             check_classification(f_data)
                             expiry_ts = f_data.get('expiry_ts', None)
                             if dtl is not None:
@@ -380,8 +392,11 @@ def import_bundle(
                                     expiry_ts = now_as_iso(dtl * 24 * 60 * 60)
                             datastore.save_or_freshen_file(f, f_data, expiry_ts, f_data['classification'],
                                                            cl_engine=Classification)
+                            src = os.path.realpath(os.path.join(current_working_dir, f))
+                            if os.path.commonpath([src, current_working_dir]) != current_working_dir:
+                                raise BundlingException(f"Bundle file path escapes working dir: {f!r}")
                             try:
-                                filestore.upload(os.path.join(current_working_dir, f), f)
+                                filestore.upload(src, f)
                             except IOError:
                                 pass
 

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/common/path.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 import os
-import string
 import sys
 from typing import Optional
 
@@ -19,8 +18,24 @@ def splitpath(path: str, sep: Optional[str] = None) -> list:
 
 
 def strip_path_inclusion(path: str, base: str) -> str:
+    """Verifies that the provided path is safe with the desired base folder destination.
+
+    If the full path is safe, and no path traversal/inclusion are detected, the full path is
+    returned. If the path is not deemed safe, a simple base filename will be return.
+    In both case, an os.path.join between the desired folder and what gets return should be
+    safe to write to.
+
+    Args:
+        path: The path that needs safety validation.
+        base: The desired destination folder.
+
+    Returns:
+        A safe full path, or only the file basename if unsafe.
+    """
     path = path.replace("\\", os.path.sep).replace("/", os.path.sep)
-    return path if os.path.abspath(os.path.join(base, path)).startswith(base) else os.path.basename(path)
+    safe_base = base if base.endswith(os.path.sep) else base + os.path.sep
+    resolved = os.path.abspath(os.path.join(base, path))
+    return path if (resolved == base.rstrip(os.path.sep) or resolved.startswith(safe_base)) else os.path.basename(path)
 
 
 ASCII_NUMBERS = list(range(48, 58))

assemblyline-4.7.5.dev15/assemblyline/common/safe_archive.py

@@ -0,0 +1,100 @@
+"""
+Safe extraction helpers for tar and zip archives.
+
+Centralises the path-traversal hardening. All Assemblyline components
+that extract archives whose contents are not fully trusted should use
+these helpers instead of calling
+`TarFile.extractall` / `ZipFile.extractall` directly.
+"""
+import os
+import tarfile
+import zipfile
+from typing import Iterator, Optional, Union
+
+
+class UnsafeArchiveMember(Exception):
+    """Raised when an archive member would escape the destination directory."""
+
+
+def _is_within(path: str, base: str) -> bool:
+    real = os.path.realpath(path)
+    return real == base or os.path.commonpath([real, base]) == base
+
+
+def safe_tar_members(tar: tarfile.TarFile, dest: str) -> Iterator[tarfile.TarInfo]:
+    """Yield only members that resolve under *dest* and are not links that escape it.
+
+    This is the fallback used when running on a Python without PEP 706
+    (`tarfile.data_filter`, backported to 3.8.17 / 3.9.17 / 3.10.12 / 3.11.4).
+    """
+    base = os.path.realpath(dest)
+    for member in tar.getmembers():
+        member_path = os.path.join(base, member.name)
+        if not _is_within(member_path, base):
+            continue
+        if member.issym() or member.islnk():
+            link_target = os.path.join(base, os.path.dirname(member.name), member.linkname)
+            if not _is_within(link_target, base):
+                continue
+        yield member
+
+
+def safe_extract_tar(tar: Union[str, tarfile.TarFile], dest: str) -> None:
+    """Extract *tar* into *dest* without allowing members to escape *dest*.
+
+    Prefers PEP 706's `filter='data'` (rejects absolute paths, ``..`` escapes,
+    symlinks and hardlinks, device nodes, and strips set[ug]id bits). On older
+    interpreters falls back to :func:`safe_tar_members`, which silently drops
+    offending members instead of raising.
+    """
+    owns = isinstance(tar, str)
+    tf = tarfile.open(tar) if owns else tar
+    try:
+        if hasattr(tarfile, "data_filter"):
+            tf.extractall(dest, filter="data")
+        else:
+            tf.extractall(dest, members=list(safe_tar_members(tf, dest)))
+    finally:
+        if owns:
+            tf.close()
+
+
+def safe_extract_zip(
+    zf: Union[str, zipfile.ZipFile],
+    dest: str,
+    *,
+    pwd: Optional[bytes] = None,
+    on_unsafe: str = "raise",
+) -> None:
+    """Extract *zf* into *dest* without allowing members to escape *dest*.
+
+    ``ZipFile.extractall`` already strips leading ``/`` and ``..`` components
+    from member names, but it does **not** defend against the destination
+    containing pre-existing symlinks (e.g. when re-extracting over a directory
+    a previous archive populated). This helper validates each target with
+    ``realpath`` before any write occurs.
+
+    :param on_unsafe: ``"raise"`` (default) to raise :class:`UnsafeArchiveMember`
+        on the first offending entry, or ``"skip"`` to silently drop it.
+    """
+    if on_unsafe not in ("raise", "skip"):
+        raise ValueError("on_unsafe must be 'raise' or 'skip'")
+
+    owns = isinstance(zf, str)
+    zfile = zipfile.ZipFile(zf, "r") if owns else zf
+    try:
+        base = os.path.realpath(dest)
+        safe = []
+        for info in zfile.infolist():
+            target = os.path.join(base, info.filename)
+            if not _is_within(target, base):
+                if on_unsafe == "raise":
+                    raise UnsafeArchiveMember(
+                        f"Path traversal detected in zip member: {info.filename!r}"
+                    )
+                continue
+            safe.append(info)
+        zfile.extractall(path=dest, members=safe, pwd=pwd)
+    finally:
+        if owns:
+            zfile.close()

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/datastore/collection.py

@@ -962,6 +962,8 @@ class ESCollection(Generic[ModelType]):
                     # If this index has an archive, check is the document was found in it.
                     if self.archive_name:
                         doc['_source']['from_archive'] = self.is_archive_index(index)
+                    elif 'from_archive' in doc['_source']:
+                        doc['_source']['from_archive'] = False
 
                     if version:
                         return self._normalize_output(doc['_source']), f"{doc['_seq_no']}---{doc['_primary_term']}"
@@ -1012,14 +1014,25 @@ class ESCollection(Generic[ModelType]):
 
     @typing.overload
     def get_if_exists(self, key: str, as_obj: typing.Literal[True] = True, index_type: typing.Optional[Index] = None,
-                      version=False) -> typing.Optional[ModelType]:
+                      version: typing.Literal[False] = False) -> typing.Optional[ModelType]:
         ...
 
     @typing.overload
     def get_if_exists(self, key: str, as_obj: typing.Literal[False], index_type: typing.Optional[Index] = None,
-                      version=False) -> typing.Optional[dict]:
+                      version: typing.Literal[False] = False) -> typing.Optional[dict]:
         ...
 
+    @typing.overload
+    def get_if_exists(self, key: str, as_obj: typing.Literal[True] = True, index_type: typing.Optional[Index] = None,
+                      version=True) -> tuple[typing.Optional[ModelType], str]:
+        ...
+
+    @typing.overload
+    def get_if_exists(self, key: str, as_obj: typing.Literal[False], index_type: typing.Optional[Index] = None,
+                      version=True) -> tuple[typing.Optional[dict], str]:
+        ...
+
+
     def get_if_exists(self, key, as_obj=True, index_type=None, version=False):
         """
         Get a document from the datastore but do not retry if not found.
@@ -1378,6 +1391,8 @@ class ESCollection(Generic[ModelType]):
         # If this index has an archive, check is the document was found in it.
         if self.archive_name:
             source_data['from_archive'] = self.is_archive_index(result['_index'])
+        elif 'from_archive' in source_data:
+            source_data['from_archive'] = False
 
         if self.model_class:
             if not fields:

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/filestore/__init__.py

@@ -128,7 +128,7 @@ def create_transport(url, connection_attempts=None):
 
     elif scheme == 's3':
         valid_str_keys = ['aws_region', 's3_bucket']
-        valid_bool_keys = ['use_ssl', 'verify', 'boto_defaults', 'read_only']
+        valid_bool_keys = ['use_ssl', 'verify', 'boto_defaults', 'read_only', 'use_batch_delete']
         extras = _get_extras(parse_qs(parsed.query), valid_str_keys=valid_str_keys, valid_bool_keys=valid_bool_keys)
 
         # If user/password not specified, access might be dictated by IAM roles
@@ -200,6 +200,24 @@ class FileStore(object):
                     trace = get_stacktrace_info(ex)
                     self.log.info('Transport problem: %s', trace)
 
+    @elasticapm.capture_span(span_type='filestore')
+    def delete_batch(self, file_list: list[str], location='all'):
+        # Don't even call into the transports if there are no files given
+        if not file_list:
+            return
+
+        with elasticapm.capture_span(name='delete_batch', span_type='filestore', labels={'batch_size': len(file_list)}):
+            for t in self.slice(location):
+                if t.read_only:
+                    # Don't attempt to delete from read only transports
+                    continue
+
+                try:
+                    t.delete_batch(file_list)
+                except Exception as ex:
+                    trace = get_stacktrace_info(ex)
+                    self.log.info('Transport problem: %s', trace)
+
     @elasticapm.capture_span(span_type='filestore')
     def download(self, src_path: str, dest_path: str, location='any'):
         successful = False

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/filestore/transport/azure.py

@@ -15,6 +15,7 @@ from azure.core.exceptions import (
     ServiceRequestError,
     TooManyRedirectsError,
 )
+from azure.core.pipeline.transport import HttpResponse
 from azure.identity import (
     ClientSecretCredential,
     DefaultAzureCredential,
@@ -22,6 +23,7 @@ from azure.identity import (
 )
 from azure.storage.blob import BlobServiceClient
 
+from assemblyline.common import chunk
 from assemblyline.common.exceptions import ChainAll
 from assemblyline.filestore.transport.base import Transport, TransportException
 
@@ -144,6 +146,17 @@ class TransportAzure(Transport):
             if not isinstance(error.cause, ResourceNotFoundError):
                 raise
 
+    def delete_batch(self, file_list: Iterable[str]):
+        """Deletes a batch of files."""
+        container_client = self.service_client.get_container_client(self.blob_container)
+        keys = [self.normalize(path) for path in file_list]
+        for key_chunk in chunk.chunk(keys, 256):
+            responses: Iterable[HttpResponse] = self.with_retries(container_client.delete_blobs(*key_chunk))
+            for response in responses:
+                if response.status_code == 404:
+                    continue
+                response.raise_for_status()
+
     def exists(self, path):
         key = self.normalize(path)
         blob_client = self.service_client.get_blob_client(self.blob_container, key)

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/filestore/transport/base.py

@@ -46,6 +46,13 @@ class Transport(object):
         """
         raise TransportException("Not Implemented")
 
+    def delete_batch(self, file_list: Iterable[str]):
+        """
+        Deletes a batch of files
+        """
+        for path in file_list:
+            self.delete(path)
+
     def exists(self, path: str) -> bool:
         """
         Returns True if the path exists, False otherwise.

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline/filestore/transport/s3.py

@@ -12,6 +12,7 @@ from botocore.exceptions import (
     EndpointConnectionError,
 )
 
+from assemblyline.common import chunk
 from assemblyline.common.exceptions import ChainAll
 from assemblyline.filestore.transport.base import Transport, TransportException
 
@@ -35,12 +36,14 @@ class TransportS3(Transport):
     DEFAULT_HOST = "s3.amazonaws.com"
 
     def __init__(self, base=None, accesskey=None, secretkey=None, aws_region=None, s3_bucket="al-storage",
-                 host=None, port=None, use_ssl=None, verify=True, connection_attempts=None, boto_defaults=False, read_only=False):
+                 host=None, port=None, use_ssl=None, verify=True, connection_attempts=None,
+                 boto_defaults=False, read_only=False, use_batch_delete: bool = False):
         self.log = logging.getLogger('assemblyline.transport.s3')
         self.base = base
         self.bucket = s3_bucket
         self.accesskey = accesskey
         self.retry_limit: int = connection_attempts
+        self.use_batch_delete: bool = use_batch_delete
 
         if use_ssl is None:
             self.use_ssl = True
@@ -139,6 +142,25 @@ class TransportS3(Transport):
         key = self.normalize(path)
         self.with_retries(self.client.delete_object, Bucket=self.bucket, Key=key)
 
+    def delete_batch(self, file_list: Iterable[str]):
+        """Deletes a batch of files."""
+        if not self.use_batch_delete:
+            return super().delete_batch(file_list)
+
+        keys = [self.normalize(path) for path in file_list]
+        for key_chunk in chunk.chunk(keys, 1000):
+            response = self.with_retries(
+                self.client.delete_objects,
+                Bucket=self.bucket,
+                Delete={
+                    'Objects': [{'Key': key} for key in key_chunk],
+                    'Quiet': True,
+                },
+            )
+
+            if response['Errors']:
+                raise TransportException('Could not delete some member of batch', response['Errors'][0]['Message'])
+
     def exists(self, path):
         # checks to see if KEY exists
         key = self.normalize(path)

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev11
+Version: 4.7.5.dev15
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

{assemblyline-4.7.5.dev11 → assemblyline-4.7.5.dev15}/assemblyline.egg-info/SOURCES.txt

@@ -55,6 +55,7 @@ assemblyline/common/null.py
 assemblyline/common/path.py
 assemblyline/common/postprocess.py
 assemblyline/common/random_user.py
+assemblyline/common/safe_archive.py
 assemblyline/common/security.py
 assemblyline/common/signaturing.py
 assemblyline/common/str_utils.py

assemblyline-4.7.5.dev11/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.5.dev11