assemblyline 4.7.5.dev9__tar.gz → 4.7.5.dev13__tar.gz

{assemblyline-4.7.5.dev9/assemblyline.egg-info → assemblyline-4.7.5.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev9
+Version: 4.7.5.dev13
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.5.dev13/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.5.dev13

{assemblyline-4.7.5.dev9 → assemblyline-4.7.5.dev13}/assemblyline/common/bundling.py

@@ -1,8 +1,11 @@
+import glob
 import json
 import logging
 import os
+import re
 import shutil
 import subprocess
+import tarfile
 import tempfile
 import time
 from copy import copy
@@ -10,6 +13,8 @@ from copy import copy
 from cart import is_cart, pack_stream, unpack_stream
 
 from assemblyline.common import forge
+from assemblyline.common.safe_archive import safe_extract_tar
+from assemblyline.odm.base import SHA256_REGEX
 from assemblyline.common.classification import InvalidClassification
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.uid import get_random_id
@@ -27,6 +32,7 @@ MAX_RETRY = 10
 WORK_DIR = "/tmp/bundling"
 BUNDLE_MAGIC = b'\x1f\x8b\x08'
 BUNDLE_TYPE = "archive/bundle/al"
+SHA256_RE = re.compile(SHA256_REGEX)
 
 log = logging.getLogger('assemblyline.bundling')
 
@@ -238,7 +244,11 @@ def create_bundle(sid, working_dir=WORK_DIR, use_alert=False, user_classificatio
                     json.dump(data, fp)
 
                 # Create the bundle
-                subprocess.check_call("tar czf %s *" % tgz_file, shell=True, cwd=current_working_dir)
+                files = glob.glob(os.path.join(current_working_dir, '*'))
+                subprocess.check_call(
+                    ["tar", "czf", tgz_file] + [os.path.basename(f) for f in files],
+                    cwd=current_working_dir
+                )
 
                 with open(target_file, 'wb') as oh:
                     with open(tgz_file, 'rb') as ih:
@@ -294,11 +304,11 @@ def import_bundle(
             else:
                 extracted_path = path
 
-        # Extract  the bundle
+        # Extract the bundle without allowing members to escape the workdir.
         try:
-            subprocess.check_call(["tar", "-zxf", extracted_path, "-C", current_working_dir])
-        except subprocess.CalledProcessError:
-            raise BundlingException("Bundle decompression failed. Not a valid bundle...")
+            safe_extract_tar(extracted_path, current_working_dir)
+        except (tarfile.TarError, OSError, ValueError) as e:
+            raise BundlingException(f"Bundle decompression failed. Not a valid bundle: {e}")
 
         with open(res_file, 'rb') as fh:
             data = json.load(fh)
@@ -369,6 +379,8 @@ def import_bundle(
                     # Make sure files meet minimum classification and save the files
                     with forge.get_filestore() as filestore:
                         for f, f_data in files['infos'].items():
+                            if not SHA256_RE.fullmatch(f):
+                                raise BundlingException(f"Invalid file key in bundle: {f!r}")
                             check_classification(f_data)
                             expiry_ts = f_data.get('expiry_ts', None)
                             if dtl is not None:
@@ -380,8 +392,11 @@ def import_bundle(
                                     expiry_ts = now_as_iso(dtl * 24 * 60 * 60)
                             datastore.save_or_freshen_file(f, f_data, expiry_ts, f_data['classification'],
                                                            cl_engine=Classification)
+                            src = os.path.realpath(os.path.join(current_working_dir, f))
+                            if os.path.commonpath([src, current_working_dir]) != current_working_dir:
+                                raise BundlingException(f"Bundle file path escapes working dir: {f!r}")
                             try:
-                                filestore.upload(os.path.join(current_working_dir, f), f)
+                                filestore.upload(src, f)
                             except IOError:
                                 pass
 

{assemblyline-4.7.5.dev9 → assemblyline-4.7.5.dev13}/assemblyline/common/path.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 import os
-import string
 import sys
 from typing import Optional
 
@@ -19,8 +18,24 @@ def splitpath(path: str, sep: Optional[str] = None) -> list:
 
 
 def strip_path_inclusion(path: str, base: str) -> str:
+    """Verifies that the provided path is safe with the desired base folder destination.
+
+    If the full path is safe, and no path traversal/inclusion are detected, the full path is
+    returned. If the path is not deemed safe, a simple base filename will be return.
+    In both case, an os.path.join between the desired folder and what gets return should be
+    safe to write to.
+
+    Args:
+        path: The path that needs safety validation.
+        base: The desired destination folder.
+
+    Returns:
+        A safe full path, or only the file basename if unsafe.
+    """
     path = path.replace("\\", os.path.sep).replace("/", os.path.sep)
-    return path if os.path.abspath(os.path.join(base, path)).startswith(base) else os.path.basename(path)
+    safe_base = base if base.endswith(os.path.sep) else base + os.path.sep
+    resolved = os.path.abspath(os.path.join(base, path))
+    return path if (resolved == base.rstrip(os.path.sep) or resolved.startswith(safe_base)) else os.path.basename(path)
 
 
 ASCII_NUMBERS = list(range(48, 58))

assemblyline-4.7.5.dev13/assemblyline/common/safe_archive.py

@@ -0,0 +1,100 @@
+"""
+Safe extraction helpers for tar and zip archives.
+
+Centralises the path-traversal hardening. All Assemblyline components
+that extract archives whose contents are not fully trusted should use
+these helpers instead of calling
+`TarFile.extractall` / `ZipFile.extractall` directly.
+"""
+import os
+import tarfile
+import zipfile
+from typing import Iterator, Optional, Union
+
+
+class UnsafeArchiveMember(Exception):
+    """Raised when an archive member would escape the destination directory."""
+
+
+def _is_within(path: str, base: str) -> bool:
+    real = os.path.realpath(path)
+    return real == base or os.path.commonpath([real, base]) == base
+
+
+def safe_tar_members(tar: tarfile.TarFile, dest: str) -> Iterator[tarfile.TarInfo]:
+    """Yield only members that resolve under *dest* and are not links that escape it.
+
+    This is the fallback used when running on a Python without PEP 706
+    (`tarfile.data_filter`, backported to 3.8.17 / 3.9.17 / 3.10.12 / 3.11.4).
+    """
+    base = os.path.realpath(dest)
+    for member in tar.getmembers():
+        member_path = os.path.join(base, member.name)
+        if not _is_within(member_path, base):
+            continue
+        if member.issym() or member.islnk():
+            link_target = os.path.join(base, os.path.dirname(member.name), member.linkname)
+            if not _is_within(link_target, base):
+                continue
+        yield member
+
+
+def safe_extract_tar(tar: Union[str, tarfile.TarFile], dest: str) -> None:
+    """Extract *tar* into *dest* without allowing members to escape *dest*.
+
+    Prefers PEP 706's `filter='data'` (rejects absolute paths, ``..`` escapes,
+    symlinks and hardlinks, device nodes, and strips set[ug]id bits). On older
+    interpreters falls back to :func:`safe_tar_members`, which silently drops
+    offending members instead of raising.
+    """
+    owns = isinstance(tar, str)
+    tf = tarfile.open(tar) if owns else tar
+    try:
+        if hasattr(tarfile, "data_filter"):
+            tf.extractall(dest, filter="data")
+        else:
+            tf.extractall(dest, members=list(safe_tar_members(tf, dest)))
+    finally:
+        if owns:
+            tf.close()
+
+
+def safe_extract_zip(
+    zf: Union[str, zipfile.ZipFile],
+    dest: str,
+    *,
+    pwd: Optional[bytes] = None,
+    on_unsafe: str = "raise",
+) -> None:
+    """Extract *zf* into *dest* without allowing members to escape *dest*.
+
+    ``ZipFile.extractall`` already strips leading ``/`` and ``..`` components
+    from member names, but it does **not** defend against the destination
+    containing pre-existing symlinks (e.g. when re-extracting over a directory
+    a previous archive populated). This helper validates each target with
+    ``realpath`` before any write occurs.
+
+    :param on_unsafe: ``"raise"`` (default) to raise :class:`UnsafeArchiveMember`
+        on the first offending entry, or ``"skip"`` to silently drop it.
+    """
+    if on_unsafe not in ("raise", "skip"):
+        raise ValueError("on_unsafe must be 'raise' or 'skip'")
+
+    owns = isinstance(zf, str)
+    zfile = zipfile.ZipFile(zf, "r") if owns else zf
+    try:
+        base = os.path.realpath(dest)
+        safe = []
+        for info in zfile.infolist():
+            target = os.path.join(base, info.filename)
+            if not _is_within(target, base):
+                if on_unsafe == "raise":
+                    raise UnsafeArchiveMember(
+                        f"Path traversal detected in zip member: {info.filename!r}"
+                    )
+                continue
+            safe.append(info)
+        zfile.extractall(path=dest, members=safe, pwd=pwd)
+    finally:
+        if owns:
+            zfile.close()

{assemblyline-4.7.5.dev9 → assemblyline-4.7.5.dev13}/assemblyline/remote/datatypes/__init__.py

@@ -30,6 +30,16 @@ def now_as_iso():
     return ''.join((s, 'Z'))
 
 
+def validate_reply_queue_name(name: str, prefix=None, suffix=None) -> bool:
+    if prefix:
+        if not name.startswith(prefix + '-'):
+            return False
+    if suffix:
+        if not name.endswith('-' + str(suffix)):
+            return False
+    return True
+
+
 def reply_queue_name(prefix=None, suffix=None):
     if prefix:
         components = [prefix]

{assemblyline-4.7.5.dev9 → assemblyline-4.7.5.dev13/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.5.dev9
+Version: 4.7.5.dev13
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

{assemblyline-4.7.5.dev9 → assemblyline-4.7.5.dev13}/assemblyline.egg-info/SOURCES.txt

@@ -55,6 +55,7 @@ assemblyline/common/null.py
 assemblyline/common/path.py
 assemblyline/common/postprocess.py
 assemblyline/common/random_user.py
+assemblyline/common/safe_archive.py
 assemblyline/common/security.py
 assemblyline/common/signaturing.py
 assemblyline/common/str_utils.py

assemblyline-4.7.5.dev9/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.5.dev9