assemblyline 4.7.2.dev28__tar.gz → 4.7.2.dev30__tar.gz

{assemblyline-4.7.2.dev28/assemblyline.egg-info → assemblyline-4.7.2.dev30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.2.dev28
+Version: 4.7.2.dev30
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team
@@ -24,25 +24,20 @@ Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
 Requires-Dist: pysftp
-Requires-Dist: netifaces
-Requires-Dist: pyroute2.core
 Requires-Dist: redis
 Requires-Dist: requests[socks]
 Requires-Dist: elasticsearch<9.0.0,>=8.0.0
 Requires-Dist: python-datemath!=3.0.2
 Requires-Dist: packaging
-Requires-Dist: tabulate
 Requires-Dist: PyYAML
-Requires-Dist: easydict
 Requires-Dist: bcrypt
 Requires-Dist: cart
-Requires-Dist: cccs-ssdeep
+Requires-Dist: assemblyline-toolbox==0.3.0
 Requires-Dist: python-magic
 Requires-Dist: pytz
 Requires-Dist: apscheduler
 Requires-Dist: websocket_client<1.0.0
 Requires-Dist: elastic-apm[flask]>=6.13.0
-Requires-Dist: cython
 Requires-Dist: docker
 Requires-Dist: kubernetes>18
 Requires-Dist: notifications-python-client
@@ -52,7 +47,6 @@ Requires-Dist: azure-identity
 Requires-Dist: msoffcrypto-tool
 Requires-Dist: chardet<6
 Requires-Dist: yara-python
-Requires-Dist: python-tlsh
 Requires-Dist: hauntedhouse==0.1.10
 Requires-Dist: magika
 Requires-Dist: paramiko<4

assemblyline-4.7.2.dev30/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.2.dev30

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30}/assemblyline/common/digests.py

@@ -1,10 +1,8 @@
 import hashlib
-import ssdeep
-import tlsh
 from typing import Dict
 
 from assemblyline.common import entropy
-
+from assemblyline_toolbox import SsdeepHasher, TlshHasher
 DEFAULT_BLOCKSIZE = 65536
 
 
@@ -14,10 +12,7 @@ def get_digests_for_file(path: str, blocksize: int = DEFAULT_BLOCKSIZE, calculat
     """ Generate digests for file reading only 'blocksize bytes at a time."""
     bc = None
     if calculate_entropy:
-        try:
-            bc = entropy.BufferedCalculator()
-        except Exception:
-            pass
+        bc = entropy.BufferedCalculator()
 
     result = {}
 
@@ -25,7 +20,9 @@ def get_digests_for_file(path: str, blocksize: int = DEFAULT_BLOCKSIZE, calculat
     sha1 = hashlib.sha1()
     sha256 = hashlib.sha256()
     if not skip_fuzzy_hashes:
-        th = tlsh.Tlsh()
+        th = TlshHasher()
+        ssdeep = SsdeepHasher()
+
     size = 0
 
     with open(path, 'rb') as f:
@@ -37,12 +34,13 @@ def get_digests_for_file(path: str, blocksize: int = DEFAULT_BLOCKSIZE, calculat
 
         while length > 0:
             if bc is not None:
-                bc.update(data, length)
+                bc.update(data)
             md5.update(data)
             sha1.update(data)
             sha256.update(data)
             if not skip_fuzzy_hashes:
                 th.update(data)
+                ssdeep.update(data)
             size += length
 
             data = f.read(blocksize)
@@ -58,13 +56,10 @@ def get_digests_for_file(path: str, blocksize: int = DEFAULT_BLOCKSIZE, calculat
     result['size'] = size
 
     if not skip_fuzzy_hashes:
-        result["ssdeep"] = ssdeep.hash_from_file(path)
-        # Try to finalise the TLSH Hash and add it to the results
-        try:
-            th.final()
-            result['tlsh'] = th.hexdigest()
-        except Exception:
-            pass
+        result["ssdeep"] = ssdeep.digest()
+        thash = th.digest()
+        if thash:
+            result['tlsh'] = thash
 
     return result
 

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30}/assemblyline/common/entropy.py

@@ -1,9 +1,8 @@
 import io
+from typing import Tuple, List, BinaryIO
 
-from math import log
-from typing import Tuple, List, BinaryIO, AnyStr
+from assemblyline_toolbox import BufferedCalculator
 
-frequency = None
 
 # The minimum partition size should be 256 bytes as the keyspace
 # for a char is 256 bytes
@@ -52,40 +51,3 @@ def calculate_partition_entropy(fin: BinaryIO, num_partitions: int = 50) -> Tupl
         full_entropy_calculator.update(partition)
     return full_entropy_calculator.entropy(), p_entropies
 
-
-class BufferedCalculator(object):
-    def __init__(self):
-        global frequency
-        import pyximport
-        pyximport.install()
-        # noinspection PyUnresolvedReferences
-        from assemblyline.common import frequency
-
-        self.c = {}
-        self.length = 0
-
-    def entropy(self) -> float:
-        if self.length == 0:
-            return 0.0
-
-        length = float(self.length)
-
-        entropy = 0.0
-        for v in self.c.values():
-            prob = float(v) / length
-            entropy += prob * log(prob, 2)
-
-        entropy *= -1
-
-        # Make sure we don't return -0.0.
-        if not entropy:
-            entropy = 0.0
-
-        return entropy
-
-    def update(self, data: AnyStr, length: int = 0):
-        if not length:
-            length = len(data)
-
-        self.length += length
-        self.c = frequency.counts(data, length, self.c)

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30}/assemblyline/common/logformat.py

@@ -10,7 +10,7 @@ except Exception:  # pylint:disable=W0702
 ip = 'x.x.x.x'
 # noinspection PyBroadException
 try:
-    from assemblyline.common.net import get_hostip
+    from assemblyline_toolbox import get_hostip
     ip = get_hostip()
 except Exception:  # pylint:disable=W0702
     pass

assemblyline-4.7.2.dev30/assemblyline/common/net.py

@@ -0,0 +1,93 @@
+from ipaddress import ip_address, IPv4Network
+import socket
+import os
+import functools
+
+
+from assemblyline.common.net_static import TLDS_ALPHA_BY_DOMAIN, TLDS_SPECIAL_BY_DOMAIN
+SYSTEM_LOCAL_TLD = os.getenv('SYSTEM_LOCAL_TLD', '')
+
+
+def is_valid_port(value: int) -> bool:
+    try:
+        if 1 <= int(value) <= 65535:
+            return True
+    except ValueError:
+        pass
+
+    return False
+
+
+@functools.cache
+def find_top_level_domains():
+    """Combine (once and memoize) the three different sources of TLD."""
+    combined_tlds = TLDS_ALPHA_BY_DOMAIN.union({d for d in TLDS_SPECIAL_BY_DOMAIN if '.' not in d})
+    local_tld = [tld.strip().strip('.').upper() for tld in SYSTEM_LOCAL_TLD.split(";")]
+    combined_tlds |= {tld for tld in local_tld if tld}
+    return combined_tlds
+
+
+def is_valid_domain(domain: str) -> bool:
+    if "@" in domain:
+        return False
+
+    if "." in domain:
+        domain = domain.upper()
+        tld = domain.split(".")[-1]
+        if not tld.isascii():
+            try:
+                tld = tld.encode('idna').decode('ascii').upper()
+            except ValueError:
+                return False
+
+        combined_tlds = find_top_level_domains()
+        if tld in combined_tlds:
+            # Single term TLD check
+            return True
+
+        elif any(domain.endswith(d) for d in TLDS_SPECIAL_BY_DOMAIN):
+            # Multi-term TLD check
+            return True
+
+    return False
+
+
+def is_valid_ip(ip: str) -> bool:
+    parts = ip.split(".")
+    if len(parts) == 4:
+        for p in parts:
+            try:
+                if not (0 <= int(p) <= 255):
+                    return False
+            except ValueError:
+                return False
+
+        if int(parts[0]) == 0:
+            return False
+
+        if int(parts[3]) == 0:
+            return False
+
+        return True
+
+    return False
+
+
+def is_ip_in_network(ip: str, network: IPv4Network) -> bool:
+    if not is_valid_ip(ip):
+        return False
+
+    return ip_address(ip) in network
+
+
+def is_valid_email(email: str) -> bool:
+    parts = email.split("@")
+    if len(parts) == 2:
+        if is_valid_domain(parts[1]):
+            return True
+
+    return False
+
+
+def get_hostname() -> str:
+    return socket.gethostname()

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30}/assemblyline/common/postprocess.py

@@ -582,291 +582,3 @@ class SubmissionFilter:
         return '<SubmissionFilter ' + str(self.expression) + '>'
 
 
-def should_resubmit(score: float, shift: float = 500) -> bool:
-
-    # Resubmit:
-    #
-    # 100%     with a score above 400.
-    # 10%      with a score of 301 to 400.
-    # 1%       with a score of 201 to 300.
-    # 0.1%     with a score of 101 to 200.
-    # 0.01%    with a score of 1 to 100.
-    # 0.001%   with a score of 0.
-    # 0%       with a score below 0.
-
-    if score < 0:
-        return False
-
-    if score >= shift:
-        return True
-
-    resubmit_probability = 1.0 / 10 ** ((shift - score) / 100)
-
-    return random.random() < resubmit_probability
-
-
-class ActionWorker:
-    def __init__(self, cache: bool, config, datastore, redis_persist) -> None:
-        # Store parameters
-        self.running_cache_tasks = cache
-
-        # Setup dependencies
-        self.config = config
-        self.datastore = datastore
-
-        # Submissions that should have alerts generated
-        self.alert_queue: NamedQueue[dict] = NamedQueue(ALERT_QUEUE_NAME, redis_persist)
-        self.unique_queue: PriorityQueue[dict] = PriorityQueue('m-unique', redis_persist)
-        self.config_hash: Hash[str] = Hash(CONFIG_HASH, redis_persist)
-
-        # Archive manager (late import due to circular import)
-        from assemblyline.common.archiving import ArchiveManager
-        self.archive_manager: ArchiveManager = ArchiveManager(self.config, self.datastore)
-
-        # Load actions
-        self.actions: dict[str, tuple[SubmissionFilter, PostprocessAction]] = {}
-        self._load_actions()
-
-        # Make sure we load any changed actions
-        self.reload_watcher: EventWatcher[str] = EventWatcher()
-        self.reload_watcher.register('system.postprocess', self._load_actions)
-        self.reload_watcher.start()
-
-        # Create an event loop to handle highly parallel webhook calls
-        self.loop = asyncio.new_event_loop()
-        threading.Thread(target=self.loop.run_forever, name='webhook_caller', daemon=True).start()
-
-    def stop(self):
-        self.reload_watcher.stop()
-        while self.loop.is_running():
-            if len(asyncio.all_tasks(self.loop)) == 0:
-                break
-            time.sleep(0.1)
-        self.loop.call_soon_threadsafe(self.loop.stop)
-
-    def _load_actions(self, _path: Optional[str] = None):
-        # Load the action data from redis
-        data = self.config_hash.get(POST_PROCESS_CONFIG_KEY)
-
-        # If nothing is in redis, fall back to legacy storage
-        if data is None:
-            try:
-                with CacheStore('system', config=self.config, datastore=self.datastore) as cache:
-                    byte_data = cache.get('postprocess_actions')
-                    if byte_data:
-                        data = byte_data.decode()
-            except Exception:
-                logger.warn("Couldn't access system files")
-
-        # Decode data
-        objects = DEFAULT_POSTPROCESS_ACTIONS
-        if data:
-            try:
-                raw: dict[str, Any] = yaml.safe_load(data)
-                objects = {
-                    key: PostprocessAction(data)
-                    for key, data in raw.items()
-                }
-            except Exception:
-                logger.exception("Couldn't load stored actions")
-
-        # Check which ones can be active
-        ready_objects: dict[str, tuple[SubmissionFilter, PostprocessAction]] = {}
-        for key, action in objects.items():
-            if not action.enabled:
-                continue
-
-            try:
-                fltr = SubmissionFilter(action.filter)
-            except Exception:
-                logger.exception("Failed to load submission filter")
-                continue
-
-            if self.running_cache_tasks and action.run_on_cache:
-                if not fltr.cache_safe:
-                    logger.error("Tried to apply non-cache-safe filter to cached submissions.")
-                    continue
-                ready_objects[key] = fltr, action
-
-            if not self.running_cache_tasks and action.run_on_completed:
-                ready_objects[key] = fltr, action
-
-        # Swap in the new actions
-        self.actions = ready_objects
-
-    def process_submission(self, submission: Submission, tags: list[dict[str, Any]]) -> bool:
-        return self.process(submission=submission, tags=tags, score=submission.max_score)
-
-    def process_cachehit(self, submission: SubmissionMessage, score: float) -> bool:
-        return self.process(submission=submission, tags=None, score=score)
-
-    def process(self, submission: Union[Submission, SubmissionMessage],
-                score: float, tags: Optional[list[dict[str, Any]]]) -> bool:
-        """ Handle any postprocessing events for a submission.
-
-        Return bool indicating if a resubmission action has happened.
-        """
-        archive_submission = submission.params.auto_archive
-        use_archive_alternate_dtl = submission.params.use_archive_alternate_dtl
-        create_alert = False
-        resubmit: Optional[set[str]] = None
-        webhooks = []
-
-        for fltr, action in self.actions.values():
-            if not fltr.test(submission, score=score, tags=tags):
-                continue
-
-            # Check if we need to launch an alert
-            create_alert |= action.raise_alert
-
-            # Check if we need to archive the submission
-            archive_submission |= action.archive_submission
-            use_archive_alternate_dtl |= action.use_archive_alternate_dtl
-
-            # Accumulate resubmit services
-            if action.resubmit is not None:
-                do_resubmit = True
-                if action.resubmit.random_below is not None:
-                    do_resubmit = should_resubmit(score, action.resubmit.random_below)
-
-                if do_resubmit:
-                    if resubmit is None:
-                        resubmit = set()
-                    resubmit.update(set(action.resubmit.additional_services))
-
-            # Accumulate hooks
-            if action.webhook is not None and action.webhook not in webhooks:
-                webhooks.append(action.webhook)
-
-        # Bail early if nothing is to be done
-        if resubmit is None and not create_alert and not webhooks and not archive_submission:
-            return False
-
-        # Prepare a message formatted submission
-        if isinstance(submission, Submission):
-            submission_msg = from_datastore_submission(submission)
-        else:
-            submission_msg = submission
-
-        # Default values
-        extended_scan = 'skipped'
-        did_resubmit = False
-        submit_to = []
-
-        # Check if we resubmit
-        if resubmit is not None:
-            selected = set(submission.params.services.selected)
-            resubmit_to = set(submission.params.services.resubmit) | resubmit
-
-            if not selected.issuperset(resubmit_to):
-                submit_to = sorted(selected | resubmit_to)
-                extended_scan = 'submitted'
-
-        # Raise alert
-        if submission.params.generate_alert and create_alert:
-            logger.info(f"[{submission_msg.sid} :: {submission_msg.files[0].sha256}] Notifying alerter to "
-                        "create or update an alert")
-
-            self.alert_queue.push(dict(
-                submission=submission_msg.as_primitives(),
-                score=score,
-                extended_scan=extended_scan,
-                ingest_id=submission_msg.metadata.get('ingest_id', None)
-            ))
-
-        if submit_to:
-            logger.info(f"[{submission.sid} :: {submission.files[0].sha256}] Resubmitted for extended analysis")
-            resubmission = SubmissionMessage(submission_msg.as_primitives())
-            resubmission.params.psid = submission.sid
-            resubmission.sid = get_random_id()
-            resubmission.scan_key = None
-            resubmission.params.services.resubmit = []
-            resubmission.params.services.selected = submit_to
-
-            self.unique_queue.push(submission.params.priority, dict(
-                score=score,
-                extended_scan=extended_scan,
-                ingest_id=submission.metadata.get('ingest_id', None),
-                submission=resubmission.as_primitives(),
-            ))
-            did_resubmit = True
-
-        # Archive the submission
-        if archive_submission:
-            if self.config.datastore.archive.enabled:
-                logger.info(f"[{submission_msg.sid} :: {submission_msg.files[0].sha256}] Evaluating if the file can"
-                            " be moved to the malware archive")
-
-                if self.archive_manager.archive_submission(
-                        submission_msg.as_primitives(),
-                        submission_msg.params.delete_after_archive,
-                        use_alternate_dtl=use_archive_alternate_dtl)['action'] == "archive":
-                    logger.info(f"[{submission_msg.sid} :: {submission_msg.files[0].sha256}] Archiver was notified "
-                                "to copy the file in the malware archive")
-                else:
-                    logger.info(f"[{submission_msg.sid} :: {submission_msg.files[0].sha256}] The file was "
-                                "re-submitted for analysis because it does not meet the minimum service requirement")
-
-            else:
-                logger.warning(f"[{submission_msg.sid} :: {submission_msg.files[0].sha256}] Trying to archive a "
-                               "submission on a system where archiving is disabled")
-
-        # Trigger webhooks
-        for hook in webhooks:
-            asyncio.run_coroutine_threadsafe(self._process_hook(hook, submission, score), self.loop)
-
-        return did_resubmit
-
-    async def _process_hook(self, hook: Webhook, submission: Union[Submission, SubmissionMessage], score: float):
-        backoff = 0.0
-        cafile = None
-
-        try:
-            is_cache = isinstance(submission, SubmissionMessage)
-            payload = json.dumps({
-                'is_cache': is_cache,
-                'score': score,
-                'submission': submission.as_primitives()
-            })
-
-            # Setup auth headers and other headers
-            auth = None
-            if hook.username and hook.password:
-                auth = aiohttp.BasicAuth(login=hook.username, password=hook.password)
-            headers = {head.name: head.value for head in hook.headers}
-            headers.setdefault('Content-Type', 'application/json')
-
-            # Setup ssl details
-            sslcontext: Union[None, bool, ssl.SSLContext] = None
-            if hook.ssl_ignore_errors:
-                sslcontext = False
-            if hook.ca_cert:
-                cafile = tempfile.NamedTemporaryFile()
-                cafile.write(hook.ca_cert.encode())
-                cafile.flush()
-                sslcontext = ssl.create_default_context(cafile=cafile.name)
-
-            # Setup setup http query details
-            async with aiohttp.ClientSession(auth=auth, headers=headers) as session:
-                # Loop up to retry limit
-                for _ in range(hook.retries):
-                    # Wait before retrying, 0 first time, so we can have this before the post
-                    # and not wait after the final failure
-                    await asyncio.sleep(backoff)
-                    backoff = min(RETRY_MAX_BACKOFF, backoff * 2) + 0.1
-
-                    # Try posting to the webhook once. If it succeeds return and let
-                    # the withs and finallys finish all the cleanup
-                    try:
-                        resp = await session.request(hook.method, hook.uri, data=payload,
-                                                     ssl=sslcontext, proxy=hook.proxy)
-                        resp.raise_for_status()
-                        return
-                    except Exception:
-                        logger.exception(f"Error pushing to webhook: {hook}")
-
-        except Exception:
-            logger.exception(f"Error reading webhook configuration: {hook}")
-        finally:
-            if cafile is not None:
-                cafile.close()

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.2.dev28
+Version: 4.7.2.dev30
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team
@@ -24,25 +24,20 @@ Requires-Dist: urllib3>=2.6.0
 Requires-Dist: python-baseconv
 Requires-Dist: boto3
 Requires-Dist: pysftp
-Requires-Dist: netifaces
-Requires-Dist: pyroute2.core
 Requires-Dist: redis
 Requires-Dist: requests[socks]
 Requires-Dist: elasticsearch<9.0.0,>=8.0.0
 Requires-Dist: python-datemath!=3.0.2
 Requires-Dist: packaging
-Requires-Dist: tabulate
 Requires-Dist: PyYAML
-Requires-Dist: easydict
 Requires-Dist: bcrypt
 Requires-Dist: cart
-Requires-Dist: cccs-ssdeep
+Requires-Dist: assemblyline-toolbox==0.3.0
 Requires-Dist: python-magic
 Requires-Dist: pytz
 Requires-Dist: apscheduler
 Requires-Dist: websocket_client<1.0.0
 Requires-Dist: elastic-apm[flask]>=6.13.0
-Requires-Dist: cython
 Requires-Dist: docker
 Requires-Dist: kubernetes>18
 Requires-Dist: notifications-python-client
@@ -52,7 +47,6 @@ Requires-Dist: azure-identity
 Requires-Dist: msoffcrypto-tool
 Requires-Dist: chardet<6
 Requires-Dist: yara-python
-Requires-Dist: python-tlsh
 Requires-Dist: hauntedhouse==0.1.10
 Requires-Dist: magika
 Requires-Dist: paramiko<4

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30}/assemblyline.egg-info/SOURCES.txt

@@ -37,7 +37,6 @@ assemblyline/common/entropy.py
 assemblyline/common/exceptions.py
 assemblyline/common/file.py
 assemblyline/common/forge.py
-assemblyline/common/frequency.pyx
 assemblyline/common/heuristics.py
 assemblyline/common/hexdump.py
 assemblyline/common/identify.py

{assemblyline-4.7.2.dev28 → assemblyline-4.7.2.dev30}/assemblyline.egg-info/requires.txt

@@ -5,25 +5,20 @@ urllib3>=2.6.0
 python-baseconv
 boto3
 pysftp
-netifaces
-pyroute2.core
 redis
 requests[socks]
 elasticsearch<9.0.0,>=8.0.0
 python-datemath!=3.0.2
 packaging
-tabulate
 PyYAML
-easydict
 bcrypt
 cart
-cccs-ssdeep
+assemblyline-toolbox==0.3.0
 python-magic
 pytz
 apscheduler
 websocket_client<1.0.0
 elastic-apm[flask]>=6.13.0
-cython
 docker
 kubernetes>18
 notifications-python-client
@@ -33,7 +28,6 @@ azure-identity
 msoffcrypto-tool
 chardet<6
 yara-python
-python-tlsh
 hauntedhouse==0.1.10
 magika
 paramiko<4