assemblyline 4.7.3.dev13__tar.gz → 4.7.3.dev16__tar.gz

{assemblyline-4.7.3.dev13/assemblyline.egg-info → assemblyline-4.7.3.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.3.dev13
+Version: 4.7.3.dev16
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.3.dev16/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.3.dev16

{assemblyline-4.7.3.dev13 → assemblyline-4.7.3.dev16}/assemblyline/datastore/helper.py

@@ -598,7 +598,7 @@ class AssemblylineDatastore(object):
 
     @elasticapm.capture_span(span_type='datastore')
     def get_or_create_file_tree(self, submission, max_depth, cl_engine=forge.get_classification(),
-                                user_classification=None):
+                                user_classification=None, get_full_tree: bool = False):
         # Generate cache key
         if user_classification is not None:
             user_classification = cl_engine.normalize_classification(user_classification, long_format=False)
@@ -616,8 +616,12 @@ class AssemblylineDatastore(object):
         num_files = len(list({x[:64] for x in submission['results']}))
         max_score = submission['max_score']
 
-        # Load / Validate cache tree if exist
-        cached_tree = self.submission_tree.get_if_exists(cache_key, as_obj=False)
+        # bypass cache if the full tree is requested as the cache may hold a non-full version
+        if get_full_tree:
+            cached_tree = None
+        else:
+            # Load / Validate cache tree if exist
+            cached_tree = self.submission_tree.get_if_exists(cache_key, as_obj=False)
         if cached_tree:
             tree = json.loads(cached_tree['tree'])
             if self._is_valid_tree(tree, num_files, max_score):
@@ -704,8 +708,8 @@ class AssemblylineDatastore(object):
             file_sha256 = current_file['sha256']
             file_name = current_file['name']
 
-            # Check if the file not already in the tree and if its allowed to be processed
-            if file_sha256 not in tree_branch \
+            # Check if the file not already in the tree (unless full tree is requested) and if its allowed to be processed
+            if (get_full_tree or file_sha256 not in tree_branch) \
                     and file_sha256 not in forbidden_files \
                     and file_sha256 not in missing_files:
 
@@ -726,7 +730,7 @@ class AssemblylineDatastore(object):
                 # Process each children of the file
                 for new_child in files.get(file_sha256, []):
                     # Check if the file has already been processed elsewhere in the tree
-                    if new_child['sha256'] in tree_cache:
+                    if not get_full_tree and new_child['sha256'] in tree_cache:
                         truncated = True
                     else:
                         # Process file children

{assemblyline-4.7.3.dev13 → assemblyline-4.7.3.dev16}/assemblyline/odm/base.py

@@ -78,7 +78,7 @@ SHA256_REGEX = r"^[a-f0-9]{64}$"
 MAC_REGEX = r"^(?:(?:[0-9a-f]{2}-){5}[0-9a-f]{2}|(?:[0-9a-f]{2}:){5}[0-9a-f]{2})$"
 URI_PATH = r"([/?#]\S*)"
 # Used for finding URIs in a blob
-URI_REGEX = f"((?:(?:[A-Za-z][A-Za-z0-9+.-]*:)//)(?:[^/?#\\s]+@)?({IP_REGEX}|{DOMAIN_REGEX})(?::\\d{{1,5}})?" \
+URI_REGEX = f"((?:(?:[A-Za-z][A-Za-z0-9+.-]*:)//)(?:[^/?#\\s]*@)?({IP_REGEX}|{DOMAIN_REGEX})(?::\\d{{1,5}})?" \
             f"{URI_PATH}?)"
 # Used for direct matching
 FULL_URI = f"^{URI_REGEX}$"

{assemblyline-4.7.3.dev13 → assemblyline-4.7.3.dev16}/assemblyline/odm/models/ontology/results/malware_config.py

@@ -1,6 +1,9 @@
 from assemblyline import odm
 from assemblyline.common.attack_map import attack_map
-from assemblyline.odm.models.ontology.results.network import REQUEST_METHODS, LOOKUP_TYPES
+from assemblyline.odm.models.ontology.results.network import (
+    LOOKUP_TYPES,
+    REQUEST_METHODS,
+)
 
 # Based on model in MaCo framework
 CATEGORIES = ["adware", "apt", "backdoor", "banker", "bootkit", "bot", "browser_hijacker", "bruteforcer",
@@ -8,7 +11,7 @@ CATEGORIES = ["adware", "apt", "backdoor", "banker", "bootkit", "bot", "browser_
               "infostealer", "keylogger", "loader", "obfuscator", "pos", "proxy", "rat", "ransomware",
               "reverse_proxy", "rootkit", "scanner", "scareware", "spammer", "trojan", "virus", "wiper",
               "webshell", "worm"]
-CONNECTION_USAGE = ["c2", "upload", "download", "propagate", "tunnel", "other", "ransom", "decoy"]
+CONNECTION_USAGE = ["c2", "upload", "download", "propagate", "tunnel", "other", "ransom", "decoy", "dead_drop_resolver"]
 
 
 @odm.model(description="Encryption details")
@@ -21,6 +24,8 @@ class Encryption(odm.Model):
     iv = odm.Optional(odm.Text(), description="Initialization Vector")
     seed = odm.Optional(odm.Text(), description="Seed")
     nonce = odm.Optional(odm.Text(), description="Nonce value")
+    password = odm.Optional(odm.Text(), description="Password")
+    salt = odm.Optional(odm.Text(), description="Salt")
     constants = odm.Optional(odm.List(odm.Text()), description="Constants")
     usage = odm.Optional(odm.Enum(values=["config", "communication", "binary", "ransom", "other"]),
                          description="Purpose of encryptions")
@@ -103,6 +108,15 @@ class DNS(odm.Model):
     usage = odm.Optional(odm.Enum(values=CONNECTION_USAGE), description="Purpose of DNS connection")
 
 
+@odm.model(description="Usage of ICMP")
+class ICMP(odm.Model):
+    type = odm.Optional(odm.Integer(), description="ICMP type")
+    code = odm.Optional(odm.Integer(), description="ICMP code")
+    header = odm.Optional(odm.Text(), description="Non-standard header fields")
+    hostname = odm.Optional(odm.Text(), description="Hostname")
+    usage = odm.Optional(odm.Enum(values=CONNECTION_USAGE), description="Purpose of ICMP connection")
+
+
 @odm.model(description="Usage of General TCP/UDP connection")
 class GeneralConnection(odm.Model):
     client_ip = odm.Optional(odm.IP(), description="Client IP")
@@ -125,7 +139,7 @@ class Service(odm.Model):
 class Cryptocurrency(odm.Model):
     coin = odm.Optional(odm.Text(), description="Name of coin used")
     address = odm.Optional(odm.Text(), description="Wallet address")
-    random_amount = odm.Optional(odm.Integer(), description="Ransom amount")
+    ransom_amount = odm.Optional(odm.Float(), description="Ransom amount")
     usage = odm.Optional(odm.Enum(values=['ransomware', 'miner', 'other']), description="Use of cryptocurrency")
 
 
@@ -187,6 +201,58 @@ class Registry(odm.Model):
         description="Use of registry key")
 
 
+SCHEDULED_TASK_USAGE = ["persistence", "defense_evasion", "privilege_escalation",
+                       "lateral_movement", "staging_data", "other"]
+SCHEDULED_TASK_OPERATIONS = ["CHANGE", "CREATE", "DELETE", "END", "QUERY", "RUN"]
+SCHEDULED_TASK_SCHEDULE_TYPES = ["MINUTE", "HOURLY", "DAILY", "WEEKLY", "MONTHLY",
+                                 "ONCE", "ONSTART", "ONLOGON", "ONIDLE", "ONEVENT", "OTHER"]
+SCHEDULED_TASK_RUN_AS = ["SYSTEM", "USER"]
+SCHEDULED_TASK_RUN_LEVELS = ["HIGHEST", "LIMITED"]
+SCHEDULED_TASK_OUTPUT_FORMATS = ["TABLE", "LIST", "CSV"]
+
+
+@odm.model(description="Scheduled task usage by malware")
+class ScheduledTask(odm.Model):
+    usage = odm.Optional(odm.Enum(values=SCHEDULED_TASK_USAGE), description="Scheduled task usage")
+    raw_command = odm.Optional(odm.Text(), description="Raw command used for the scheduled task")
+    task_type = odm.Optional(odm.Enum(values=SCHEDULED_TASK_OPERATIONS), description="Task operation type")
+    schedule_type = odm.Optional(odm.Enum(values=SCHEDULED_TASK_SCHEDULE_TYPES), description="Task schedule type")
+    task_name = odm.Optional(odm.Text(), description="Name of the scheduled task")
+    task_run = odm.Optional(odm.Text(), description="Program or command that the task runs")
+    remote_computer = odm.Optional(odm.Text(), description="Name or IP of a remote computer")
+    user_domain = odm.Optional(odm.Text(), description="User account domain")
+    user_account = odm.Optional(odm.Text(), description="User account to use when running the task")
+    user_password = odm.Optional(odm.Text(), description="Password for the user account")
+    run_as = odm.Optional(odm.Enum(values=SCHEDULED_TASK_RUN_AS), description="Account to run the task as")
+    run_as_domain = odm.Optional(odm.Text(), description="Domain of the account to run the task as")
+    run_as_user = odm.Optional(odm.Text(), description="User of the account to run the task as")
+    run_as_password = odm.Optional(odm.Text(), description="Password of the account to run the task as")
+    modifier = odm.Optional(odm.Text(), description="Modifier for the schedule type")
+    day = odm.Optional(odm.Text(), description="How often the task runs within its schedule type")
+    month = odm.Optional(odm.Text(), description="Month(s) during which the scheduled task should run")
+    idle_time = odm.Optional(odm.Text(), description="Idle time to wait before running the task")
+    start_time = odm.Optional(odm.Text(), description="Start time to run the task (HH:mm 24-hour)")
+    interval = odm.Optional(odm.Text(), description="Repetition interval for the task")
+    end_time = odm.Optional(odm.Text(), description="End time for the task")
+    duration = odm.Optional(odm.Text(), description="Duration for which the task should run")
+    kill = odm.Optional(odm.Boolean(), description="Terminate task if it runs longer than end time or duration")
+    start_date = odm.Optional(odm.Text(), description="Start date to run the task (MM/dd/yyyy)")
+    end_date = odm.Optional(odm.Text(), description="End date to run the task (MM/dd/yyyy)")
+    channel_name = odm.Optional(odm.Text(), description="Event log channel for event-based task")
+    interactive = odm.Optional(odm.Boolean(), description="Task runs only when user is logged on interactively")
+    no_password = odm.Optional(odm.Boolean(), description="Task does not require a password")
+    auto_delete = odm.Optional(odm.Boolean(), description="Task will be deleted after it runs")
+    xml = odm.Optional(odm.Text(), description="XML file containing the task definition")
+    v1 = odm.Optional(odm.Boolean(), description="Create using version 1 task scheduler")
+    force = odm.Optional(odm.Boolean(), description="Create/delete the task and suppress warnings")
+    run_level = odm.Optional(odm.Enum(values=SCHEDULED_TASK_RUN_LEVELS), description="Run level for the task")
+    delay_time = odm.Optional(odm.Text(), description="Wait time to delay running the task after trigger")
+    hresult = odm.Optional(odm.Text(), description="Process exit code in HRESULT format")
+    output_format = odm.Optional(odm.Enum(values=SCHEDULED_TASK_OUTPUT_FORMATS), description="Query output format")
+    no_header = odm.Optional(odm.Boolean(), description="Display column headers in output")
+    add_advanced_properties = odm.Optional(odm.Boolean(), description="Display all properties in output")
+
+
 @odm.model(description="Extracted Malware Configuration")
 class MalwareConfig(odm.Model):
     config_extractor = odm.Keyword(description="Name of extractor")
@@ -216,6 +282,7 @@ class MalwareConfig(odm.Model):
     ssh = odm.Optional(odm.List(odm.Compound(SSH)), description="SSHs")
     proxy = odm.Optional(odm.List(odm.Compound(Proxy)), description="Proxies")
     dns = odm.Optional(odm.List(odm.Compound(DNS)), description="DNS")
+    icmp = odm.Optional(odm.List(odm.Compound(ICMP)), description="ICMPs")
     tcp = odm.Optional(odm.List(odm.Compound(GeneralConnection)), description="TCPs")
     udp = odm.Optional(odm.List(odm.Compound(GeneralConnection)), description="UDPs")
     encryption = odm.Optional(odm.List(odm.Compound(Encryption)), description="Encryptions")
@@ -223,5 +290,6 @@ class MalwareConfig(odm.Model):
     cryptocurrency = odm.Optional(odm.List(odm.Compound(Cryptocurrency)), description="Cryptocurrencies")
     paths = odm.Optional(odm.List(odm.Compound(Path)), description="Paths")
     registry = odm.Optional(odm.List(odm.Compound(Registry)), description="Registry")
+    scheduled_tasks = odm.Optional(odm.List(odm.Compound(ScheduledTask)), description="Scheduled Tasks")
 
     other = odm.Optional(odm.Mapping(odm.Any()), description="Other information")

{assemblyline-4.7.3.dev13 → assemblyline-4.7.3.dev16/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.3.dev13
+Version: 4.7.3.dev16
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.3.dev13/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.3.dev13