assemblyline 4.7.2.dev41__tar.gz → 4.7.2.dev43__tar.gz

{assemblyline-4.7.2.dev41/assemblyline.egg-info → assemblyline-4.7.2.dev43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.2.dev41
+Version: 4.7.2.dev43
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.2.dev43/assemblyline/VERSION

@@ -0,0 +1 @@
+4.7.2.dev43

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/backupmanager.py

@@ -106,7 +106,7 @@ class DistributedBackup(object):
         self.done_queue: NamedQueue[dict[str, Any]] = NamedQueue(f"r-done-{self.instance_id}", ttl=1800)
         self.hash_queue: Hash[str] = Hash(f"r-hash-{self.instance_id}")
         self.bucket_error: list[str] = []
-        self.valid_buckets: list[str] = sorted(list(self.datastore.ds.get_models().keys()))
+        self.valid_buckets: list[str] = sorted(self.datastore.ds.get_models().keys())
         self.worker_count = worker_count
         self.spawn_workers = spawn_workers
         self.total_count = 0
@@ -187,17 +187,17 @@ class DistributedBackup(object):
         for k, v in self.map_count.items():
             summary += "\t%15s: %s\n" % (k.upper(), v)
 
-        if len(self.missing_map_count.keys()) > 0:
+        if self.missing_map_count:
             summary += "\n\nMissing data:\n\n"
             for k, v in self.missing_map_count.items():
                 summary += "\t%15s: %s\n" % (k.upper(), v)
 
-        if len(self.error_map_count.keys()) > 0:
+        if self.error_map_count:
             summary += "\n\nErrors:\n\n"
             for k, v in self.error_map_count.items():
                 summary += "\t%15s: %s\n" % (k.upper(), v)
 
-        if len(self.bucket_error) > 0:
+        if self.bucket_error:
             summary += f"\nThese buckets failed to {title.lower()} completely: {self.bucket_error}\n"
         if self.logger:
             self.logger.info(summary)

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/bundling.py

@@ -330,17 +330,17 @@ def import_bundle(
 
                 # Check if we have all the service results
                 for res_key in submission['results']:
-                    if results is None or (res_key not in results['results'].keys() and not allow_incomplete):
+                    if results is None or (res_key not in results['results'] and not allow_incomplete):
                         raise IncompleteBundle("Incomplete results in bundle. Skipping %s..." % sid)
 
                 # Check if we have all files
-                for sha256 in list(set([x[:64] for x in submission['results']])):
-                    if files is None or (sha256 not in files['infos'].keys() and not allow_incomplete):
+                for sha256 in list({x[:64] for x in submission['results']}):
+                    if files is None or (sha256 not in files['infos'] and not allow_incomplete):
                         raise IncompleteBundle("Incomplete files in bundle. Skipping %s..." % sid)
 
                 # Check if we all errors
                 for err_key in submission['errors']:
-                    if errors is None or (err_key not in errors['errors'].keys() and not allow_incomplete):
+                    if errors is None or (err_key not in errors['errors'] and not allow_incomplete):
                         raise IncompleteBundle("Incomplete errors in bundle. Skipping %s..." % sid)
 
                 # Check if the submission does not already exist

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/classification.py

@@ -258,7 +258,7 @@ class Classification(object):
 
         if long_format:
             return sorted([self.access_req_map_stl[r] for r in return_set]), unused
-        return sorted(list(return_set)), unused
+        return sorted(return_set), unused
 
     def _get_c12n_groups(self, c12n: list[str], long_format: bool = True, get_dynamic_groups: bool = True,
                          auto_select: bool = False) -> Tuple[List, List, list[str]]:
@@ -276,7 +276,7 @@ class Classification(object):
             if gp.startswith("REL "):
                 gp = gp.replace("REL TO ", "")
                 gp = gp.replace("REL ", "")
-                temp_group = set([x.strip() for x in gp.split(",")])
+                temp_group = {x.strip() for x in gp.split(",")}
                 for t in temp_group:
                     groups.extend(t.split("/"))
             else:
@@ -330,7 +330,7 @@ class Classification(object):
         for subgroup in g2_set:
             limited_to_group = self.params_map.get(subgroup, {}).get("limited_to_group", None)
             if limited_to_group is not None:
-                if len(g1_set) > 1 or (len(g1_set) == 1 and g1_set != set([limited_to_group])):
+                if len(g1_set) > 1 or (len(g1_set) == 1 and g1_set != {limited_to_group}):
                     raise InvalidClassification(f"Subgroup {subgroup} is limited to group "
                                                 f"{limited_to_group} (found: {', '.join(g1_set)})")
 
@@ -347,7 +347,7 @@ class Classification(object):
                 sorted([self.subgroups_map_stl[r] for r in g2_set]),
                 list(others)
             )
-        return sorted(list(g1_set)), sorted(list(g2_set)), list(others)
+        return sorted(g1_set), sorted(g2_set), list(others)
 
     @staticmethod
     def _can_see_required(user_req: List, req: List) -> bool:
@@ -355,14 +355,10 @@ class Classification(object):
 
     @staticmethod
     def _can_see_groups(user_groups: List, req: List) -> bool:
-        if len(req) == 0:
+        if not req:
             return True
 
-        for g in user_groups:
-            if g in req:
-                return True
-
-        return False
+        return any(g in req for g in user_groups)
 
     # noinspection PyTypeChecker
     def _get_normalized_classification_text(self, lvl_idx: int, req: List, groups: List, subgroups: List,
@@ -391,10 +387,10 @@ class Classification(object):
         # 3. Add auto-selected subgroups
         if long_format:
             if len(subgroups) > 0 and len(self.subgroups_auto_select) > 0 and not skip_auto_select:
-                subgroups = sorted(list(set(subgroups).union(set(self.subgroups_auto_select))))
+                subgroups = sorted(set(subgroups).union(set(self.subgroups_auto_select)))
         else:
             if len(subgroups) > 0 and len(self.subgroups_auto_select_short) > 0 and not skip_auto_select:
-                subgroups = sorted(list(set(subgroups).union(set(self.subgroups_auto_select_short))))
+                subgroups = sorted(set(subgroups).union(set(self.subgroups_auto_select_short)))
 
         # 4. For every subgroup, check if the subgroup requires or is limited to a specific group
         temp_groups = []
@@ -420,10 +416,10 @@ class Classification(object):
         # 5. Add auto-selected groups
         if long_format:
             if len(groups) > 0 and len(self.groups_auto_select) > 0 and not skip_auto_select:
-                groups = sorted(list(set(groups).union(set(self.groups_auto_select))))
+                groups = sorted(set(groups).union(set(self.groups_auto_select)))
         else:
             if len(groups) > 0 and len(self.groups_auto_select_short) > 0 and not skip_auto_select:
-                groups = sorted(list(set(groups).union(set(self.groups_auto_select_short))))
+                groups = sorted(set(groups).union(set(self.groups_auto_select_short)))
 
         if groups:
             groups = sorted(groups)
@@ -786,9 +782,9 @@ class Classification(object):
 
         cur_part = parts.pop(0)
         # First parts as to be a classification level part
-        if cur_part not in self.levels_aliases.keys() and \
-                cur_part not in self.levels_map_lts.keys() and \
-                cur_part not in self.levels_map_stl.keys():
+        if cur_part not in self.levels_aliases and \
+                cur_part not in self.levels_map_lts and \
+                cur_part not in self.levels_map_stl:
             return False
 
         check_groups = False
@@ -810,19 +806,19 @@ class Classification(object):
             for i in items:
                 if not check_groups:
                     # If current item not found in access req, we might already be dealing with groups
-                    if i not in self.access_req_aliases.keys() and \
-                            i not in self.access_req_map_stl.keys() and \
-                            i not in self.access_req_map_lts.keys():
+                    if i not in self.access_req_aliases and \
+                            i not in self.access_req_map_stl and \
+                            i not in self.access_req_map_lts:
                         check_groups = True
 
                 if check_groups and not self.dynamic_groups:
                     # If not groups. That stuff does not exists...
-                    if i not in self.groups_aliases.keys() and \
-                            i not in self.groups_map_stl.keys() and \
-                            i not in self.groups_map_lts.keys() and \
-                            i not in self.subgroups_aliases.keys() and \
-                            i not in self.subgroups_map_stl.keys() and \
-                            i not in self.subgroups_map_lts.keys():
+                    if i not in self.groups_aliases and \
+                            i not in self.groups_map_stl and \
+                            i not in self.groups_map_lts and \
+                            i not in self.subgroups_aliases and \
+                            i not in self.subgroups_map_stl and \
+                            i not in self.subgroups_map_lts:
                         return False
 
         return True

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/identify.py

@@ -373,8 +373,8 @@ class Identify:
                 with open(path, newline='') as csvfile:
                     try:
                         # Try to read the file as a normal csv without special sniffed dialect
-                        complete_data = [x for x in islice(csv.reader(csvfile), 100)]
-                        if len(complete_data) > 2 and len(set([len(x) for x in complete_data])) == 1:
+                        complete_data = list(islice(csv.reader(csvfile), 100))
+                        if len(complete_data) > 2 and len({len(x) for x in complete_data}) == 1:
                             data["type"] = "text/csv"
                             # Final type identified, shortcut further processing
                             return data
@@ -385,8 +385,8 @@ class Identify:
                         # Normal CSV didn't work, try sniffing the csv to see how we could parse it
                         dialect = csv.Sniffer().sniff(csvfile.read(1024))
                         csvfile.seek(0)
-                        complete_data = [x for x in islice(csv.reader(csvfile, dialect), 100)]
-                        if len(complete_data) > 2 and len(set([len(x) for x in complete_data])) == 1:
+                        complete_data = list(islice(csv.reader(csvfile, dialect), 100))
+                        if len(complete_data) > 2 and len({len(x) for x in complete_data}) == 1:
                             data["type"] = "text/csv"
                             # Final type identified, shortcut further processing
                             return data
@@ -463,7 +463,7 @@ def zip_ident(path: str, fallback: str) -> str:
 
     try:
         with zipfile.ZipFile(path, "r") as zf:
-            file_list = [zfname for zfname in zf.namelist()]
+            file_list = list(zf.namelist())
     except Exception:
         try:
             stdout, _ = subprocess.Popen(

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/path.py

@@ -37,10 +37,7 @@ VALID_UNC_CHARS = [chr(x) for x in ASCII_LOWER_CASE_LETTERS +
 def is_unc_legal(path: str) -> bool:
     """Determine whether or not a given string representing a Windows file path is legal
     or not as per the Unified Naming Convention (UNC) specifications."""
-    if len(path) <= 0:
+    if not path:
         return False
 
-    for char in path:
-        if char not in VALID_UNC_CHARS:
-            return False
-    return True
+    return all(char in VALID_UNC_CHARS for char in path)

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/postprocess.py

@@ -102,16 +102,10 @@ class AndOperatorNode(NodeInterface):
         return f'({" AND ".join(repr(a) for a in self.args)})'
 
     def test(self, sub: Submission, score=None, tags: Optional[list[dict[str, Any]]] = None) -> bool:
-        for arg in self.args:
-            if not arg.test(sub, score, tags):
-                return False
-        return True
+        return all(arg.test(sub, score, tags) for arg in self.args)
 
     def test_value(self, field, value) -> bool:
-        for arg in self.args:
-            if not arg.test_value(field, value):
-                return False
-        return True
+        return all(arg.test_value(field, value) for arg in self.args)
 
 
 class OrOperatorNode(NodeInterface):
@@ -122,16 +116,10 @@ class OrOperatorNode(NodeInterface):
         return f'({" OR ".join(repr(a) for a in self.args)})'
 
     def test(self, sub: Submission, score=None, tags: Optional[list[dict[str, Any]]] = None) -> bool:
-        for arg in self.args:
-            if arg.test(sub, score, tags):
-                return True
-        return False
+        return any(arg.test(sub, score, tags) for arg in self.args)
 
     def test_value(self, field, value) -> bool:
-        for arg in self.args:
-            if arg.test_value(field, value):
-                return True
-        return False
+        return any(arg.test_value(field, value) for arg in self.args)
 
 
 class NotOperatorNode(NodeInterface):

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/common/str_utils.py

@@ -50,7 +50,7 @@ def wrap_bidir_unicode_string(uni_str: Union[str, bytes]) -> Union[str, bytes]:
     affecting their left-to-right direction
     """
 
-    if len(uni_str) == 0 or isinstance(uni_str, bytes):  # Not str, return it unchanged
+    if not uni_str or isinstance(uni_str, bytes):  # Not str, return it unchanged
         return uni_str
 
     re_obj = re.search(rf"[{''.join(CONTROL_CHARS)}]", uni_str)
@@ -243,7 +243,7 @@ class StringTable(object):
 
         # we also import the list as attributes so things like
         # tab completion and introspection still work.
-        for s in self._value_map.keys():
+        for s in self._value_map:
             setattr(self, s, s)
 
     def name_for_value(self, v):

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/datastore/helper.py

@@ -9,7 +9,7 @@ from typing import Any, List, Optional, Tuple, Union
 import elasticapm
 
 from assemblyline.common import forge
-from assemblyline.common.classification import InvalidClassification
+from assemblyline.common.classification import Classification, InvalidClassification
 from assemblyline.common.dict_utils import flatten, recursive_update
 from assemblyline.common.isotime import now_as_iso
 from assemblyline.common.tagging import tag_dict_to_ai_list
@@ -280,7 +280,9 @@ class AssemblylineDatastore(object):
         return self.ds.with_retries(self.ds.client.bulk, operations=operations)
 
     @elasticapm.capture_span(span_type='datastore')
-    def delete_submission_tree_bulk(self, sid, cl_engine=forge.get_classification(), cleanup=True, transport=None):
+    def delete_submission_tree_bulk(
+            self, sid: str, cl_engine: Classification = forge.get_classification(),
+            cleanup: bool = True, transport: FileStore | None = None) -> None:
         submission = self.submission.get(sid, as_obj=False)
         params = SubmissionParams(submission['params'])
         if not submission:
@@ -309,28 +311,30 @@ class AssemblylineDatastore(object):
         results = submission["results"]
         files_to_delete = set()
         fix_classification_files = set()
-        supp_map = {}
+        supp_map: dict[str, set[str]] = {}
 
-        temp_files = [x['sha256'] for x in submission['files']]
-        temp_files.extend([x[:64] for x in errors])
-        temp_files.extend([x[:64] for x in results])
-        temp_files = set(temp_files)
+        temp_files = {x['sha256'] for x in submission['files']}
+        temp_files.update(x[:64] for x in errors)
+        temp_files.update(x[:64] for x in results)
 
-        # Gather all supplementary files
-        for result in self.result.search("response.supplementary.sha256:*",
-                                         fl="*", rows=len(results), as_obj=False, key_space=results)['items']:
+        # Gather all supplementary files and extracted files
+        for result in self.result.search("response.supplementary.sha256:* OR response.extracted.sha256:*",
+                                         fl="sha256,response.supplementary.sha256,response.extracted.sha256",
+                                         rows=len(results), as_obj=False, key_space=results)['items']:
             sha256 = result['sha256']
-            supp_map.setdefault(sha256, set())
-            for supp in result['response']['supplementary']:
+            if sha256 not in supp_map:
+                supp_map[sha256] = set()
+            for supp in result['response'].get('supplementary', []):
                 supp_map[sha256].add(supp['sha256'])
+            for extracted in result['response'].get('extracted', []):
+                temp_files.add(extracted['sha256'])
 
         # Inspect each files to see if they are reused
         for temp in temp_files:
             # Check if we delete the file or update the classification
             if self.submission.search(f"errors:{temp}* OR results:{temp}*", rows=0, as_obj=False)["total"] < 2:
                 files_to_delete.add(temp)
-                supp_list = supp_map.pop(temp, set())
-                files_to_delete = files_to_delete.union(supp_list)
+                files_to_delete |= supp_map.pop(temp, set())
             else:
                 fix_classification_files.add(temp)
 
@@ -402,8 +406,8 @@ class AssemblylineDatastore(object):
 
     @elasticapm.capture_span(span_type='datastore')
     def delete_submission_tree(
-            self, sid, cl_engine=forge.get_classification(),
-            cleanup=True, transport: FileStore = None):
+            self, sid: str, cl_engine: Classification = forge.get_classification(),
+            cleanup: bool = True, transport: FileStore | None = None) -> None:
         submission = self.submission.get(sid, as_obj=False)
         if not submission:
             return
@@ -413,27 +417,29 @@ class AssemblylineDatastore(object):
         results = submission["results"]
         files_to_delete = set()
         fix_classification_files = set()
-        supp_map = {}
+        supp_map: dict[str, set[str]] = {}
 
-        temp_files = [x[:64] for x in errors]
-        temp_files.extend([x[:64] for x in results])
-        temp_files = set(temp_files)
+        temp_files = {x[:64] for x in errors}
+        temp_files.update(x[:64] for x in results)
 
-        # Gather all supplementary files
-        for result in self.result.search("response.supplementary.sha256:*",
-                                         fl="*", rows=len(results), as_obj=False, key_space=results)['items']:
+        # Gather all supplementary files and extracted files
+        for result in self.result.search("response.supplementary.sha256:* OR response.extracted.sha256:*",
+                                         fl="sha256,response.supplementary.sha256,response.extracted.sha256",
+                                         rows=len(results), as_obj=False, key_space=results)['items']:
             sha256 = result['sha256']
-            supp_map.setdefault(sha256, set())
-            for supp in result['response']['supplementary']:
+            if sha256 not in supp_map:
+                supp_map[sha256] = set()
+            for supp in result['response'].get('supplementary', []):
                 supp_map[sha256].add(supp['sha256'])
+            for extracted in result['response'].get('extracted', []):
+                temp_files.add(extracted['sha256'])
 
         # Inspect each files to see if they are reused
         for temp in temp_files:
             # Check if we delete the file or update the classification
-            if self.submission.search(f"errors:{temp}* OR results:{temp}*", rows=0, as_obj=False)["total"] < 2:
+            if self.submission.search(f"NOT sid:{sid} AND (errors:{temp}* OR results:{temp}*)", rows=0, as_obj=False)["total"] == 0:
                 files_to_delete.add(temp)
-                supp_list = supp_map.pop(temp, set())
-                files_to_delete = files_to_delete.union(supp_list)
+                files_to_delete |= supp_map.pop(temp, set())
             else:
                 fix_classification_files.add(temp)
 
@@ -607,7 +613,7 @@ class AssemblylineDatastore(object):
             submission = submission.as_primitives()
 
         # Get number of files and score
-        num_files = len(list(set([x[:64] for x in submission['results']])))
+        num_files = len(list({x[:64] for x in submission['results']}))
         max_score = submission['max_score']
 
         # Load / Validate cache tree if exist
@@ -822,7 +828,7 @@ class AssemblylineDatastore(object):
             return out
 
         keys = [x for x in list(keys) if not x.endswith(".e")]
-        file_keys = list(set([x[:64] for x in keys]))
+        file_keys = list({x[:64] for x in keys})
         try:
             items = self.result.multiget(keys, as_obj=False)
         except MultiKeyError as e:
@@ -1436,9 +1442,9 @@ class AssemblylineDatastore(object):
         if min_score:
             query += f" AND result.score:>={min_score}"
 
-        item_list = [x for x in self.result.stream_search(query, fl="id,created,response.service_name,result.score",
+        item_list = list(self.result.stream_search(query, fl="id,created,response.service_name,result.score",
                                                           access_control=access_control, as_obj=False,
-                                                          index_type=index_type)]
+                                                          index_type=index_type))
 
         item_list.sort(key=lambda k: k["created"], reverse=True)
 
@@ -1563,7 +1569,7 @@ class MetadataValidator:
 
             # Determine if there's extra metadata being set that isn't validated/known to the system
             # Ignore metadata fields that have been set by the system
-            system_configured_metadata = set(['ingest_id', 'ts', 'type'])
+            system_configured_metadata = {'ingest_id', 'ts', 'type'}
             extra_metadata = list(set(metadata.keys()) - set(validation_scheme.keys()) - system_configured_metadata)
             if missing_metadata:
                 return (None, f"Required metadata is missing from submission: {missing_metadata}")

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/datastore/support/build.py

@@ -222,10 +222,7 @@ def any_indexed_part(field) -> bool:
     elif isinstance(field, Compound):
         if field.index is not None:
             return field.index
-        for subfield in field.fields().values():
-            if any_indexed_part(subfield):
-                return True
-        return False
+        return any(any_indexed_part(subfield) for subfield in field.fields().values())
 
     else:
         return field.index

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/odm/base.py

@@ -928,7 +928,7 @@ class TypedMapping(dict):
         self.store = store
         self.sanitizer = sanitizer
 
-        for key in items.keys():
+        for key in items:
             if not self.sanitizer.match(key):
                 raise KeyError(f"Illegal key: {key}")
         super().__init__({key: type_p.check(el, ignore_extra_values=ignore_extra_values) for key, el in items.items()})
@@ -943,7 +943,7 @@ class TypedMapping(dict):
         # Update supports three input layouts:
         # 1. A single dictionary
         if len(args) == 1 and isinstance(args[0], dict):
-            for key in args[0].keys():
+            for key in args[0]:
                 if not self.sanitizer.match(key):
                     raise KeyError(f"Illegal key: {key}")
             return super().update({key: self.type.check(item) for key, item in args[0].items()})
@@ -957,7 +957,7 @@ class TypedMapping(dict):
 
         # 3. Key values as arguments, can be combined with others
         if kwargs:
-            for key in kwargs.keys():
+            for key in kwargs:
                 if not self.sanitizer.match(key):
                     raise KeyError(f"Illegal key: {key}")
             return super().update({key: self.type.check(item) for key, item in kwargs.items()})

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/odm/models/alert.py

@@ -80,7 +80,7 @@ class DetailedResults(odm.Model):
     yara = odm.List(odm.Compound(DetailedItem), default=[], description="Information on YARA rule matches that contributed to the alert.")
 
     def update(self, other: DetailedResults) -> None:
-        for field in self.fields().keys():
+        for field in self.fields():
             setattr(self, field, list(set(getattr(self, field) + getattr(other, field))))
 
 

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/odm/models/config.py

@@ -1687,6 +1687,7 @@ class UI(odm.Model):
     api_proxies: APIProxies = odm.Mapping(
         odm.Compound(APIProxies),
         default=DEFAULT_API_PROXIES, description="Proxy requests to the configured API target and add headers")
+    apikey_cache_seconds = odm.optional(odm.integer(description="Number of seconds to cache apikey validation."))
     audit: bool = odm.Boolean(description="Should API calls be audited and saved to a separate log file?")
     audit_login: bool = odm.Boolean(description="Should login successes and failures be part of the audit log as well?")
     banner: Dict[str, str] = odm.Optional(odm.Mapping(
@@ -1746,6 +1747,7 @@ DEFAULT_UI = {
     "allow_replay": False,
     "allow_url_submissions": True,
     "api_proxies": DEFAULT_API_PROXIES,
+    "apikey_cache_seconds": 5 * 60,
     "audit": True,
     "audit_login": False,
     "banner": None,

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/odm/models/ontology/results/network.py

@@ -82,7 +82,7 @@ class NetworkConnection(odm.Model):
             # Include any details involved in the request for hashing
             hash_dict['http_details'] = {
                 field: http_details.get(field)
-                for field in NetworkHTTP.fields().keys() if field.startswith('request_')
+                for field in NetworkHTTP.fields() if field.startswith('request_')
             }
         elif connection_type == "dns":
             # Include the requested domain as part of the hash

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/odm/models/user.py

@@ -200,7 +200,7 @@ def load_roles_form_acls(acls, curRoles):
 
     # Otherwise load the roles from the api_key ACLs
     roles = set({})
-    for acl in ACL_MAP.keys():
+    for acl in ACL_MAP:
         if acl in acls:
             roles = roles.union(ACL_MAP[acl])
 
@@ -215,7 +215,7 @@ def load_roles(types, curRoles):
 
     # Otherwise load the roles from the user type
     roles = set({})
-    for user_type in USER_TYPE_DEP.keys():
+    for user_type in USER_TYPE_DEP:
         if user_type in types:
             roles = roles.union(USER_TYPE_DEP[user_type])
 

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/odm/random_data/__init__.py

@@ -73,7 +73,7 @@ def create_alerts(ds, alert_count=50, submission_list=None, log=None, workflows=
                     # Overwrite with user information
                     event.entity_type = 'user'
                     event.entity_id = get_random_word()
-                event.labels = list(set([get_random_word() for _ in range(random.randint(0, 20))]))
+                event.labels = list({get_random_word() for _ in range(random.randint(0, 20))})
                 event.status = random.choice(list(STATUSES) + [None])
                 event.priority = random.choice(list(PRIORITIES) + [None])
                 return event
@@ -127,7 +127,7 @@ def create_alerts(ds, alert_count=50, submission_list=None, log=None, workflows=
 
 
 def create_heuristics(ds, log=None):
-    for srv in SERVICES.keys():
+    for srv in SERVICES:
         for x in range(5):
             h = random_model_obj(Heuristic)
             h.heur_id = f"{srv.upper()}.{x + 1}"
@@ -402,7 +402,7 @@ def create_users(ds, log=None):
     user_pass = os.getenv("DEV_USER_PASS", 'user') or 'user'
 
     acl = ["R", "W"]
-    roles = [r for r in load_roles_form_acls(acl, [])]
+    roles = list(load_roles_form_acls(acl, []))
 
     user_data = User({
         "agrees_with_tos": "NOW",

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/remote/datatypes/exporting_counter.py

@@ -102,7 +102,7 @@ class AutoExportingCounters(object):
             log.debug(f"{pprint.pformat(thread_copy)}")
 
             # Only export if needs be
-            if self.export_zero or any([isinstance(x, int) and x != 0 for x in thread_copy.values()]):
+            if self.export_zero or any(isinstance(x, int) and x != 0 for x in thread_copy.values()):
                 self.channel.publish(thread_copy)
 
             return thread_copy

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/remote/datatypes/hash.py

@@ -142,7 +142,7 @@ class Hash(Generic[T]):
         return retry_call(self.c.hset, self.name, key, json.dumps(value))
 
     def multi_set(self, data: dict[str, T]):
-        if any(isinstance(key, bytes) for key in data.keys()):
+        if any(isinstance(key, bytes) for key in data):
             raise ValueError("Cannot use bytes for hashmap keys")
         encoded = {key: json.dumps(value) for key, value in data.items()}
         return retry_call(self.c.hset, self.name, mapping=encoded)

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/remote/datatypes/queues/named.py

@@ -87,7 +87,7 @@ def select(*queues, **kw):
     timeout = kw.get('timeout', 0)
     if len(queues) < 1:
         raise TypeError('At least one queue must be specified')
-    if any([type(q) != NamedQueue for q in queues]):
+    if any(type(q) != NamedQueue for q in queues):
         raise TypeError('Only NamedQueues supported')
 
     c = queues[0].c

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43}/assemblyline/remote/datatypes/queues/priority.py

@@ -183,7 +183,7 @@ def select(*queues, **kw):
     timeout = kw.get('timeout', 0)
     if len(queues) < 1:
         raise TypeError('At least one queue must be specified')
-    if any([type(q) != PriorityQueue for q in queues]):
+    if any(type(q) != PriorityQueue for q in queues):
         raise TypeError('Only NamedQueues supported')
 
     c = queues[0].c

{assemblyline-4.7.2.dev41 → assemblyline-4.7.2.dev43/assemblyline.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline
-Version: 4.7.2.dev41
+Version: 4.7.2.dev43
 Summary: Assemblyline 4 - Automated malware analysis framework
 Home-page: https://github.com/CybercentreCanada/assemblyline-base
 Author: CCCS Assemblyline development team

assemblyline-4.7.2.dev41/assemblyline/VERSION

@@ -1 +0,0 @@
-4.7.2.dev41