assemblyline-v4-service 4.5.0.27__py3-none-any.whl → 4.5.0.29__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.5.0.27
+4.5.0.29

assemblyline_v4_service/common/task.py

@@ -88,6 +88,7 @@ class Task:
     def _add_file(self, path: str, name: str, description: str,
                   classification: Optional[Classification] = None,
                   is_section_image: bool = False,
+                  is_supplementary: bool = False,
                   allow_dynamic_recursion: bool = False,
                   parent_relation: str = PARENT_RELATION.EXTRACTED) -> Optional[Dict[str, str]]:
         # Reject empty files
@@ -112,6 +113,7 @@ class Task:
             classification=self._classification.max_classification(self.min_classification, classification),
             path=path,
             is_section_image=is_section_image,
+            is_supplementary=is_supplementary,
             allow_dynamic_recursion=allow_dynamic_recursion,
             parent_relation=parent_relation
         )
@@ -173,9 +175,8 @@ class Task:
         if not description:
             raise ValueError("Description cannot be empty")
 
-        file = self._add_file(
-            path, name, description, classification, is_section_image, parent_relation=parent_relation
-        )
+        file = self._add_file(path, name, description, classification, is_section_image,
+                              is_supplementary=True, parent_relation=parent_relation)
 
         if not file:
             return None

assemblyline_v4_service/run_privileged_service.py

@@ -261,6 +261,7 @@ class RunPrivilegedService(ServerBase):
                     file_info['classification'],
                     task.ttl,
                     file_info.get('is_section_image', False),
+                    file_info.get('is_supplementary', False),
                     expected_sha256=file_info['sha256']
                 )
 

assemblyline_v4_service/updater/updater.py

@@ -320,11 +320,20 @@ class ServiceUpdater(ThreadedCoreBase):
 
         if self._service.update_config.generates_signatures:
             output_directory = tempfile.mkdtemp(prefix="update_dir_", dir=UPDATER_DIR)
+            sources_removed_locally = False
+            if self._update_dir:
+                current_update_dir = os.path.join(self._update_dir, self.updater_type)
 
-            # Check if new signatures have been added
+                if os.path.exists(current_update_dir):
+                    sources_removed_locally = set(os.listdir(current_update_dir)) - \
+                        set([s.name for s in self._service.update_config.sources])
+
+
+            # Check if new signatures have been added (or it there's been a local change since the last update)
             self.log.info("Check for new signatures.")
-            if self.client.signature.update_available(since=epoch_to_iso(old_update_time) or None,
-                                                      sig_type=self.updater_type):
+            if sources_removed_locally or \
+                self.client.signature.update_available(since=epoch_to_iso(old_update_time) or None,
+                                                       sig_type=self.updater_type):
                 self.log.info("An update is available for download from the datastore")
 
                 self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}")

{assemblyline_v4_service-4.5.0.27.dist-info → assemblyline_v4_service-4.5.0.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.5.0.27
+Version: 4.5.0.29
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.5.0.27.dist-info → assemblyline_v4_service-4.5.0.29.dist-info}/RECORD

@@ -1,8 +1,8 @@
-assemblyline_v4_service/VERSION,sha256=p2Vdvfc45LvNQUDOPwM_x-SO5DzFjpjGpf4vWn6HGic,9
+assemblyline_v4_service/VERSION,sha256=5iHmLzWZaCCho1OVr_tp40DbE-FbtCYT4q-9Dc1CJeY,9
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/run_privileged_service.py,sha256=qd4DmHo5G_Tpv8tb0A96qNfCGuXBu5MlIKB205z5e5w,14514
+assemblyline_v4_service/run_privileged_service.py,sha256=ipCjdB_PZXt85Hmu9r9q7QjKXI8Y6iF-w6Aa4PMn9S4,14576
 assemblyline_v4_service/run_service.py,sha256=NiFX52NfsbBQY6E3nrjoB3e2XxIlwFcQpYYY-rADIk4,5996
 assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/common/api.py,sha256=Xzp8j4HCCfjPvNSGKiZl5ttH2_Itg47cjlH0NXNtth0,6849
@@ -12,7 +12,7 @@ assemblyline_v4_service/common/ocr.py,sha256=3fV0PyY3oui_ucAM9dkolP0VRYKACKJuGY4
 assemblyline_v4_service/common/ontology_helper.py,sha256=QpwerYoS5hXjWzpx3Pmwv6j2330PQVYqxYGamjcpW3I,7890
 assemblyline_v4_service/common/request.py,sha256=ZP80rDWIyVotwVDN70L7ujN9RoxZRdmaeUN0DFIvrT4,11732
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
-assemblyline_v4_service/common/task.py,sha256=7yNvXk5B27QookJNp4-Y1iGhV8LfhLDaYWzF-uFGWgo,13576
+assemblyline_v4_service/common/task.py,sha256=xX771hC3vkjRtnR4MjPh_26ZmAkQ_jceuu6U7BSFMd8,13704
 assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/dev/run_service_once.py,sha256=caqUHYc27JLuJPOwkEuhf8-wlaYPvzDrRqfEuhMMxqw,10455
@@ -22,7 +22,7 @@ assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lC
 assemblyline_v4_service/updater/client.py,sha256=oOOIzh-Q-b5uOTo3C4rVSfQxqUO5Y_ogDmUhvWfm5uo,9518
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
 assemblyline_v4_service/updater/helper.py,sha256=-B35wdjpeY4t1R9SPDrTFHFKHwE3uzy9N69mV6mHy-g,9532
-assemblyline_v4_service/updater/updater.py,sha256=5cJH82VTP3VrwoNE97ubKP2dr90AI_T-AKRistvN5rI,29013
+assemblyline_v4_service/updater/updater.py,sha256=6LFvqBkNu6mDaBKZGEaFGtE8g3DtDNBYWt8jXx6fnOw,29507
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/conftest.py,sha256=W3SieQpZsZpGEmtLqY4aIlxREDSsHceyCrFcFsWUM0U,1851
 test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -34,12 +34,12 @@ test/test_common/test_base.py,sha256=fuJSSlPxIDHq6HU1xbvaMFitw2z1spOZNHD2SJ4UUic
 test/test_common/test_helper.py,sha256=sO6YAiBhKTqaxlpLhFYDuy2ZdbuF2cg07Ylzo83ZzQs,2575
 test/test_common/test_ocr.py,sha256=mt_PgElgwQKJmNrp2nRVx9NjfMedVk40I6IV317vATI,1753
 test/test_common/test_ontology_helper.py,sha256=KhHEBg_ecJyQbDw79NMT4FzUyA4C1Aak3HEQCwBfM2s,7914
-test/test_common/test_request.py,sha256=VzWw-NZKTRIXKwCACIT2SSHE70HMZ3IeBk1AJiT-cjU,11472
-test/test_common/test_result.py,sha256=0WK_Zfw7e6IJl5nTSUUt6LFTvbu6cVynaTOdpuYkBvg,42559
-test/test_common/test_task.py,sha256=RWsGEN0L-xKoRAsGv2x4JZcSAySS9aBF4dwL9t7tepo,18668
+test/test_common/test_request.py,sha256=zoBURe3QbycWK4I7uvkixcmykrOCqY_TchmxJinNXjU,11822
+test/test_common/test_result.py,sha256=6BiOKxEPrKBjOY44jv3TY-yiXm0qI1ok_CZBnjP9TM4,45447
+test/test_common/test_task.py,sha256=LzEIfFHJEcB_YPucOZGtm1TwyHnqJE-0Qc2MDsH4TN4,18957
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.5.0.27.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.5.0.27.dist-info/METADATA,sha256=Z_bZhozIN3PH4RyjQ2DdrD3IjPCGJ0iNQdWde8sb-Ag,9495
-assemblyline_v4_service-4.5.0.27.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-assemblyline_v4_service-4.5.0.27.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.5.0.27.dist-info/RECORD,,
+assemblyline_v4_service-4.5.0.29.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.5.0.29.dist-info/METADATA,sha256=qtnn26Vo3FLqi9Jkp8SWsXVGQOQ6sYiz_KuJSnjzlVo,9495
+assemblyline_v4_service-4.5.0.29.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+assemblyline_v4_service-4.5.0.29.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.5.0.29.dist-info/RECORD,,

test/test_common/test_request.py

@@ -13,6 +13,7 @@ from assemblyline.odm.messages.task import Task as ServiceTask
 # Ensure service manifest is instantiated before importing from OCR submodule
 setup_module()
 
+
 @pytest.fixture
 def service_request():
     st = ServiceTask({
@@ -71,11 +72,12 @@ def test_add_extracted(service_request):
             'classification': 'TLP:C',
             'description': 'description',
             'is_section_image': False,
+            'is_supplementary': False,
             'name': 'name',
             'parent_relation': 'EXTRACTED',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
-         },
+        },
     ]
 
     # Raise MaxExtractedExceeded
@@ -97,17 +99,20 @@ def test_add_extracted(service_request):
             'classification': 'TLP:C',
             'description': 'description',
             'is_section_image': False,
+            'is_supplementary': False,
             'name': 'name',
             'parent_relation': 'DYNAMIC',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
-         },
+        },
     ]
 
 
 @pytest.mark.skipif(len(TESSERACT_LIST) < 1, reason="Requires tesseract-ocr apt package")
 def test_add_image(service_request):
-    image_path = os.path.join(os.path.dirname(__file__), "b32969aa664e3905c20f865cdd7b921f922678f5c3850c78e4c803fbc1757a8e")
+    image_path = os.path.join(
+        os.path.dirname(__file__),
+        "b32969aa664e3905c20f865cdd7b921f922678f5c3850c78e4c803fbc1757a8e")
 
     # Basic
     assert service_request.add_image(image_path, "image_name", "description of image") == {
@@ -131,6 +136,7 @@ def test_add_image(service_request):
             'classification': 'TLP:C',
             'description': 'description of image',
             'is_section_image': True,
+            'is_supplementary': True,
             'name': 'image_name',
             'parent_relation': 'INFORMATION',
             'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329'
@@ -140,6 +146,7 @@ def test_add_image(service_request):
             'classification': 'TLP:C',
             'description': 'description of image (thumbnail)',
             'is_section_image': True,
+            'is_supplementary': True,
             'name': 'image_name.thumb',
             'parent_relation': 'INFORMATION',
             'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901'
@@ -152,7 +159,8 @@ def test_add_image(service_request):
     ocr_heuristic_id = 1
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
-    data = service_request.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io)
+    data = service_request.add_image(image_path, "image_name", "description of image",
+                                     "TLP:A", ocr_heuristic_id, ocr_io)
     assert data["img"] == {
         'description': 'description of image',
         'name': 'image_name',
@@ -188,17 +196,23 @@ def test_add_image(service_request):
 
     heur_dict = get_heuristic_primitives(data["ocr_section"].__dict__["_heuristic"])
 
-    assert heur_dict == {'heur_id': 1, 'score': 1200, 'attack_ids': ['T1005'], 'signatures': {'ransomware_strings': 8}, 'frequency': 0, 'score_map': {}}
+    assert heur_dict == {
+        'heur_id': 1, 'score': 1200, 'attack_ids': ['T1005'],
+        'signatures': {'ransomware_strings': 8},
+        'frequency': 0, 'score_map': {}}
 
     assert service_request.temp_submission_data == {}
 
     service_request.task.supplementary.clear()
 
     # Classification, OCR heuristic, OCR_IO, image with password
-    image_path = os.path.join(os.path.dirname(__file__), "4031ed8786439eee24b87f84901e38038a76b8c55e9d87dd5a7d88df2806c1cf")
+    image_path = os.path.join(
+        os.path.dirname(__file__),
+        "4031ed8786439eee24b87f84901e38038a76b8c55e9d87dd5a7d88df2806c1cf")
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
-    data = service_request.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io)
+    data = service_request.add_image(image_path, "image_name", "description of image",
+                                     "TLP:A", ocr_heuristic_id, ocr_io)
     assert data["img"] == {
         'description': 'description of image',
         'name': 'image_name',
@@ -217,9 +231,12 @@ def test_add_image(service_request):
 
     heur_dict = get_heuristic_primitives(data["ocr_section"].__dict__["_heuristic"])
 
-    assert heur_dict == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'], 'signatures': {'password_strings': 1}, 'frequency': 0, 'score_map': {}}
+    assert heur_dict == {
+        'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'],
+        'signatures': {'password_strings': 1},
+        'frequency': 0, 'score_map': {}}
 
-    assert service_request.temp_submission_data == {'passwords':[' 975', '975', 'DOCUMENT', 'PASSWORD', 'PASSWORD:']}
+    assert service_request.temp_submission_data == {'passwords': [' 975', '975', 'DOCUMENT', 'PASSWORD', 'PASSWORD:']}
 
 
 def test_add_supplementary(service_request):
@@ -239,11 +256,12 @@ def test_add_supplementary(service_request):
             'classification': 'TLP:C',
             'description': 'description',
             'is_section_image': False,
+            'is_supplementary': True,
             'name': 'name',
             'parent_relation': 'INFORMATION',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
-         },
+        },
     ]
 
 
@@ -280,7 +298,7 @@ def test_get_param(service_request):
         service_request.get_param("blah")
 
     # Submission parameter exists
-    service_request.task.service_config = {"blah":"blah"}
+    service_request.task.service_config = {"blah": "blah"}
     assert service_request.get_param("blah") == "blah"
 
 

test/test_common/test_result.py

@@ -1,3 +1,15 @@
+from assemblyline.odm.messages.task import Task as ServiceTask
+from assemblyline_v4_service.common.task import Task
+from assemblyline_v4_service.common.result import (
+    BODY_FORMAT, DividerSectionBody, GraphSectionBody, Heuristic, ImageSectionBody, InvalidFunctionException,
+    InvalidHeuristicException, JSONSectionBody, KVSectionBody, MemorydumpSectionBody, MultiSectionBody,
+    OrderedKVSectionBody, ProcessItem, ProcessTreeSectionBody, Result, ResultAggregationException, ResultGraphSection,
+    ResultImageSection, ResultJSONSection, ResultKeyValueSection, ResultMemoryDumpSection, ResultMultiSection,
+    ResultOrderedKeyValueSection, ResultProcessTreeSection, ResultSection, ResultTableSection, ResultTextSection,
+    ResultTimelineSection, ResultURLSection, SectionBody, TableRow, TableSectionBody, TextSectionBody,
+    TimelineSectionBody, TypeSpecificResultSection, URLSectionBody, get_heuristic_primitives)
+from assemblyline_v4_service.common.request import ServiceRequest
+import pytest
 import os
 import tempfile
 from test.test_common import TESSERACT_LIST, setup_module
@@ -6,13 +18,6 @@ from test.test_common import TESSERACT_LIST, setup_module
 setup_module()
 
 
-import pytest
-from assemblyline_v4_service.common.request import ServiceRequest
-from assemblyline_v4_service.common.result import *
-from assemblyline_v4_service.common.task import Task
-
-from assemblyline.odm.messages.task import Task as ServiceTask
-
 @pytest.fixture
 def heuristic():
     return Heuristic(1)
@@ -595,11 +600,21 @@ def test_imagesectionbody_init(service_request):
 @pytest.mark.skipif(len(TESSERACT_LIST) < 1, reason="Requires tesseract-ocr apt package")
 def test_imagesectionbody_add_image(service_request):
     isb = ImageSectionBody(service_request)
-    image_path = os.path.join(os.path.dirname(__file__), "b32969aa664e3905c20f865cdd7b921f922678f5c3850c78e4c803fbc1757a8e")
+    image_path = os.path.join(
+        os.path.dirname(__file__),
+        "b32969aa664e3905c20f865cdd7b921f922678f5c3850c78e4c803fbc1757a8e")
 
     # Basic
     assert isb.add_image(image_path, "image_name", "description of image") is None
-    assert isb._data == [{'img': {'name': 'image_name', 'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329', 'description': 'description of image'}, 'thumb': {'name': 'image_name.thumb', 'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901', 'description': 'description of image (thumbnail)'}}]
+    assert isb._data == [
+        {'img': {
+            'name': 'image_name',
+            'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329',
+            'description': 'description of image'},
+         'thumb': {
+             'name': 'image_name.thumb',
+             'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901',
+             'description': 'description of image (thumbnail)'}}]
 
     isb._data.clear()
 
@@ -607,8 +622,25 @@ def test_imagesectionbody_add_image(service_request):
     ocr_heuristic_id = 1
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
-    assert isb.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io).body == '{"ransomware": ["YOUR FILES HAVE BEEN ENCRYPTED AND YOU WON\'T BE ABLE TO DECRYPT THEM.", "YOU CAN BUY DECRYPTION SOFTWARE FROM US, THIS SOFTWARE WILL ALLOW YOU TO RECOVER ALL OF YOUR DATA AND", "RANSOMWARE FROM YOUR COMPUTER. THE PRICE OF THE SOFTWARE IS $.2..%.. PAYMENT CAN BE MADE IN BITCOIN OR XMR.", "How 00! PAY, WHERE DO | GET BITCOIN OR XMR?", "YOURSELF TO FIND OUT HOW TO BUY BITCOIN OR XMR.", "PAYMENT INFORMATION: SEND $15, TO ONE OF OUR CRYPTO ADDRESSES, THEN SEND US EMAIL WITH PAYMENT", "CONFIRMATION AND YOU\'LL GET THE DECRYPTION SOFTWARE IN EMAIL.", "BTC ADDRESS : bciqsht77cpgw7kv420r4secmu88g34wvn96dsyc5s"]}'
-    assert isb._data == [{'img': {'name': 'image_name', 'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329', 'description': 'description of image'}, 'thumb': {'name': 'image_name.thumb', 'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901', 'description': 'description of image (thumbnail)'}}]
+    assert isb.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id,
+                         ocr_io).body == '{"ransomware": ["YOUR FILES HAVE BEEN ENCRYPTED AND YOU WON\'T BE ABLE TO ' \
+                                         'DECRYPT THEM.", "YOU CAN BUY DECRYPTION SOFTWARE FROM US, THIS SOFTWARE ' \
+                                         'WILL ALLOW YOU TO RECOVER ALL OF YOUR DATA AND", "RANSOMWARE FROM YOUR ' \
+                                         'COMPUTER. THE PRICE OF THE SOFTWARE IS $.2..%.. PAYMENT CAN BE MADE IN ' \
+                                         'BITCOIN OR XMR.", "How 00! PAY, WHERE DO | GET BITCOIN OR XMR?", "YOURSELF ' \
+                                         'TO FIND OUT HOW TO BUY BITCOIN OR XMR.", "PAYMENT INFORMATION: SEND $15, ' \
+                                         'TO ONE OF OUR CRYPTO ADDRESSES, THEN SEND US EMAIL WITH PAYMENT", ' \
+                                         '"CONFIRMATION AND YOU\'LL GET THE DECRYPTION SOFTWARE IN EMAIL.", ' \
+                                         '"BTC ADDRESS : bciqsht77cpgw7kv420r4secmu88g34wvn96dsyc5s"]}'
+    assert isb._data == [
+        {'img': {
+            'name': 'image_name',
+            'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329',
+            'description': 'description of image'},
+         'thumb': {
+            'name': 'image_name.thumb',
+            'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901',
+            'description': 'description of image (thumbnail)'}}]
 
 
     # Ensure that the image files added are marked as `is_image_section`
@@ -653,7 +685,8 @@ def test_timelinesectionbody_add_node():
 
     tsb.add_node("title", "content", "opposite_content")
     assert tsb._format == BODY_FORMAT.TIMELINE
-    assert tsb._data == [{'title': 'title', 'content': 'content', 'opposite_content': 'opposite_content', 'icon': None, 'signatures': [], 'score': 0}]
+    assert tsb._data == [{'title': 'title', 'content': 'content',
+                          'opposite_content': 'opposite_content', 'icon': None, 'signatures': [], 'score': 0}]
 
 
 def test_resultsection_init():
@@ -710,7 +743,8 @@ def test_resultsection_init():
     assert rs._body_format == BODY_FORMAT.GRAPH_DATA
     assert rs._body_config == {}
     assert rs.classification == 'TLP:AMBER'
-    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'], 'signatures': {}, 'frequency': 1, 'score_map': {}}
+    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': [
+        'T1005'], 'signatures': {}, 'frequency': 1, 'score_map': {}}
     assert rs._tags == {"a": "b"}
     assert rs.zeroize_on_tag_safe is True
     assert rs.auto_collapse is True
@@ -939,14 +973,16 @@ def test_resultsection_set_heuristic():
 
     # Pass int heuristic
     assert rs.set_heuristic(1) is None
-    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'], 'signatures': {}, 'frequency': 1, 'score_map': {}}
+    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': [
+        'T1005'], 'signatures': {}, 'frequency': 1, 'score_map': {}}
 
     rs._heuristic = None
 
     # Pass Heuristic heuristic
     heur = Heuristic(1)
     assert rs.set_heuristic(heur) is None
-    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'], 'signatures': {}, 'frequency': 1, 'score_map': {}}
+    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': [
+        'T1005'], 'signatures': {}, 'frequency': 1, 'score_map': {}}
 
     # Try adding a heuristic again
     with pytest.raises(InvalidHeuristicException):
@@ -957,13 +993,21 @@ def test_resultsection_set_heuristic():
     # Set the Heuristic heuristic with attack ID and signature
     heur = Heuristic(1)
     assert rs.set_heuristic(heur, attack_id="T1001", signature="blah") is None
-    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005', 'T1001'], 'signatures': {"blah": 1}, 'frequency': 1, 'score_map': {}}
+    assert get_heuristic_primitives(
+        rs._heuristic) == {
+        'heur_id': 1, 'score': 250, 'attack_ids': ['T1005', 'T1001'],
+        'signatures': {"blah": 1},
+        'frequency': 1, 'score_map': {}}
 
     rs._heuristic = None
 
     # Set the int heuristic with attack ID and signature
     assert rs.set_heuristic(1, attack_id="T1001", signature="blah") is None
-    assert get_heuristic_primitives(rs._heuristic) == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005', 'T1001'], 'signatures': {"blah": 1}, 'frequency': 1, 'score_map': {}}
+    assert get_heuristic_primitives(
+        rs._heuristic) == {
+        'heur_id': 1, 'score': 250, 'attack_ids': ['T1005', 'T1001'],
+        'signatures': {"blah": 1},
+        'frequency': 1, 'score_map': {}}
 
 
 def test_resultsection_set_tags():
@@ -1241,7 +1285,15 @@ def test_resultimagesection_add_image(service_request):
 
     # Basic
     assert ris.add_image(image_path, "image_name", "description of image") is None
-    assert ris.section_body._data == [{'img': {'name': 'image_name', 'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329', 'description': 'description of image'}, 'thumb': {'name': 'image_name.thumb', 'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901', 'description': 'description of image (thumbnail)'}}]
+    assert ris.section_body._data == [{
+        'img':
+                                      {'name': 'image_name',
+                                       'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329',
+                                       'description': 'description of image'},
+                                      'thumb':
+                                      {'name': 'image_name.thumb',
+                                       'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901',
+                                       'description': 'description of image (thumbnail)'}}]
 
     ris = ResultImageSection(service_request, "title_text_as_str")
 
@@ -1250,8 +1302,15 @@ def test_resultimagesection_add_image(service_request):
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
     assert ris.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io) is None
-    assert ris.section_body._data == [{'img': {'name': 'image_name', 'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329', 'description': 'description of image'}, 'thumb': {'name': 'image_name.thumb', 'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901', 'description': 'description of image (thumbnail)'}}]
-
+    assert ris.section_body._data == [{
+        'img':
+                                      {'name': 'image_name',
+                                       'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329',
+                                       'description': 'description of image'},
+                                      'thumb':
+                                      {'name': 'image_name.thumb',
+                                       'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901',
+                                       'description': 'description of image (thumbnail)'}}]
 
     ris = ResultImageSection(service_request, "title_text_as_str")
 
@@ -1259,8 +1318,28 @@ def test_resultimagesection_add_image(service_request):
     ocr_heuristic_id = 1
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
-    assert ris.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io, auto_add_ocr_section=False).body == '{"ransomware": ["YOUR FILES HAVE BEEN ENCRYPTED AND YOU WON\'T BE ABLE TO DECRYPT THEM.", "YOU CAN BUY DECRYPTION SOFTWARE FROM US, THIS SOFTWARE WILL ALLOW YOU TO RECOVER ALL OF YOUR DATA AND", "RANSOMWARE FROM YOUR COMPUTER. THE PRICE OF THE SOFTWARE IS $.2..%.. PAYMENT CAN BE MADE IN BITCOIN OR XMR.", "How 00! PAY, WHERE DO | GET BITCOIN OR XMR?", "YOURSELF TO FIND OUT HOW TO BUY BITCOIN OR XMR.", "PAYMENT INFORMATION: SEND $15, TO ONE OF OUR CRYPTO ADDRESSES, THEN SEND US EMAIL WITH PAYMENT", "CONFIRMATION AND YOU\'LL GET THE DECRYPTION SOFTWARE IN EMAIL.", "BTC ADDRESS : bciqsht77cpgw7kv420r4secmu88g34wvn96dsyc5s"]}'
-    assert ris.section_body._data == [{'img': {'name': 'image_name', 'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329', 'description': 'description of image'}, 'thumb': {'name': 'image_name.thumb', 'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901', 'description': 'description of image (thumbnail)'}}]
+    assert ris.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io,
+                         auto_add_ocr_section=False).body == '{"ransomware": ["YOUR FILES HAVE BEEN ENCRYPTED AND ' \
+                                                             'YOU WON\'T BE ABLE TO DECRYPT THEM.", "YOU CAN BUY ' \
+                                                             'DECRYPTION SOFTWARE FROM US, THIS SOFTWARE WILL ALLOW ' \
+                                                             'YOU TO RECOVER ALL OF YOUR DATA AND", "RANSOMWARE FROM ' \
+                                                             'YOUR COMPUTER. THE PRICE OF THE SOFTWARE IS $.2..%.. ' \
+                                                             'PAYMENT CAN BE MADE IN BITCOIN OR XMR.", "How 00! PAY, ' \
+                                                             'WHERE DO | GET BITCOIN OR XMR?", "YOURSELF TO FIND OUT ' \
+                                                             'HOW TO BUY BITCOIN OR XMR.", "PAYMENT INFORMATION: ' \
+                                                             'SEND $15, TO ONE OF OUR CRYPTO ADDRESSES, THEN SEND ' \
+                                                             'US EMAIL WITH PAYMENT", "CONFIRMATION AND YOU\'LL GET ' \
+                                                             'THE DECRYPTION SOFTWARE IN EMAIL.", "BTC ADDRESS : ' \
+                                                             'bciqsht77cpgw7kv420r4secmu88g34wvn96dsyc5s"]}'
+    assert ris.section_body._data == [{
+        'img': {
+            'name': 'image_name',
+            'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329',
+            'description': 'description of image'},
+        'thumb': {
+            'name': 'image_name.thumb',
+            'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901',
+            'description': 'description of image (thumbnail)'}}]
 
     # Ensure that the image files added are marked as `is_image_section`
     image_hashes = [img['sha256'] for img in ris.section_body._data[0].values()]
@@ -1281,7 +1360,9 @@ def test_resulttimelinesection_add_node():
 
     rts.add_node("title", "content", "opposite_content")
     assert rts.body_format == BODY_FORMAT.TIMELINE
-    assert rts.section_body._data == [{'title': 'title', 'content': 'content', 'opposite_content': 'opposite_content', 'icon': None, 'signatures': [], 'score': 0}]
+    assert rts.section_body._data == [
+        {'title': 'title', 'content': 'content',
+         'opposite_content': 'opposite_content', 'icon': None, 'signatures': [], 'score': 0}]
 
 
 def test_resultmultisection_init():

test/test_common/test_task.py

@@ -6,7 +6,7 @@ from logging import Logger
 import pytest
 from assemblyline_v4_service.common import helper
 from assemblyline_v4_service.common.result import ResultSection
-from assemblyline_v4_service.common.task import *
+from assemblyline_v4_service.common.task import Task, Classification, MaxExtractedExceeded, ServiceAPI, Result
 
 from assemblyline.odm.messages.task import DataItem, TagItem
 from assemblyline.odm.messages.task import Task as ServiceTask
@@ -160,6 +160,7 @@ def test_task_add_file(servicetask):
         "classification": "TLP:C",
         "path": path,
         "is_section_image": False,
+        'is_supplementary': False,
         "allow_dynamic_recursion": False,
         "parent_relation": "EXTRACTED",
     }
@@ -174,6 +175,7 @@ def test_task_add_file(servicetask):
         "classification": "TLP:C",
         "path": path,
         "is_section_image": False,
+        'is_supplementary': False,
         "allow_dynamic_recursion": True,
         "parent_relation": "DYNAMIC",
     }
@@ -211,7 +213,7 @@ def test_task_add_extracted(servicetask, mocker):
     # Safelisted file hash
     service_attributes = helper.get_service_attributes()
     sa = ServiceAPI(service_attributes, None)
-    with mocker.patch.object(sa, "lookup_safelist", return_value={"enabled": True, "type": "file"}) as m:
+    with mocker.patch.object(sa, "lookup_safelist", return_value={"enabled": True, "type": "file"}) as _:
         t.safelist_config.enabled = True
         assert t.add_extracted(path, "name", "description", safelist_interface=sa) is False
 
@@ -225,6 +227,7 @@ def test_task_add_extracted(servicetask, mocker):
             "classification": "TLP:C",
             "path": path,
             "is_section_image": False,
+            'is_supplementary': False,
             "allow_dynamic_recursion": False,
             "parent_relation": "EXTRACTED",
         }
@@ -260,6 +263,7 @@ def test_task_add_supplementary(servicetask):
         "classification": "TLP:C",
         "path": path,
         "is_section_image": False,
+        'is_supplementary': True,
         "allow_dynamic_recursion": False,
         "parent_relation": "INFORMATION",
     }
@@ -271,6 +275,7 @@ def test_task_add_supplementary(servicetask):
             "classification": "TLP:C",
             "path": path,
             "is_section_image": False,
+            'is_supplementary': True,
             "allow_dynamic_recursion": False,
             "parent_relation": "INFORMATION",
         }
@@ -405,6 +410,7 @@ def test_task_get_service_result(servicetask):
                     "classification": "TLP:C",
                     "description": "description",
                     "is_section_image": False,
+                    "is_supplementary": False,
                     "name": "name",
                     "parent_relation": "EXTRACTED",
                     "path": path,