assemblyline-v4-service 4.5.1.dev112__py3-none-any.whl → 4.5.1.dev115__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.5.1.dev112
+4.5.1.dev115

assemblyline_v4_service/common/task.py

@@ -89,6 +89,7 @@ class Task:
     def _add_file(self, path: str, name: str, description: str,
                   classification: Optional[Classification] = None,
                   is_section_image: bool = False,
+                  is_supplementary: bool = False,
                   allow_dynamic_recursion: bool = False,
                   parent_relation: str = PARENT_RELATION.EXTRACTED) -> Optional[Dict[str, str]]:
         # Reject empty files
@@ -113,6 +114,7 @@ class Task:
             classification=self._classification.max_classification(self.min_classification, classification),
             path=path,
             is_section_image=is_section_image,
+            is_supplementary=is_supplementary,
             allow_dynamic_recursion=allow_dynamic_recursion,
             parent_relation=parent_relation
         )
@@ -174,9 +176,8 @@ class Task:
         if not description:
             raise ValueError("Description cannot be empty")
 
-        file = self._add_file(
-            path, name, description, classification, is_section_image, parent_relation=parent_relation
-        )
+        file = self._add_file(path, name, description, classification, is_section_image,
+                              is_supplementary=True, parent_relation=parent_relation)
 
         if not file:
             return None

assemblyline_v4_service/run_privileged_service.py

@@ -261,6 +261,7 @@ class RunPrivilegedService(ServerBase):
                     file_info['classification'],
                     task.ttl,
                     file_info.get('is_section_image', False),
+                    file_info.get('is_supplementary', False),
                     expected_sha256=file_info['sha256']
                 )
 

{assemblyline_v4_service-4.5.1.dev112.dist-info → assemblyline_v4_service-4.5.1.dev115.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.5.1.dev112
+Version: 4.5.1.dev115
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.5.1.dev112.dist-info → assemblyline_v4_service-4.5.1.dev115.dist-info}/RECORD

@@ -1,8 +1,8 @@
-assemblyline_v4_service/VERSION,sha256=1aaOnR8tLUctx75UHlBlgamlcqzvnvdRLqyU2tB2hyo,13
+assemblyline_v4_service/VERSION,sha256=CiVxl_3WaAaw-Ltgv-KKY-JNT4YecspFdmL3zVkXfq0,13
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/run_privileged_service.py,sha256=qd4DmHo5G_Tpv8tb0A96qNfCGuXBu5MlIKB205z5e5w,14514
+assemblyline_v4_service/run_privileged_service.py,sha256=ipCjdB_PZXt85Hmu9r9q7QjKXI8Y6iF-w6Aa4PMn9S4,14576
 assemblyline_v4_service/run_service.py,sha256=NiFX52NfsbBQY6E3nrjoB3e2XxIlwFcQpYYY-rADIk4,5996
 assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/common/api.py,sha256=Xzp8j4HCCfjPvNSGKiZl5ttH2_Itg47cjlH0NXNtth0,6849
@@ -12,7 +12,7 @@ assemblyline_v4_service/common/ocr.py,sha256=3fV0PyY3oui_ucAM9dkolP0VRYKACKJuGY4
 assemblyline_v4_service/common/ontology_helper.py,sha256=QpwerYoS5hXjWzpx3Pmwv6j2330PQVYqxYGamjcpW3I,7890
 assemblyline_v4_service/common/request.py,sha256=W7fqC2xQE3i5i2jlCDyUDp3ZqJQQqSshNW0mQfJMkFg,11792
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
-assemblyline_v4_service/common/task.py,sha256=-AhtT6qSMLndBB3IOqOF-MV5K8Xp_67yvcgMuzr1IT0,13720
+assemblyline_v4_service/common/task.py,sha256=OIE_V-xyGq9vvuT-U_3DIoGdlY3F4oWV9WIc_jhWXsA,13848
 assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/dev/run_service_once.py,sha256=4K3ljw0MnfPGw0-6lzc_vtUYg1EbntJbbsWvNU-ZM_A,10456
@@ -34,12 +34,12 @@ test/test_common/test_base.py,sha256=fuJSSlPxIDHq6HU1xbvaMFitw2z1spOZNHD2SJ4UUic
 test/test_common/test_helper.py,sha256=sO6YAiBhKTqaxlpLhFYDuy2ZdbuF2cg07Ylzo83ZzQs,2575
 test/test_common/test_ocr.py,sha256=mt_PgElgwQKJmNrp2nRVx9NjfMedVk40I6IV317vATI,1753
 test/test_common/test_ontology_helper.py,sha256=KhHEBg_ecJyQbDw79NMT4FzUyA4C1Aak3HEQCwBfM2s,7914
-test/test_common/test_request.py,sha256=PPhHfrwpwMdNZ33P1Z_0h1Zaz9ao9VFiDr_MJrBS3Lg,11492
+test/test_common/test_request.py,sha256=Ceyds8BNO1O0f1kH1VEb84faJcaupvSjVKIrGdHexsc,11842
 test/test_common/test_result.py,sha256=0WK_Zfw7e6IJl5nTSUUt6LFTvbu6cVynaTOdpuYkBvg,42559
-test/test_common/test_task.py,sha256=jnfF68EgJIu30Pz_4jiJHkncfI-3XpGaut5r79KIXOA,18718
+test/test_common/test_task.py,sha256=P44mNcSe-3tJgDk9ppN3KbM7oN4LBVIuhONG-Gveh74,19007
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.5.1.dev112.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.5.1.dev112.dist-info/METADATA,sha256=IHoYJdSAN35urjB5HBE9eLK_T79vVOT7t3EjsinOjcM,9499
-assemblyline_v4_service-4.5.1.dev112.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-assemblyline_v4_service-4.5.1.dev112.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.5.1.dev112.dist-info/RECORD,,
+assemblyline_v4_service-4.5.1.dev115.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.5.1.dev115.dist-info/METADATA,sha256=2mfPgv9gtIGZAFtoWkSbNK7iM9AHeaJp2uJpY0FEy9E,9499
+assemblyline_v4_service-4.5.1.dev115.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+assemblyline_v4_service-4.5.1.dev115.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.5.1.dev115.dist-info/RECORD,,

test/test_common/test_request.py

@@ -13,6 +13,7 @@ from assemblyline.odm.messages.task import Task as ServiceTask
 # Ensure service manifest is instantiated before importing from OCR submodule
 setup_module()
 
+
 @pytest.fixture
 def service_request():
     st = ServiceTask({
@@ -71,11 +72,12 @@ def test_add_extracted(service_request):
             'classification': 'TLP:C',
             'description': 'description',
             'is_section_image': False,
+            'is_supplementary': False,
             'name': 'name',
             'parent_relation': 'EXTRACTED',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
-         },
+        },
     ]
 
     # Raise MaxExtractedExceeded
@@ -97,17 +99,20 @@ def test_add_extracted(service_request):
             'classification': 'TLP:C',
             'description': 'description',
             'is_section_image': False,
+            'is_supplementary': False,
             'name': 'name',
             'parent_relation': 'DYNAMIC',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
-         },
+        },
     ]
 
 
 @pytest.mark.skipif(len(TESSERACT_LIST) < 1, reason="Requires tesseract-ocr apt package")
 def test_add_image(service_request):
-    image_path = os.path.join(os.path.dirname(__file__), "b32969aa664e3905c20f865cdd7b921f922678f5c3850c78e4c803fbc1757a8e")
+    image_path = os.path.join(
+        os.path.dirname(__file__),
+        "b32969aa664e3905c20f865cdd7b921f922678f5c3850c78e4c803fbc1757a8e")
 
     # Basic
     assert service_request.add_image(image_path, "image_name", "description of image") == {
@@ -131,6 +136,7 @@ def test_add_image(service_request):
             'classification': 'TLP:C',
             'description': 'description of image',
             'is_section_image': True,
+            'is_supplementary': True,
             'name': 'image_name',
             'parent_relation': 'INFORMATION',
             'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329'
@@ -140,6 +146,7 @@ def test_add_image(service_request):
             'classification': 'TLP:C',
             'description': 'description of image (thumbnail)',
             'is_section_image': True,
+            'is_supplementary': True,
             'name': 'image_name.thumb',
             'parent_relation': 'INFORMATION',
             'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901'
@@ -152,7 +159,8 @@ def test_add_image(service_request):
     ocr_heuristic_id = 1
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
-    data = service_request.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io)
+    data = service_request.add_image(image_path, "image_name", "description of image",
+                                     "TLP:A", ocr_heuristic_id, ocr_io)
     assert data["img"] == {
         'description': 'description of image',
         'name': 'image_name',
@@ -188,17 +196,23 @@ def test_add_image(service_request):
 
     heur_dict = get_heuristic_primitives(data["ocr_section"].__dict__["_heuristic"])
 
-    assert heur_dict == {'heur_id': 1, 'score': 1200, 'attack_ids': ['T1005'], 'signatures': {'ransomware_strings': 8}, 'frequency': 0, 'score_map': {}}
+    assert heur_dict == {
+        'heur_id': 1, 'score': 1200, 'attack_ids': ['T1005'],
+        'signatures': {'ransomware_strings': 8},
+        'frequency': 0, 'score_map': {}}
 
     assert service_request.temp_submission_data == {}
 
     service_request.task.supplementary.clear()
 
     # Classification, OCR heuristic, OCR_IO, image with password
-    image_path = os.path.join(os.path.dirname(__file__), "4031ed8786439eee24b87f84901e38038a76b8c55e9d87dd5a7d88df2806c1cf")
+    image_path = os.path.join(
+        os.path.dirname(__file__),
+        "4031ed8786439eee24b87f84901e38038a76b8c55e9d87dd5a7d88df2806c1cf")
     _, path = tempfile.mkstemp()
     ocr_io = open(path, "w")
-    data = service_request.add_image(image_path, "image_name", "description of image", "TLP:A", ocr_heuristic_id, ocr_io)
+    data = service_request.add_image(image_path, "image_name", "description of image",
+                                     "TLP:A", ocr_heuristic_id, ocr_io)
     assert data["img"] == {
         'description': 'description of image',
         'name': 'image_name',
@@ -217,9 +231,12 @@ def test_add_image(service_request):
 
     heur_dict = get_heuristic_primitives(data["ocr_section"].__dict__["_heuristic"])
 
-    assert heur_dict == {'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'], 'signatures': {'password_strings': 1}, 'frequency': 0, 'score_map': {}}
+    assert heur_dict == {
+        'heur_id': 1, 'score': 250, 'attack_ids': ['T1005'],
+        'signatures': {'password_strings': 1},
+        'frequency': 0, 'score_map': {}}
 
-    assert service_request.temp_submission_data == {'passwords':[' 975', '975', 'DOCUMENT', 'PASSWORD', 'PASSWORD:']}
+    assert service_request.temp_submission_data == {'passwords': [' 975', '975', 'DOCUMENT', 'PASSWORD', 'PASSWORD:']}
 
 
 def test_add_supplementary(service_request):
@@ -239,11 +256,12 @@ def test_add_supplementary(service_request):
             'classification': 'TLP:C',
             'description': 'description',
             'is_section_image': False,
+            'is_supplementary': True,
             'name': 'name',
             'parent_relation': 'INFORMATION',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
-         },
+        },
     ]
 
 
@@ -280,7 +298,7 @@ def test_get_param(service_request):
         service_request.get_param("blah")
 
     # Submission parameter exists
-    service_request.task.service_config = {"blah":"blah"}
+    service_request.task.service_config = {"blah": "blah"}
     assert service_request.get_param("blah") == "blah"
 
 

test/test_common/test_task.py

@@ -6,7 +6,7 @@ from logging import Logger
 import pytest
 from assemblyline_v4_service.common import helper
 from assemblyline_v4_service.common.result import ResultSection
-from assemblyline_v4_service.common.task import *
+from assemblyline_v4_service.common.task import Task, Classification, MaxExtractedExceeded, ServiceAPI, Result
 
 from assemblyline.odm.messages.task import DataItem, TagItem
 from assemblyline.odm.messages.task import Task as ServiceTask
@@ -159,6 +159,7 @@ def test_task_add_file(servicetask):
         "classification": "TLP:C",
         "path": path,
         "is_section_image": False,
+        'is_supplementary': False,
         "allow_dynamic_recursion": False,
         "parent_relation": "EXTRACTED",
     }
@@ -173,6 +174,7 @@ def test_task_add_file(servicetask):
         "classification": "TLP:C",
         "path": path,
         "is_section_image": False,
+        'is_supplementary': False,
         "allow_dynamic_recursion": True,
         "parent_relation": "DYNAMIC",
     }
@@ -210,7 +212,7 @@ def test_task_add_extracted(servicetask, mocker):
     # Safelisted file hash
     service_attributes = helper.get_service_attributes()
     sa = ServiceAPI(service_attributes, None)
-    with mocker.patch.object(sa, "lookup_safelist", return_value={"enabled": True, "type": "file"}) as m:
+    with mocker.patch.object(sa, "lookup_safelist", return_value={"enabled": True, "type": "file"}) as _:
         t.safelist_config.enabled = True
         assert t.add_extracted(path, "name", "description", safelist_interface=sa) is False
 
@@ -224,6 +226,7 @@ def test_task_add_extracted(servicetask, mocker):
             "classification": "TLP:C",
             "path": path,
             "is_section_image": False,
+            'is_supplementary': False,
             "allow_dynamic_recursion": False,
             "parent_relation": "EXTRACTED",
         }
@@ -259,6 +262,7 @@ def test_task_add_supplementary(servicetask):
         "classification": "TLP:C",
         "path": path,
         "is_section_image": False,
+        'is_supplementary': True,
         "allow_dynamic_recursion": False,
         "parent_relation": "INFORMATION",
     }
@@ -270,6 +274,7 @@ def test_task_add_supplementary(servicetask):
             "classification": "TLP:C",
             "path": path,
             "is_section_image": False,
+            'is_supplementary': True,
             "allow_dynamic_recursion": False,
             "parent_relation": "INFORMATION",
         }
@@ -405,6 +410,7 @@ def test_task_get_service_result(servicetask):
                     "classification": "TLP:C",
                     "description": "description",
                     "is_section_image": False,
+                    "is_supplementary": False,
                     "name": "name",
                     "parent_relation": "EXTRACTED",
                     "path": path,