assemblyline-v4-service 4.4.1.dev297__py3-none-any.whl → 4.4.1.dev299__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.4.1.dev297
+4.4.1.dev299

assemblyline_v4_service/common/api.py

@@ -32,11 +32,11 @@ class ServiceAPI:
         if self.service_api_host.startswith('https'):
             self.session.verify = os.environ.get('SERVICE_SERVER_ROOT_CA_PATH', '/etc/assemblyline/ssl/al_root-ca.crt')
 
-    def _with_retries(self, func, url):
+    def _with_retries(self, func, url, **kwargs):
         retries = 0
         while True:
             try:
-                resp = func(url)
+                resp = func(url, **kwargs)
                 if resp.ok:
                     return resp.json()['api_response']
                 else:
@@ -60,17 +60,17 @@ class ServiceAPI:
                 time.sleep(min(2, 2 ** (retries - 7)))
 
     def lookup_badlist_tags(self, tag_map: dict):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or not tag_map:
             return []
 
         if not isinstance(tag_map, dict) and not all([isinstance(x, list) for x in tag_map.values()]):
             raise ValueError("Parameter tag_list should be a dictionary tag_type mapping to a list of tag_values.")
         url = f"{self.service_api_host}/api/v1/badlist/tags/"
 
-        return self._with_retries(self.session.post, url, data=tag_map)
+        return self._with_retries(self.session.post, url, json=tag_map)
 
     def lookup_badlist(self, qhash):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or qhash is None:
             return None
         try:
             return self._with_retries(self.session.get, f"{self.service_api_host}/api/v1/badlist/{qhash}/")
@@ -81,26 +81,26 @@ class ServiceAPI:
                 raise
 
     def lookup_badlist_ssdeep(self, ssdeep):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or ssdeep is None:
             return []
         try:
             data = {"ssdeep": ssdeep}
-            return self._with_retries(self.session.post, f"{self.service_api_host}/api/v1/badlist/ssdeep/", data=data)
+            return self._with_retries(self.session.post, f"{self.service_api_host}/api/v1/badlist/ssdeep/", json=data)
         except ServiceAPIError as e:
             if e.status_code == 404:
-                return None
+                return []
             else:
                 raise
 
     def lookup_badlist_tlsh(self, tlsh):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or tlsh is None:
             return []
         try:
             data = {"tlsh": tlsh}
-            return self._with_retries(self.session.post, f"{self.service_api_host}/api/v1/badlist/tlsh/", data=data)
+            return self._with_retries(self.session.post, f"{self.service_api_host}/api/v1/badlist/tlsh/", json=data)
         except ServiceAPIError as e:
             if e.status_code == 404:
-                return None
+                return []
             else:
                 raise
 
@@ -136,7 +136,7 @@ class PrivilegedServiceAPI:
         self.safelist_client = SafelistClient()
 
     def lookup_badlist_tags(self, tag_map):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or not tag_map:
             return []
 
         if not isinstance(tag_map, dict) and not all([isinstance(x, list) for x in tag_map.values()]):
@@ -145,17 +145,17 @@ class PrivilegedServiceAPI:
         return self.badlist_client.exists_tags(tag_map)
 
     def lookup_badlist(self, qhash):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or qhash is None:
             return None
         return self.badlist_client.exists(qhash)
 
     def lookup_badlist_ssdeep(self, ssdeep):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or ssdeep is None:
             return []
         return self.badlist_client.find_similar_ssdeep(ssdeep)
 
     def lookup_badlist_tlsh(self, tlsh):
-        if DEVELOPMENT_MODE:
+        if DEVELOPMENT_MODE or tlsh is None:
             return []
         return self.badlist_client.find_similar_tlsh(tlsh)
 

assemblyline_v4_service/common/request.py

@@ -1,19 +1,19 @@
-import logging
 import hashlib
+import logging
 import tempfile
 from typing import Any, Dict, List, Optional, TextIO, Union
 
-from assemblyline.common import forge
-from assemblyline.common import log as al_log
-from assemblyline.common.file import make_uri_file
-from assemblyline.common.classification import Classification
-from PIL import Image
-
 from assemblyline_v4_service.common.api import PrivilegedServiceAPI, ServiceAPI
 from assemblyline_v4_service.common.ocr import ocr_detections
 from assemblyline_v4_service.common.result import Heuristic, Result, ResultKeyValueSection
-from assemblyline_v4_service.common.task import MaxExtractedExceeded, Task
+from assemblyline_v4_service.common.task import PARENT_RELATION, MaxExtractedExceeded, Task
 from assemblyline_v4_service.common.utils import extract_passwords
+from PIL import Image
+
+from assemblyline.common import forge
+from assemblyline.common import log as al_log
+from assemblyline.common.classification import Classification
+from assemblyline.common.file import make_uri_file
 
 CLASSIFICATION = forge.get_classification()
 WEBP_MAX_SIZE = 16383
@@ -32,7 +32,7 @@ class ServiceRequest:
     def add_extracted(self, path: str, name: str, description: str,
                       classification: Optional[Classification] = None,
                       safelist_interface: Optional[Union[ServiceAPI, PrivilegedServiceAPI]] = None,
-                      allow_dynamic_recursion: bool = False, parent_relation: str = 'EXTRACTED') -> bool:
+                      allow_dynamic_recursion: bool = False, parent_relation: str = PARENT_RELATION.EXTRACTED) -> bool:
         """
         Add an extracted file for additional processing.
 
@@ -56,7 +56,7 @@ class ServiceRequest:
 
     def add_extracted_uri(self, description: str, uri: str, params=None,
                           classification: Optional[Classification] = None, allow_dynamic_recursion: bool = False,
-                          parent_relation: str = 'EXTRACTED') -> bool:
+                          parent_relation: str = PARENT_RELATION.EXTRACTED) -> bool:
         if params:
             self.set_uri_metadata(uri, params)
         filepath = make_uri_file(self._working_directory, uri, params)
@@ -144,8 +144,11 @@ class ServiceRequest:
 
         return data
 
-    def add_supplementary(self, path: str, name: str, description: str,
-                          classification: Optional[Classification] = None) -> bool:
+    def add_supplementary(
+            self, path: str, name: str, description: str,
+            classification: Optional[Classification] = None,
+            parent_relation: str = PARENT_RELATION.INFORMATION
+        ) -> bool:
         """
         Add a supplementary file.
 
@@ -153,10 +156,13 @@ class ServiceRequest:
         :param name: Display name of the supplementary file
         :param description: Descriptive text about the supplementary file
         :param classification: Classification of the supplementary file (default: service classification)
+        :param parent_relation: File relation to parent, if any.
         :return: None
         """
 
-        return self.task.add_supplementary(path, name, description, classification)
+        return self.task.add_supplementary(
+            path, name, description, classification, parent_relation=parent_relation
+        )
 
     def drop(self) -> None:
         """

assemblyline_v4_service/common/task.py

@@ -4,15 +4,26 @@ import os
 import tempfile
 from typing import Any, Dict, List, Optional, Union
 
+from assemblyline_v4_service.common.api import PrivilegedServiceAPI, ServiceAPI
+from assemblyline_v4_service.common.helper import get_service_manifest
+from assemblyline_v4_service.common.result import Result
+
 from assemblyline.common import forge
 from assemblyline.common import log as al_log
 from assemblyline.common.classification import Classification
 from assemblyline.common.digests import get_digests_for_file, get_sha256_for_file
 from assemblyline.common.isotime import now_as_iso
+from assemblyline.common.str_utils import StringTable
 from assemblyline.odm.messages.task import Task as ServiceTask
-from assemblyline_v4_service.common.api import PrivilegedServiceAPI, ServiceAPI
-from assemblyline_v4_service.common.helper import get_service_manifest
-from assemblyline_v4_service.common.result import Result
+
+PARENT_RELATION = StringTable('PARENT_RELATION', [
+    ('ROOT', 0),
+    ('EXTRACTED', 1),
+    ('INFORMATION', 2),
+    ('DYNAMIC', 3),
+    ('MEMDUMP', 4),
+    ('DOWNLOADED', 5),
+])
 
 
 class MaxExtractedExceeded(Exception):
@@ -78,13 +89,18 @@ class Task:
                   classification: Optional[Classification] = None,
                   is_section_image: bool = False,
                   allow_dynamic_recursion: bool = False,
-                  parent_relation: str = "EXTRACTED") -> Optional[Dict[str, str]]:
+                  parent_relation: str = PARENT_RELATION.EXTRACTED) -> Optional[Dict[str, str]]:
         # Reject empty files
         if os.path.getsize(path) == 0:
             self.log.info(f"Adding empty extracted or supplementary files is not allowed. "
                           f"Empty file ({name}) was ignored.")
             return
 
+        if parent_relation not in PARENT_RELATION.keys():
+            raise ValueError(
+                f"An invalid 'parent_relation' was provided: '{parent_relation}'. Possible values are: '{PARENT_RELATION.keys()}'"
+            )
+
         # If file classification not provided, then use the default result classification
         if not classification:
             classification = self.service_default_result_classification
@@ -105,7 +121,7 @@ class Task:
     def add_extracted(self, path: str, name: str, description: str,
                       classification: Optional[Classification] = None,
                       safelist_interface: Optional[Union[ServiceAPI, PrivilegedServiceAPI]] = None,
-                      allow_dynamic_recursion: bool = False, parent_relation: str = 'EXTRACTED') -> bool:
+                      allow_dynamic_recursion: bool = False, parent_relation: str = PARENT_RELATION.EXTRACTED) -> bool:
 
         # Service-based safelisting of files has to be configured at the global configuration
         # Allows the administrator to be selective about the types of hashes to lookup in the safelist
@@ -142,9 +158,12 @@ class Task:
         self.extracted.append(file)
         return True
 
-    def add_supplementary(self, path: str, name: str, description: str,
-                          classification: Optional[Classification] = None,
-                          is_section_image: bool = False) -> Optional[dict]:
+    def add_supplementary(
+        self, path: str, name: str, description: str,
+        classification: Optional[Classification] = None,
+        is_section_image: bool = False,
+        parent_relation: str = PARENT_RELATION.INFORMATION
+    ) -> Optional[dict]:
         if not path:
             raise ValueError("Path cannot be empty")
 
@@ -154,7 +173,9 @@ class Task:
         if not description:
             raise ValueError("Description cannot be empty")
 
-        file = self._add_file(path, name, description, classification, is_section_image)
+        file = self._add_file(
+            path, name, description, classification, is_section_image, parent_relation=parent_relation
+        )
 
         if not file:
             return None

{assemblyline_v4_service-4.4.1.dev297.dist-info → assemblyline_v4_service-4.4.1.dev299.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.4.1.dev297
+Version: 4.4.1.dev299
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.4.1.dev297.dist-info → assemblyline_v4_service-4.4.1.dev299.dist-info}/RECORD

@@ -1,17 +1,17 @@
-assemblyline_v4_service/VERSION,sha256=dhJU3wqjXFTelDUtk4QwQVyJA5-9ey0tOIkFEFQEuic,13
+assemblyline_v4_service/VERSION,sha256=3bOsB5czUFhct5ojTZCT36XAMaeukmjFJMwYSQarekE,13
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
 assemblyline_v4_service/run_privileged_service.py,sha256=9uTfHetXR5G-EDKMDrgfWUOw34yr64-cj6Cm9eZaCbQ,14547
 assemblyline_v4_service/run_service.py,sha256=RCqxdm-OAwJhl15BnKFkuavpQ5k6eTX3ZGeSna5JJBw,5557
 assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/common/api.py,sha256=dIGU_k0mYZjC545WUwox0wp2GVZa8v1taI80Kofgh4Y,6538
+assemblyline_v4_service/common/api.py,sha256=n76hLmuAZ8E8K0U_uLe7eKXHzaHUu0shaeOVBZ-XNuY,6686
 assemblyline_v4_service/common/base.py,sha256=9xufnspN99J1EHTru1fdkflRwB6PGdfyCUDvYwUIBEk,13610
 assemblyline_v4_service/common/helper.py,sha256=xs9quuf-M1JOdKieBqOmWaOece0CtzXFhhe85xQYmuY,3289
 assemblyline_v4_service/common/ocr.py,sha256=erKJMioiOL53i7qiEq9zve4-FnwW22twILboX19M5eQ,4555
 assemblyline_v4_service/common/ontology_helper.py,sha256=QpwerYoS5hXjWzpx3Pmwv6j2330PQVYqxYGamjcpW3I,7890
-assemblyline_v4_service/common/request.py,sha256=Ji_xCptED_-xhSrks06wBxaVUNHZBpp_sY_1E_phYKE,11477
+assemblyline_v4_service/common/request.py,sha256=NxtWxp8-ttC72i3Vnchc3fZTRKqPQAoMC4KAVwEj8-4,11714
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
-assemblyline_v4_service/common/task.py,sha256=erg6-pGi_Avpzj0Fkn6opUTevi6xFgaWK2oT-RaZ7fg,12985
+assemblyline_v4_service/common/task.py,sha256=7yNvXk5B27QookJNp4-Y1iGhV8LfhLDaYWzF-uFGWgo,13576
 assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/dev/run_service_once.py,sha256=4K3ljw0MnfPGw0-6lzc_vtUYg1EbntJbbsWvNU-ZM_A,10456
@@ -23,21 +23,21 @@ assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAt
 assemblyline_v4_service/updater/helper.py,sha256=iUR2D3BQ8nVuC6hj4EoTBLGL9DiuikFfY0Dc9Ohs--s,9426
 assemblyline_v4_service/updater/updater.py,sha256=UDqkKF4tQQbBF13fhCQzfJcmlNhYS5iI23odbpKpofE,32131
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/test_healthz.py,sha256=oOUjzfcuxNuu5DcdFK6JyhH1XcvU3944lL8UTdlpoF8,133
+test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
 test/test_run_privileged_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
-test/test_run_service.py,sha256=t4s_HdD7MfP2QXEKFOf6XY0JzbcWY08r_QQtqaMJIIg,54
+test/test_run_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
 test/test_common/__init__.py,sha256=uWT8xNaiOOpVznGoUPhQu_mem9ha5l8OeVhiZ_1IP2M,1209
 test/test_common/test_api.py,sha256=rh0-0CwqLtpRfKlcUFT-T2xgmBe7nP9fCZ0jpzufKek,4666
 test/test_common/test_base.py,sha256=3B35CTxumj1jI-Yap17aWuoruWp0DYwUwnPl-hxrpkA,13143
 test/test_common/test_helper.py,sha256=4r5S4W7EXUx4ldq7LZny0j3oohpnskya9pjMhrWBHD4,2388
 test/test_common/test_ocr.py,sha256=NGYI81WDvP1Ou3iU65lJwzYa37_EONYG0oj_LqKvpdk,1816
 test/test_common/test_ontology_helper.py,sha256=snkj61_iPzU0cE1dzJounZikkG2LY-GQrDgyqYqA5Hs,7784
-test/test_common/test_request.py,sha256=CoTIwz4gL2SH-upGrzyuwOC1p86kBVeHImWx6hkzGoA,11665
+test/test_common/test_request.py,sha256=q_-5_507mdb1dz1_CUqwit1_2CinxHaqAKOO-qSLFLE,11671
 test/test_common/test_result.py,sha256=sJHJ4CXHv_FkqBFp1ELV6XsSjUhqKY4Qa1nCoyXI8Os,42088
-test/test_common/test_task.py,sha256=WQ1nZG2ina5BiDHRSWxC4RHJKebZ76t0rL8D6eajpsI,18506
+test/test_common/test_task.py,sha256=RWsGEN0L-xKoRAsGv2x4JZcSAySS9aBF4dwL9t7tepo,18668
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.4.1.dev297.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.4.1.dev297.dist-info/METADATA,sha256=3ryUKFIYS2_q1ugpVS3oNh3f0sWaJe8a-hgQ-ZBhqE4,9691
-assemblyline_v4_service-4.4.1.dev297.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-assemblyline_v4_service-4.4.1.dev297.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.4.1.dev297.dist-info/RECORD,,
+assemblyline_v4_service-4.4.1.dev299.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.4.1.dev299.dist-info/METADATA,sha256=kdWQ5OrP8ZTp4Bkfd3xDrFRfZCLuecB-Pfby7ryNV9I,9691
+assemblyline_v4_service-4.4.1.dev299.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+assemblyline_v4_service-4.4.1.dev299.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.4.1.dev299.dist-info/RECORD,,

test/test_common/test_request.py

@@ -133,7 +133,7 @@ def test_add_image(service_request):
             'description': 'description of image',
             'is_section_image': True,
             'name': 'image_name',
-            'parent_relation': 'EXTRACTED',
+            'parent_relation': 'INFORMATION',
             'sha256': '09bf99ab5431af13b701a06dc2b04520aea9fd346584fa2a034d6d4af0c57329'
         },
         {
@@ -142,7 +142,7 @@ def test_add_image(service_request):
             'description': 'description of image (thumbnail)',
             'is_section_image': True,
             'name': 'image_name.thumb',
-            'parent_relation': 'EXTRACTED',
+            'parent_relation': 'INFORMATION',
             'sha256': '1af0e0d99845493b64cf402b3704170f17ecf15001714016e48f9d4854218901'
         },
     ]
@@ -244,7 +244,7 @@ def test_add_supplementary(service_request):
             'description': 'description',
             'is_section_image': False,
             'name': 'name',
-            'parent_relation': 'EXTRACTED',
+            'parent_relation': 'INFORMATION',
             'path': path,
             'sha256': '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'
          },

test/test_common/test_task.py

@@ -148,6 +148,10 @@ def test_task_add_file(servicetask):
     with open(path, "w") as f:
         f.write("test")
 
+    # Incorrect parent_relation
+    with pytest.raises(ValueError):
+        t._add_file(path, "name", "description", parent_relation="BLAHBLAHBLAH") is None
+
     # File is not empty
     assert t._add_file(path, "name", "description") == {
         "name": "name",
@@ -257,7 +261,7 @@ def test_task_add_supplementary(servicetask):
         "path": path,
         "is_section_image": False,
         "allow_dynamic_recursion": False,
-        "parent_relation": "EXTRACTED",
+        "parent_relation": "INFORMATION",
     }
     assert t.supplementary == [
         {
@@ -268,7 +272,7 @@ def test_task_add_supplementary(servicetask):
             "path": path,
             "is_section_image": False,
             "allow_dynamic_recursion": False,
-            "parent_relation": "EXTRACTED",
+            "parent_relation": "INFORMATION",
         }
     ]
 

test/test_healthz.py

@@ -1,6 +1 @@
-from assemblyline_v4_service.healthz import *
-
-
-def test_perform_check():
-    # TODO: Requires overriding environ variables
-    pass
+# TODO

test/test_run_service.py

@@ -1,3 +1 @@
-from assemblyline_v4_service.healthz import *
-
 # TODO