assemblyline-v4-service 4.4.1.dev288__py3-none-any.whl → 4.4.1.dev290__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.4.1.dev288
+4.4.1.dev290

assemblyline_v4_service/common/api.py

@@ -2,6 +2,7 @@ import os
 import time
 
 import requests
+from assemblyline_core.badlist_client import BadlistClient
 from assemblyline_core.safelist_client import SafelistClient
 from assemblyline_v4_service.common.utils import DEVELOPMENT_MODE
 
@@ -58,6 +59,51 @@ class ServiceAPI:
                 retries += 1
                 time.sleep(min(2, 2 ** (retries - 7)))
 
+    def lookup_badlist_tags(self, tag_map: dict):
+        if DEVELOPMENT_MODE:
+            return []
+
+        if not isinstance(tag_map, dict) and not all([isinstance(x, list) for x in tag_map.values()]):
+            raise ValueError("Parameter tag_list should be a dictionary tag_type mapping to a list of tag_values.")
+        url = f"{self.service_api_host}/api/v1/badlist/tags/"
+
+        return self._with_retries(self.session.post, url, data=tag_map)
+
+    def lookup_badlist(self, qhash):
+        if DEVELOPMENT_MODE:
+            return None
+        try:
+            return self._with_retries(self.session.get, f"{self.service_api_host}/api/v1/badlist/{qhash}/")
+        except ServiceAPIError as e:
+            if e.status_code == 404:
+                return None
+            else:
+                raise
+
+    def lookup_badlist_ssdeep(self, ssdeep):
+        if DEVELOPMENT_MODE:
+            return []
+        try:
+            data = {"ssdeep": ssdeep}
+            return self._with_retries(self.session.post, f"{self.service_api_host}/api/v1/badlist/ssdeep/", data=data)
+        except ServiceAPIError as e:
+            if e.status_code == 404:
+                return None
+            else:
+                raise
+
+    def lookup_badlist_tlsh(self, tlsh):
+        if DEVELOPMENT_MODE:
+            return []
+        try:
+            data = {"tlsh": tlsh}
+            return self._with_retries(self.session.post, f"{self.service_api_host}/api/v1/badlist/tlsh/", data=data)
+        except ServiceAPIError as e:
+            if e.status_code == 404:
+                return None
+            else:
+                raise
+
     def get_safelist(self, tag_list=None):
         if DEVELOPMENT_MODE:
             return {}
@@ -86,8 +132,33 @@ class ServiceAPI:
 class PrivilegedServiceAPI:
     def __init__(self, logger):
         self.log = logger
+        self.badlist_client = BadlistClient()
         self.safelist_client = SafelistClient()
 
+    def lookup_badlist_tags(self, tag_map):
+        if DEVELOPMENT_MODE:
+            return []
+
+        if not isinstance(tag_map, dict) and not all([isinstance(x, list) for x in tag_map.values()]):
+            raise ValueError("Parameter tag_list should be a dictionary tag_type mapping to a list of tag_values.")
+
+        return self.badlist_client.exists_tags(tag_map)
+
+    def lookup_badlist(self, qhash):
+        if DEVELOPMENT_MODE:
+            return None
+        return self.badlist_client.exists(qhash)
+
+    def lookup_badlist_ssdeep(self, ssdeep):
+        if DEVELOPMENT_MODE:
+            return []
+        return self.badlist_client.find_similar_ssdeep(ssdeep)
+
+    def lookup_badlist_tlsh(self, tlsh):
+        if DEVELOPMENT_MODE:
+            return []
+        return self.badlist_client.find_similar_tlsh(tlsh)
+
     def get_safelist(self, tag_list=None):
         if DEVELOPMENT_MODE:
             return {}

assemblyline_v4_service/common/base.py

@@ -69,7 +69,7 @@ class ServiceBase:
 
         self._working_directory = None
 
-        # Initialize interface for interacting with system safelist
+        # Initialize interface for interacting with system badlist and safelist
         self._api_interface = None
 
         self.dependencies = self._get_dependencies_info()
@@ -261,7 +261,8 @@ class ServiceBase:
             resp = requests.get(url_base + 'status', verify=verify)
             resp.raise_for_status()
             status = resp.json()
-            if self.update_time is not None and self.update_time >= status['local_update_time'] and self.update_hash == status['local_update_hash']:
+            if self.update_time is not None and self.update_time >= status['local_update_time'] and \
+                    self.update_hash == status['local_update_hash']:
                 self.log.info(f"There are no new signatures. ({self.update_time} >= {status['local_update_time']})")
                 return
             if status['download_available']:

assemblyline_v4_service/common/request.py

@@ -287,3 +287,11 @@ class ServiceRequest:
     @property
     def sid(self) -> str:
         return self.task.sid
+
+    @property
+    def ssdeep(self) -> str:
+        return self.task.fileinfo.ssdeep
+
+    @property
+    def tlsh(self) -> str:
+        return self.task.fileinfo.tlsh

assemblyline_v4_service/common/result.py

@@ -424,7 +424,7 @@ class TableSectionBody(SectionBody):
         self._data.append(row)
         self.set_column_order(list(row.keys()))
 
-    def set_column_order(self, order: List[str])-> None:
+    def set_column_order(self, order: List[str]) -> None:
         if not order:
             return
 

assemblyline_v4_service/updater/helper.py

@@ -2,8 +2,6 @@ import os
 import shutil
 import tempfile
 import time
-import traceback
-from io import StringIO
 from logging import Logger
 from shutil import make_archive
 from typing import Any, Dict, List, Optional, Tuple
@@ -48,7 +46,8 @@ def filter_downloads(output_path, pattern, default_pattern=".*") -> List[Tuple[s
 
     f_files = []
     if not pattern:
-        # Regex will either match on the filename, directory, or filepath, either with default or given pattern for source
+        # Regex will either match on the filename, directory, or filepath,
+        # either with default or given pattern for source
         pattern = default_pattern
 
     if os.path.isfile(output_path):

assemblyline_v4_service/updater/updater.py

@@ -12,8 +12,8 @@ import random
 import tarfile
 import threading
 import subprocess
+import hashlib
 from contextlib import contextmanager
-from hashlib import sha256
 from passlib.hash import bcrypt
 from zipfile import ZipFile, BadZipFile
 
@@ -53,7 +53,8 @@ SOURCE_STATUS_KEY = 'status'
 UI_SERVER = os.getenv('UI_SERVER', 'https://nginx')
 UI_SERVER_ROOT_CA = os.environ.get('UI_SERVER_ROOT_CA', '/etc/assemblyline/ssl/al_root-ca.crt')
 UPDATER_DIR = os.getenv('UPDATER_DIR', os.path.join(tempfile.gettempdir(), 'updater'))
-UPDATER_API_ROLES = ['signature_import', 'signature_download', 'signature_view', 'safelist_manage', 'apikey_access', 'signature_manage']
+UPDATER_API_ROLES = ['badlist_manage', 'signature_import', 'signature_download',
+                     'signature_view', 'safelist_manage', 'apikey_access', 'signature_manage']
 STATUS_FILE = '/tmp/status'
 
 classification = forge.get_classification()
@@ -187,7 +188,7 @@ class ServiceUpdater(ThreadedCoreBase):
         return 0
 
     def get_local_update_hash(self) -> str:
-        return sha256(open(self._update_tar, "rb").read()).hexdigest()
+        return hashlib.sha256(open(self._update_tar, "rb").read()).hexdigest()
 
     def status(self):
         return {
@@ -393,7 +394,9 @@ class ServiceUpdater(ThreadedCoreBase):
         username = self.ensure_service_account()
         with temporary_api_key(self.datastore, username) as api_key:
             with tempfile.TemporaryDirectory() as update_dir:
-                al_client = get_client(UI_SERVER, apikey=(username, api_key), verify=self.verify, datastore=self.datastore)
+                al_client = get_client(
+                    UI_SERVER, apikey=(username, api_key),
+                    verify=self.verify, datastore=self.datastore)
                 self.log.info("Connected!")
 
                 # Parse updater configuration
@@ -550,7 +553,8 @@ class ServiceUpdater(ThreadedCoreBase):
         new_tar = ''
 
         # Before serving directory, let's maintain a map of the different signatures and their current deployment state
-        # This map allows the service to be more responsive to changes made locally to the system such as classification changes
+        # This map allows the service to be more responsive to changes made locally to the system such as
+        # classification changes.
         # This also avoids the need to have to insert this kind of metadata into the signature itself
         if self._service.update_config.generates_signatures:
             # Pull signature metadata from the API
@@ -564,7 +568,7 @@ class ServiceUpdater(ThreadedCoreBase):
             signature_map = {
                 source.name: {'classification': source['default_classification'].value}
                 for source in self._service.update_config.sources
-             }
+            }
         open(os.path.join(new_directory, SIGNATURES_META_FILENAME), 'w').write(json.dumps(signature_map, indent=2))
 
         try:

{assemblyline_v4_service-4.4.1.dev288.dist-info → assemblyline_v4_service-4.4.1.dev290.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.4.1.dev288
+Version: 4.4.1.dev290
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.4.1.dev288.dist-info → assemblyline_v4_service-4.4.1.dev290.dist-info}/RECORD

@@ -1,16 +1,16 @@
-assemblyline_v4_service/VERSION,sha256=1wkp3_ZyDBd9jZk_bzjp5mFrUkx5gW8z7JiHWKLRiFA,13
+assemblyline_v4_service/VERSION,sha256=-vORzedMvp3ySVvPpGg60CPBgZgsXvGmwJm2eb5TwX4,13
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
 assemblyline_v4_service/run_privileged_service.py,sha256=9uTfHetXR5G-EDKMDrgfWUOw34yr64-cj6Cm9eZaCbQ,14547
 assemblyline_v4_service/run_service.py,sha256=RCqxdm-OAwJhl15BnKFkuavpQ5k6eTX3ZGeSna5JJBw,5557
 assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/common/api.py,sha256=KPEwOiK1yUlX5GDjZJ-VRXXCCqdHgtnayW1JFdEjpKk,3989
-assemblyline_v4_service/common/base.py,sha256=f0ocyMh59RwAoMbXlFMDgXChQMrFiO78j6CU-DEwz7Y,13576
+assemblyline_v4_service/common/api.py,sha256=dIGU_k0mYZjC545WUwox0wp2GVZa8v1taI80Kofgh4Y,6538
+assemblyline_v4_service/common/base.py,sha256=9xufnspN99J1EHTru1fdkflRwB6PGdfyCUDvYwUIBEk,13610
 assemblyline_v4_service/common/helper.py,sha256=xs9quuf-M1JOdKieBqOmWaOece0CtzXFhhe85xQYmuY,3289
 assemblyline_v4_service/common/ocr.py,sha256=erKJMioiOL53i7qiEq9zve4-FnwW22twILboX19M5eQ,4555
 assemblyline_v4_service/common/ontology_helper.py,sha256=QpwerYoS5hXjWzpx3Pmwv6j2330PQVYqxYGamjcpW3I,7890
-assemblyline_v4_service/common/request.py,sha256=QeRJhJSQGffiQxLuaKtFb0xQGnEtyDtGiawRPxDVGT4,11311
-assemblyline_v4_service/common/result.py,sha256=SNj-qAOF2Lvvws7YM6--6hyBcyG_rFT-Bf62SzZyryY,32345
+assemblyline_v4_service/common/request.py,sha256=Ji_xCptED_-xhSrks06wBxaVUNHZBpp_sY_1E_phYKE,11477
+assemblyline_v4_service/common/result.py,sha256=GiFEFbHlaJorOfCB0XWtFs_jloI8Y5KeiF-g1x2-w98,32346
 assemblyline_v4_service/common/task.py,sha256=erg6-pGi_Avpzj0Fkn6opUTevi6xFgaWK2oT-RaZ7fg,12985
 assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,8 +20,8 @@ assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfve
 assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
 assemblyline_v4_service/updater/client.py,sha256=VKmkiDz3klvMzXiGSQTH7QN-MfHuJnwaIR7BaXy9aZM,5156
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
-assemblyline_v4_service/updater/helper.py,sha256=br-sT6mnhaII50HMBhGNdYwqPXLVFQ_Xmi9yPek7lOs,9457
-assemblyline_v4_service/updater/updater.py,sha256=VB8jt5qg604HZyG6epHqGqG-7yPUCE5-i6uHOO2n72s,32045
+assemblyline_v4_service/updater/helper.py,sha256=iUR2D3BQ8nVuC6hj4EoTBLGL9DiuikFfY0Dc9Ohs--s,9426
+assemblyline_v4_service/updater/updater.py,sha256=UDqkKF4tQQbBF13fhCQzfJcmlNhYS5iI23odbpKpofE,32131
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/test_healthz.py,sha256=oOUjzfcuxNuu5DcdFK6JyhH1XcvU3944lL8UTdlpoF8,133
 test/test_run_privileged_service.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -36,8 +36,8 @@ test/test_common/test_request.py,sha256=CoTIwz4gL2SH-upGrzyuwOC1p86kBVeHImWx6hkz
 test/test_common/test_result.py,sha256=sJHJ4CXHv_FkqBFp1ELV6XsSjUhqKY4Qa1nCoyXI8Os,42088
 test/test_common/test_task.py,sha256=WQ1nZG2ina5BiDHRSWxC4RHJKebZ76t0rL8D6eajpsI,18506
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.4.1.dev288.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.4.1.dev288.dist-info/METADATA,sha256=hmJoehvGacIIa7nFWv33AecZXOQqeOTpDUwoSoYJXXA,9691
-assemblyline_v4_service-4.4.1.dev288.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-assemblyline_v4_service-4.4.1.dev288.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.4.1.dev288.dist-info/RECORD,,
+assemblyline_v4_service-4.4.1.dev290.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.4.1.dev290.dist-info/METADATA,sha256=ta7QZExSW8wvO_eJTrstj8cEBtc6W-oVa3gN-J_ybz0,9691
+assemblyline_v4_service-4.4.1.dev290.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+assemblyline_v4_service-4.4.1.dev290.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.4.1.dev290.dist-info/RECORD,,