assemblyline-v4-service 4.4.0.9__py3-none-any.whl → 4.4.0.14__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.4.0.9
+4.4.0.14

assemblyline_v4_service/common/base.py

@@ -29,6 +29,7 @@ LOG_LEVEL = logging.getLevelName(os.environ.get("LOG_LEVEL", "INFO"))
 UPDATES_DIR = os.environ.get('UPDATES_DIR', '/updates')
 UPDATES_CA = os.environ.get('UPDATES_CA', '/etc/assemblyline/ssl/al_root-ca.crt')
 PRIVILEGED = os.environ.get('PRIVILEGED', 'false') == 'true'
+MIN_SECONDS_BETWEEN_UPDATES = float(os.environ.get('MIN_SECONDS_BETWEEN_UPDATES', '10.0'))
 
 RECOVERABLE_RE_MSG = [
     "cannot schedule new futures after interpreter shutdown",
@@ -77,6 +78,7 @@ class ServiceBase:
         self.rules_directory: str = None
         self.rules_list: list = []
         self.update_time: int = None
+        self.update_check_time: float = 0.0
         self.rules_hash: str = None
 
     @property
@@ -236,6 +238,12 @@ class ServiceBase:
 
     # Only relevant for services using updaters (reserving 'updates' as the defacto container name)
     def _download_rules(self):
+        # check if we just tried to download rules to reduce traffic
+        if time.time() - self.update_check_time < MIN_SECONDS_BETWEEN_UPDATES:
+            return
+        self.update_check_time = time.time()
+
+        # Resolve the update target
         scheme, verify = 'http', None
         if os.path.exists(UPDATES_CA):
             scheme, verify = 'https', UPDATES_CA

assemblyline_v4_service/common/dynamic_service_helper.py

@@ -2,46 +2,29 @@ from datetime import datetime
 from hashlib import sha256
 from json import dumps
 from logging import getLogger
-from re import compile, escape, sub, findall, match as re_match
+from re import compile, escape, findall
+from re import match as re_match
+from re import sub
 from typing import Any, Dict, List, Optional, Union
 from urllib.parse import urlparse
 from uuid import UUID, uuid4
 
 from assemblyline.common import log as al_log
-from assemblyline.common.attack_map import (
-    attack_map,
-    software_map,
-    group_map,
-    revoke_map,
-)
+from assemblyline.common.attack_map import attack_map, group_map, revoke_map, software_map
 from assemblyline.common.digests import get_sha256_for_file
-from assemblyline.common.isotime import (
-    epoch_to_local,
-    LOCAL_FMT,
-    local_to_epoch,
-    MAX_TIME,
-    MIN_TIME,
-    format_time,
-)
+from assemblyline.common.isotime import LOCAL_FMT, MAX_TIME, MIN_TIME, epoch_to_local, format_time, local_to_epoch
 from assemblyline.common.uid import get_random_id
-from assemblyline.odm.base import DOMAIN_REGEX, IP_REGEX, IPV4_REGEX, FULL_URI, URI_PATH
-from assemblyline.odm.models.ontology.results import (
-    Process as ProcessModel, Sandbox as SandboxModel,
-    Signature as SignatureModel,
-    NetworkConnection as NetworkConnectionModel
-)
-
+from assemblyline.odm.base import DOMAIN_REGEX, FULL_URI, IP_REGEX, IPV4_REGEX, URI_PATH
+from assemblyline.odm.models.ontology.results import NetworkConnection as NetworkConnectionModel
+from assemblyline.odm.models.ontology.results import Process as ProcessModel
+from assemblyline.odm.models.ontology.results import Sandbox as SandboxModel
+from assemblyline.odm.models.ontology.results import Signature as SignatureModel
 # from assemblyline_v4_service.common.balbuzard.patterns import PatternMatch
 from assemblyline_v4_service.common.base import ServiceBase
 from assemblyline_v4_service.common.request import ServiceRequest
-from assemblyline_v4_service.common.result import (
-    ResultSection,
-    ProcessItem,
-    ResultProcessTreeSection,
-    ResultTableSection,
-    TableRow,
-)
-from assemblyline_v4_service.common.safelist_helper import is_tag_safelisted, URL_REGEX
+from assemblyline_v4_service.common.result import (ProcessItem, ResultProcessTreeSection, ResultSection,
+                                                   ResultTableSection, TableRow)
+from assemblyline_v4_service.common.safelist_helper import URL_REGEX, is_tag_safelisted
 from assemblyline_v4_service.common.tag_helper import add_tag
 from assemblyline_v4_service.common.task import MaxExtractedExceeded
 
@@ -1875,13 +1858,17 @@ class OntologyResults:
         else:
             return network_http_with_details[0]
 
-    def get_network_connection_by_network_http(self, network_http: NetworkHTTP) -> Optional[NetworkHTTP]:
+    def get_network_connection_by_network_http(self, network_http: NetworkHTTP) -> Optional[NetworkConnection]:
         """
         This method returns the network connection corresponding to the given network http object
         :param network_http: The given network http object
         :return: The corresponding network connection
         """
-        return next((netflow for netflow in self.netflows if netflow.http_details == network_http), None)
+        for netflow in self.netflows:
+            if netflow.http_details == network_http:
+                return netflow
+
+        return None
 
     # Process manipulation methods
     def set_processes(self, processes: List[Process]) -> None:

assemblyline_v4_service/common/result.py

@@ -14,6 +14,9 @@ from assemblyline_v4_service.common.helper import get_service_attributes, get_he
 if TYPE_CHECKING:  # Avoid circular dependency
     from assemblyline_v4_service.common.request import ServiceRequest
 
+# Type of values in KV sections
+KV_VALUE_TYPE = Union[str, bool, int]
+
 al_log.init_logging('service.result')
 log = logging.getLogger('assemblyline.service.result')
 
@@ -211,7 +214,7 @@ class SectionBody:
         return self._format
 
     @property
-    def body(self):
+    def body(self) -> str | None:
         if not self._data:
             return None
         elif not isinstance(self._data, str):
@@ -278,21 +281,23 @@ class GraphSectionBody(SectionBody):
 
 
 class KVSectionBody(SectionBody):
-    def __init__(self, **kwargs) -> None:
+    def __init__(self, **kwargs: KV_VALUE_TYPE) -> None:
+        self._data: dict[str, KV_VALUE_TYPE]
         super().__init__(BODY_FORMAT.KEY_VALUE, body=kwargs)
 
-    def set_item(self, key: str, value: Union[str, bool, int]) -> None:
+    def set_item(self, key: str, value: KV_VALUE_TYPE) -> None:
         self._data[str(key)] = value
 
-    def update_items(self, new_dict: dict):
+    def update_items(self, new_dict: dict[str, KV_VALUE_TYPE]):
         self._data.update({str(k): v for k, v in new_dict.items()})
 
 
 class OrderedKVSectionBody(SectionBody):
-    def __init__(self) -> None:
-        super().__init__(BODY_FORMAT.ORDERED_KEY_VALUE, body=[])
+    def __init__(self, **kwargs: KV_VALUE_TYPE) -> None:
+        self._data: list[tuple[str, KV_VALUE_TYPE]]
+        super().__init__(BODY_FORMAT.ORDERED_KEY_VALUE, body=[(str(key), value) for key, value in kwargs.items()])
 
-    def add_item(self, key: str, value: Union[str, bool, int]) -> None:
+    def add_item(self, key: str, value: KV_VALUE_TYPE) -> None:
         self._data.append((str(key), value))
 
 
@@ -650,6 +655,7 @@ class ResultMemoryDumpSection(ResultSection):
 
 class ResultGraphSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List],  **kwargs):
+        self.section_body: GraphSectionBody
         super().__init__(title_text, GraphSectionBody(), **kwargs)
 
     def set_colormap(self, cmap_min: int, cmap_max: int, values: List[int]) -> None:
@@ -658,6 +664,7 @@ class ResultGraphSection(TypeSpecificResultSection):
 
 class ResultURLSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
+        self.section_body: URLSectionBody
         super().__init__(title_text, URLSectionBody(), **kwargs)
 
     def add_url(self, url: str, name: Optional[str] = None) -> None:
@@ -665,8 +672,9 @@ class ResultURLSection(TypeSpecificResultSection):
 
 
 class ResultKeyValueSection(TypeSpecificResultSection):
-    def __init__(self, title_text: Union[str, List], **kwargs):
-        super().__init__(title_text, KVSectionBody(), **kwargs)
+    def __init__(self, title_text: Union[str, List], body: dict[str, KV_VALUE_TYPE] | None = None, **kwargs):
+        self.section_body: KVSectionBody
+        super().__init__(title_text, KVSectionBody(**(body if body else {})), **kwargs)
 
     def set_item(self, key: str, value: Union[str, bool, int]) -> None:
         self.section_body.set_item(key, value)
@@ -676,8 +684,9 @@ class ResultKeyValueSection(TypeSpecificResultSection):
 
 
 class ResultOrderedKeyValueSection(TypeSpecificResultSection):
-    def __init__(self, title_text: Union[str, List], **kwargs):
-        super().__init__(title_text, OrderedKVSectionBody(), **kwargs)
+    def __init__(self, title_text: Union[str, List], body: dict[str, KV_VALUE_TYPE] | None = None, **kwargs):
+        self.section_body: OrderedKVSectionBody
+        super().__init__(title_text, OrderedKVSectionBody(**(body if body else {})), **kwargs)
 
     def add_item(self, key: str, value: Union[str, bool, int]) -> None:
         self.section_body.add_item(key, value)
@@ -685,6 +694,7 @@ class ResultOrderedKeyValueSection(TypeSpecificResultSection):
 
 class ResultJSONSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
+        self.section_body: JSONSectionBody
         super().__init__(title_text, JSONSectionBody(), **kwargs)
 
     def set_json(self, json_body: dict) -> None:
@@ -696,6 +706,7 @@ class ResultJSONSection(TypeSpecificResultSection):
 
 class ResultProcessTreeSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
+        self.section_body: ProcessTreeSectionBody
         super().__init__(title_text, ProcessTreeSectionBody(), **kwargs)
 
     def add_process(self, process: ProcessItem) -> None:
@@ -704,6 +715,7 @@ class ResultProcessTreeSection(TypeSpecificResultSection):
 
 class ResultTableSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
+        self.section_body: TableSectionBody
         super().__init__(title_text, TableSectionBody(), **kwargs)
 
     def add_row(self, row: TableRow) -> None:
@@ -731,6 +743,7 @@ class ResultImageSection(TypeSpecificResultSection):
 
 class ResultTimelineSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
+        self.section_body: TimelineSectionBody
         super().__init__(title_text,  TimelineSectionBody(), **kwargs)
 
     def add_node(self, title: str, content: str, opposite_content: str, icon: str = None, signatures: List[str] = [],
@@ -741,6 +754,7 @@ class ResultTimelineSection(TypeSpecificResultSection):
 
 class ResultMultiSection(TypeSpecificResultSection):
     def __init__(self, title_text: Union[str, List], **kwargs):
+        self.section_body: MultiSectionBody
         super().__init__(title_text,  MultiSectionBody(), **kwargs)
 
     def add_section_part(self, section_part: SectionBody) -> bool:

assemblyline_v4_service/updater/updater.py

@@ -222,17 +222,20 @@ class ServiceUpdater(ThreadedCoreBase):
             return 0
         return hash(json.dumps(service.update_config.as_primitives()))
 
-    def _handle_source_update_event(self, data: list[str]):
-        # Received an event regarding a change to source
-        self.log.info(f'Triggered to update the following: {data}')
-        self.do_source_update(self._service, specific_sources=data)
-        self.local_update_flag.set()
+    def _handle_source_update_event(self, data: Optional[list[str]]):
+        if data is not None:
+            # Received an event regarding a change to source
+            self.log.info(f'Triggered to update the following: {data}')
+            self.do_source_update(self._service, specific_sources=data)
+            self.local_update_flag.set()
+        else:
+            self.source_update_flag.set()
 
-    def _handle_signature_change_event(self, data: SignatureChange):
+    def _handle_signature_change_event(self, data: Optional[SignatureChange]):
         self.local_update_flag.set()
 
-    def _handle_service_change_event(self, data: ServiceChange):
-        if data.operation == Operation.Modified:
+    def _handle_service_change_event(self, data: Optional[ServiceChange]):
+        if data is None or data.operation == Operation.Modified:
             self._pull_settings()
 
     def _sync_settings(self):

{assemblyline_v4_service-4.4.0.9.dist-info → assemblyline_v4_service-4.4.0.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.4.0.9
+Version: 4.4.0.14
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.4.0.9.dist-info → assemblyline_v4_service-4.4.0.14.dist-info}/RECORD

@@ -1,18 +1,18 @@
-assemblyline_v4_service/VERSION,sha256=pR-jPwo1vPSCpW8ZFA987RgtXDPQFXFlqbcSMtqBQEc,8
+assemblyline_v4_service/VERSION,sha256=A9KFeWrS1AwZZBnk60nZgClg6j-A0_X4-XyN5wWDKPE,9
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
 assemblyline_v4_service/run_privileged_service.py,sha256=9uTfHetXR5G-EDKMDrgfWUOw34yr64-cj6Cm9eZaCbQ,14547
 assemblyline_v4_service/run_service.py,sha256=RCqxdm-OAwJhl15BnKFkuavpQ5k6eTX3ZGeSna5JJBw,5557
 assemblyline_v4_service/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/common/api.py,sha256=U908p3wlW9fEydx77GgI2E-6wW6T8Nc3R91nNOKU0H0,4453
-assemblyline_v4_service/common/base.py,sha256=3wjhmd-NltaRpMMF9Sh3BWJJCfPsw-JK-34Fc1Vn-5k,12655
-assemblyline_v4_service/common/dynamic_service_helper.py,sha256=_V0v5Q7j4woAgTs1zXUl65T9CBSomnmK_F2fTVEGCMU,147405
+assemblyline_v4_service/common/base.py,sha256=STzfZ9dwqvbgbKiFs-aLk05pdhyK6Psz4hZ3_fOmQYM,13039
+assemblyline_v4_service/common/dynamic_service_helper.py,sha256=xIuuJLtf9Wz_mXdoquzCL7TmHtH_trH4a1PfRL9A-GM,147593
 assemblyline_v4_service/common/helper.py,sha256=Fgimk8DhnS23aijTGewA1HwvPoAM61UUbHlrGBnSzL0,3290
 assemblyline_v4_service/common/icap.py,sha256=phT3CT5uII3Qm90Nzi4O-eDkQ2jmr3zHcVVra4sqYSc,5376
 assemblyline_v4_service/common/keytool_parse.py,sha256=e829hrNNG5LFw1kjLsYVZsafCm2S3NpgM6jBc6JKawY,2219
 assemblyline_v4_service/common/ontology_helper.py,sha256=HJdFvZCP6OPNGySjjmXa-fLqTwaa9V-lnhCSv-isztQ,7768
 assemblyline_v4_service/common/request.py,sha256=p8A9boDZ6KuVxl3EdhvaU1D_5K6_gAVoIbJYDz8TzjA,9711
-assemblyline_v4_service/common/result.py,sha256=TivKKDgh-6xhGbyBTAcjU8HFVRyXzTI19vFMa9PA3AQ,28908
+assemblyline_v4_service/common/result.py,sha256=LEzZq0YjEoHZvhVZmlMd5AqAhsHE6mKqotX1mdmbTbg,29727
 assemblyline_v4_service/common/safelist_helper.py,sha256=QHTuG8q52o3U307AADPgrIgug7aYFK2uQE4-EtWG3yQ,3037
 assemblyline_v4_service/common/section_reducer.py,sha256=JJOT7eFfBn4hFJKHY9UeVEbHS-E8FpmQ_dPZC-dWla0,1513
 assemblyline_v4_service/common/tag_helper.py,sha256=om3TVPY_XDeFDqVW2iUA349xbljSAy5tv667jCiA7JI,4186
@@ -45,9 +45,9 @@ assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfve
 assemblyline_v4_service/updater/app.py,sha256=Ass5DZtOCr0tdoRbLo7Qn8Ujlw8T8mUDroAaHxx2oMo,3198
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
 assemblyline_v4_service/updater/helper.py,sha256=JD0gX3KHY-wvsFjTbWkT83F0d5Up3OfubMPinuNzbTQ,9069
-assemblyline_v4_service/updater/updater.py,sha256=1AYBERGFoIEEXkgXLwozWsLuucBw7ei9FHinkSBq1t0,29016
-assemblyline_v4_service-4.4.0.9.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.4.0.9.dist-info/METADATA,sha256=0-4bMl9VDjZLjLSSx7KRceA3ZXX6biOy9JFICJU2U9k,9327
-assemblyline_v4_service-4.4.0.9.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-assemblyline_v4_service-4.4.0.9.dist-info/top_level.txt,sha256=Ut5IqePObcxlJ8rv2--dOAzYbxzqlllfiV_51cbqjbA,24
-assemblyline_v4_service-4.4.0.9.dist-info/RECORD,,
+assemblyline_v4_service/updater/updater.py,sha256=7Ep-qeQLuKffnrvOwi9Gb8eFyDxBtLUbFNkUlRpu2Fc,29163
+assemblyline_v4_service-4.4.0.14.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.4.0.14.dist-info/METADATA,sha256=0j_Omnpx1W995VjMeUoszqHLPfdl-CuAjDXJFgLW7nI,9328
+assemblyline_v4_service-4.4.0.14.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+assemblyline_v4_service-4.4.0.14.dist-info/top_level.txt,sha256=Ut5IqePObcxlJ8rv2--dOAzYbxzqlllfiV_51cbqjbA,24
+assemblyline_v4_service-4.4.0.14.dist-info/RECORD,,