assemblyline-v4-service 4.5.0.64__py3-none-any.whl → 4.5.0.66__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.5.0.64
+4.5.0.66

assemblyline_v4_service/common/request.py

@@ -202,6 +202,9 @@ class ServiceRequest:
         """
         return self.task.get_param(name)
 
+    def partial(self) -> None:
+        self.task.partial()
+
     @property
     def result(self) -> Result:
         """

assemblyline_v4_service/common/task.py

@@ -70,7 +70,8 @@ class Task:
         self.min_classification = task.min_classification.value
         self.max_extracted = task.max_files
         self.metadata = task.metadata
-        self.result: Optional[Result] = None
+        self.partial_result: bool = False
+        self.result: Result = Result()
         self.safelist_config: Dict[str, Any] = task.safelist_config
         self.service_config: Dict[str, Any] = dict(task.service_config)
         self.service_context: Optional[str] = None
@@ -263,11 +264,15 @@ class Task:
             type=self.file_type,
             size=self.file_size,
             drop_file=self.drop_file,
+            partial=self.partial_result,
             temp_submission_data=self.temp_submission_data,
         )
 
         return result
 
+    def partial(self) -> None:
+        self.partial_result = True
+
     def save_error(self, stack_info: str, recoverable: bool) -> None:
         self.error_message = stack_info
 

assemblyline_v4_service/updater/client.py

@@ -17,6 +17,7 @@ class SyncableBadlistClient(BadlistClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
+        self.classification_override = None
 
     def add_update_many(self, data: List[Union[dict, BadlistModel]]) -> Dict[str, Any]:
         return hashlist_add_update_many(self, self.datastore.badlist, data)
@@ -26,6 +27,7 @@ class SyncableSafelistClient(SafelistClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
+        self.classification_override = None
 
     def add_update_many(self, data: List[Union[dict, SafelistModel]]) -> Dict[str, Any]:
         return hashlist_add_update_many(self, self.datastore.safelist, data)
@@ -55,9 +57,14 @@ def hashlist_add_update_many(client: Union[SyncableBadlistClient, SyncableSafeli
             # Set the source name
             source = d["sources"][0]["name"]
 
+        if client.classification_override:
+            # Override the classification of the signature based on what's assigned to the client
+            d["sources"][0]["classification"] = client.classification_override
+
         if client.sync:
             # Compute the expected ID and add it to the list
             current_ids.add(client._preprocess_object(d))
+
         # Update with JSON-friendly version of data to be sent to API
         data[i] = d
 
@@ -122,6 +129,7 @@ class SyncableSignatureClient(SignatureClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
+        self.classification_override = None
 
     def add_update_many(self, source: str, sig_type: str, data: List[Union[dict, SignatureModel]],
                         dedup_name: bool = True) -> Dict[str, Any]:
@@ -144,6 +152,10 @@ class SyncableSignatureClient(SignatureClient):
                 d['status'] = sig_exists['status']
                 d['state_change_user'] = sig_exists['state_change_user']
 
+            if self.classification_override:
+                # Override the classification of the signature based on what's assigned to the client
+                d['classification'] = self.classification_override
+
             if self.sync:
                 # Add signature ID to the list
                 current_signature_ids.add(sig_id)
@@ -208,6 +220,7 @@ class UpdaterClient(object):
     def __init__(self, datastore) -> None:
         self.datastore = datastore
         self._sync = False
+        self._classification_override = None
         self.badlist = SyncableBadlistClient(datastore)
         self.safelist = SyncableSafelistClient(datastore)
         self.signature = SyncableSignatureClient(datastore)
@@ -223,3 +236,15 @@ class UpdaterClient(object):
         self.safelist.sync = value
         self.signature.sync = value
         self._sync = value
+
+    @property
+    def classification_override(self):
+        return self._classification_override
+
+    @classification_override.setter
+    def classification_override(self, value: str):
+        # Set the classification override
+        self.badlist.classification_override = value
+        self.safelist.classification_override = value
+        self.signature.classification_override = value
+        self._classification_override = value

assemblyline_v4_service/updater/helper.py

@@ -88,6 +88,7 @@ def url_download(source: Dict[str, Any], previous_update: int, logger: Logger, o
     ca_cert = source.get('ca_cert', None)
     ignore_ssl_errors = source.get('ssl_ignore_errors', False)
     auth = (username, password) if username and password else None
+    fetch_method = source.get('fetch_method', 'GET').lower()
 
     proxy = source.get('proxy', None)
     headers_list = source.get('headers', [])
@@ -134,7 +135,10 @@ def url_download(source: Dict[str, Any], previous_update: int, logger: Logger, o
                 else:
                     headers = {'If-Modified-Since': previous_update}
 
-            response = session.get(uri, auth=auth, headers=headers, proxies=proxies, stream=True)
+            if fetch_method in ['get', 'post']:
+                response = getattr(session, fetch_method)(uri, auth=auth, headers=headers, proxies=proxies, stream=True)
+            else:
+                raise ValueError(f"Unknown fetch method: {fetch_method}")
 
         # Check the response code
         if response.status_code == requests.codes['not_modified']:
@@ -188,7 +192,13 @@ def git_clone_repo(source: Dict[str, Any], previous_update: int = None, logger=N
     ignore_ssl_errors = source.get("ssl_ignore_errors", False)
     ca_cert = source.get("ca_cert")
     proxy = source.get('proxy', None)
-    auth = f'{username}:{password}@' if username and password else None
+    auth = None
+    if username and password:
+        # Basic authentication scheme
+        auth = f'{username}:{password}@'
+    elif password:
+        # Token-based authentication
+        auth = f'{password}@'
 
     git_env = {}
 

assemblyline_v4_service/updater/updater.py

@@ -378,14 +378,40 @@ class ServiceUpdater(ThreadedCoreBase):
                     source_obj = sources[source_name]
                     old_update_time = self.get_source_update_time()
 
-                    self.push_status("UPDATING", "Starting..")
-                    source = source_obj.as_primitives()
-                    uri: str = source['uri']
-                    default_classification = source.get('default_classification', classification.UNRESTRICTED)
-                    # Enable syncing if the source specifies it
-                    self.client.sync = source.get('sync', False)
-
+                    # Are we ignoring the cache for this source?
+                    if source_obj.ignore_cache:
+                        old_update_time = 0
                     try:
+
+                        source = source_obj.as_primitives()
+                        uri: str = source_obj.uri
+
+                        # If source is not currently enabled/active, skip..
+                        if not source_obj.enabled:
+                            raise SkipSource
+
+                        # Is it time for this source to run?
+                        elapsed_time = time.time() - old_update_time
+                        update_interval = source.get('update_interval') or service.update_config.update_interval_seconds
+                        if elapsed_time < update_interval:
+                            # Too early to run the update for this particular source, skip for now
+                            raise SkipSource
+
+
+                        self.push_status("UPDATING", "Starting..")
+                        fetch_method = source.get('fetch_method', 'GET')
+                        default_classification = source.get('default_classification', classification.UNRESTRICTED)
+
+                        # Configure the client as necessary
+
+                        # Enable syncing if the source specifies it
+                        self.client.sync = source.get('sync', False)
+                        # Override classfication of signatures if specified
+                        # Reset client back to original classification state between updates
+                        self.client.classification_override = None
+                        if source.get('override_classification', False):
+                            self.client.classification_override = default_classification
+
                         self.push_status("UPDATING", "Pulling..")
                         output = None
                         seen_fetch = seen_fetches.get(uri)
@@ -397,21 +423,20 @@ class ServiceUpdater(ThreadedCoreBase):
                             self.log.info(f'Already visited {uri} in this run. Using cached download path..')
                             output = seen_fetches[uri]
                         else:
-                            # Pull sources from external locations (method depends on the URL)
-                            try:
+                            self.log.info(f"Fetching {source_name} using {fetch_method}")
+                            # Pull sources from external locations
+                            if uri.startswith("file:///"):
+                                # Perform an update using a local mount
+                                output = uri.split("file://", 1)[1]
+                                if not os.path.exists(output):
+                                    raise FileNotFoundError(f"{output} doesn't exist within container.")
+                            elif fetch_method == "GIT" or uri.endswith('.git'):
                                 # First we'll attempt by performing a Git clone
                                 # (since not all services hint at being a repository in their URL),
                                 output = git_clone_repo(source, old_update_time, self.log, update_dir)
-                            except SkipSource:
-                                raise
-                            except Exception as git_ex:
-                                # Should that fail, we'll attempt a direct-download using Python Requests
-                                if not uri.endswith('.git'):
-                                    # Proceed with direct download, raise exception as required if necessary
-                                    output = url_download(source, old_update_time, self.log, update_dir)
-                                else:
-                                    # Raise Git Exception
-                                    raise git_ex
+                            else:
+                                # Other fetch methods are meant for URL downloads using Requests
+                                output = url_download(source, old_update_time, self.log, update_dir)
                             # Add output path to the list of seen fetches in this run
                             seen_fetches[uri] = output
 
@@ -430,7 +455,8 @@ class ServiceUpdater(ThreadedCoreBase):
 
                         self.push_status("UPDATING", "Importing..")
                         # Import into Assemblyline
-                        self.import_update(validated_files, source_name, default_classification)
+                        self.import_update(validated_files, source_name, default_classification,
+                                           source.get('configuration') or {})
                         self.push_status("DONE", "Signature(s) Imported.")
                     except SkipSource:
                         # This source hasn't changed, no need to re-import into Assemblyline
@@ -457,7 +483,8 @@ class ServiceUpdater(ThreadedCoreBase):
         return True
 
     # Define how your source update gets imported into Assemblyline
-    def import_update(self, files_sha256: List[Tuple[str, str]], source_name: str, default_classification=None):
+    def import_update(self, files_sha256: List[Tuple[str, str]], source_name: str, default_classification=None,
+                      configuration: dict = {}, *args, **kwargs):
         raise NotImplementedError()
 
     # Define how to prepare the output directory before being served, must return the path of the directory to serve.
@@ -486,7 +513,10 @@ class ServiceUpdater(ThreadedCoreBase):
         while self.running:
             # Stringify and hash the the current update configuration
             service = self._service
-            update_interval = service.update_config.update_interval_seconds
+
+            # The update interval (or sleep interval) will be based on the smallest interval across sources
+            update_interval = min([service.update_config.update_interval_seconds] +
+                                  [s.update_interval for s in service.update_config.sources if s.update_interval])
 
             # Is it time to update yet?
             if time.time() - self.get_scheduled_update_time() < update_interval \

{assemblyline_v4_service-4.5.0.64.dist-info → assemblyline_v4_service-4.5.0.66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.5.0.64
+Version: 4.5.0.66
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.5.0.64.dist-info → assemblyline_v4_service-4.5.0.66.dist-info}/RECORD

@@ -1,4 +1,4 @@
-assemblyline_v4_service/VERSION,sha256=glDcmo_TI9B72qG6u3dZOf7DlHNMv80lFKQ2kz3j2p0,9
+assemblyline_v4_service/VERSION,sha256=HYTwnZwSWV_SzMwzWkIBOOzMK6Rw4M8JAfuZ40V-_BY,9
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -10,19 +10,19 @@ assemblyline_v4_service/common/base.py,sha256=4rnK_183qsSfkRkuVp_0wy-haW49umx4jJ
 assemblyline_v4_service/common/helper.py,sha256=xs9quuf-M1JOdKieBqOmWaOece0CtzXFhhe85xQYmuY,3289
 assemblyline_v4_service/common/ocr.py,sha256=3fV0PyY3oui_ucAM9dkolP0VRYKACKJuGY4M64DudIE,8841
 assemblyline_v4_service/common/ontology_helper.py,sha256=9Ad81qbddg_pRMupT8o_KzxbKgpodaRqpc3mPoEKLtw,8494
-assemblyline_v4_service/common/request.py,sha256=ZP80rDWIyVotwVDN70L7ujN9RoxZRdmaeUN0DFIvrT4,11732
+assemblyline_v4_service/common/request.py,sha256=W7fqC2xQE3i5i2jlCDyUDp3ZqJQQqSshNW0mQfJMkFg,11792
 assemblyline_v4_service/common/result.py,sha256=9AqM6qCYiia_Bpyn_fBFhzNQMcqJbtFSiGjp57fXW2E,32713
-assemblyline_v4_service/common/task.py,sha256=CJ5Mw1Lre09HGf9rbWfKxQdxmRS_mImnHLlE1oQk5dM,14089
+assemblyline_v4_service/common/task.py,sha256=dJsvRpW0x88CCF_LW6w87jQ_UKTVaOs2Gb117IDNiU8,14233
 assemblyline_v4_service/common/utils.py,sha256=k2__d-V5LjB6o2IKbjVe7tJWKcKuUHto5TyT5oKhIa0,3890
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/dev/run_service_once.py,sha256=cGwsNGWNp5KcZ4iVqMf76uGHR21faYhpEkEclMwEdcI,10505
 assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
 assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
-assemblyline_v4_service/updater/client.py,sha256=7_56Ie-2073FlrVWscS-3lIIkg8F-81u97FVNqEJLyQ,9798
+assemblyline_v4_service/updater/client.py,sha256=eBOK8doMu9nsIO6kXWtaBMy4hJIKRtQG0iC_21a5CG4,10849
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
-assemblyline_v4_service/updater/helper.py,sha256=-B35wdjpeY4t1R9SPDrTFHFKHwE3uzy9N69mV6mHy-g,9532
-assemblyline_v4_service/updater/updater.py,sha256=knl8X1xG5wF9nl8j--0WC3BmzQ0PO1Zq7RW58euVjn0,29687
+assemblyline_v4_service/updater/helper.py,sha256=DhxF2TVVpUY6S5dJnjRlTl6eBJrtTqw1uUAcvVWVkhM,9895
+assemblyline_v4_service/updater/updater.py,sha256=8o-vEMMJS388Ci9mVk_6nwFayM5NToliOTFnnPWHTYE,31449
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/conftest.py,sha256=W3SieQpZsZpGEmtLqY4aIlxREDSsHceyCrFcFsWUM0U,1851
 test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -34,12 +34,12 @@ test/test_common/test_base.py,sha256=fuJSSlPxIDHq6HU1xbvaMFitw2z1spOZNHD2SJ4UUic
 test/test_common/test_helper.py,sha256=sO6YAiBhKTqaxlpLhFYDuy2ZdbuF2cg07Ylzo83ZzQs,2575
 test/test_common/test_ocr.py,sha256=mt_PgElgwQKJmNrp2nRVx9NjfMedVk40I6IV317vATI,1753
 test/test_common/test_ontology_helper.py,sha256=Q9-Eqeo8Ih7XlbFmlUAXCtgnfW8JCDqqlYFb56077h4,10331
-test/test_common/test_request.py,sha256=zoBURe3QbycWK4I7uvkixcmykrOCqY_TchmxJinNXjU,11822
+test/test_common/test_request.py,sha256=Ceyds8BNO1O0f1kH1VEb84faJcaupvSjVKIrGdHexsc,11842
 test/test_common/test_result.py,sha256=6BiOKxEPrKBjOY44jv3TY-yiXm0qI1ok_CZBnjP9TM4,45447
-test/test_common/test_task.py,sha256=LzEIfFHJEcB_YPucOZGtm1TwyHnqJE-0Qc2MDsH4TN4,18957
+test/test_common/test_task.py,sha256=P44mNcSe-3tJgDk9ppN3KbM7oN4LBVIuhONG-Gveh74,19007
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.5.0.64.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.5.0.64.dist-info/METADATA,sha256=3KEghl9lwtO8Q0IzJ6EI_l3miqNw74VMVwUGXdkNfk0,9489
-assemblyline_v4_service-4.5.0.64.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-assemblyline_v4_service-4.5.0.64.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.5.0.64.dist-info/RECORD,,
+assemblyline_v4_service-4.5.0.66.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.5.0.66.dist-info/METADATA,sha256=UycdUpsYtr-vDw9QMoQ8AYMCheXMOP4My8sCfhXSpXM,9489
+assemblyline_v4_service-4.5.0.66.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+assemblyline_v4_service-4.5.0.66.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.5.0.66.dist-info/RECORD,,

test/test_common/test_request.py

@@ -5,7 +5,7 @@ from test.test_common import TESSERACT_LIST, setup_module
 
 import pytest
 from assemblyline_v4_service.common.request import ServiceRequest
-from assemblyline_v4_service.common.result import get_heuristic_primitives
+from assemblyline_v4_service.common.result import Result, get_heuristic_primitives
 from assemblyline_v4_service.common.task import MaxExtractedExceeded, Task
 
 from assemblyline.odm.messages.task import Task as ServiceTask
@@ -303,7 +303,7 @@ def test_get_param(service_request):
 
 
 def test_result_getter(service_request):
-    assert service_request.result is None
+    assert isinstance(service_request.result, Result)
 
 
 def test_result_setter(service_request):

test/test_common/test_task.py

@@ -61,7 +61,6 @@ def test_task_init(servicetask):
     assert t.metadata == {}
     assert t.md5 == "d41d8cd98f00b204e9800998ecf8427e"
     assert t.mime is None
-    assert t.result is None
     assert isinstance(t.safelist_config, ServiceSafelist)
     assert t.service_config == {}
     assert t.service_context is None
@@ -380,6 +379,7 @@ def test_task_get_service_result(servicetask):
             "service_debug_info": None,
         },
         "result": {"score": 0, "sections": []},
+        "partial": False,
         "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
         "type": "text/plain",
         "size": 0,
@@ -423,6 +423,7 @@ def test_task_get_service_result(servicetask):
         "result": {
             "score": 0,
         },
+        "partial": False,
         "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
         "type": "text/plain",
         "size": 0,
@@ -486,6 +487,7 @@ def test_task_save_result(servicetask):
             "service_debug_info": None,
         },
         "result": {"score": 0, "sections": []},
+        "partial": False,
         "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
         "type": "text/plain",
         "size": 0,