assemblyline-v4-service 4.5.1.dev379__py3-none-any.whl → 4.5.1.dev381__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.5.1.dev379
+4.5.1.dev381

assemblyline_v4_service/updater/client.py

@@ -17,6 +17,7 @@ class SyncableBadlistClient(BadlistClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
+        self.classification_override = None
 
     def add_update_many(self, data: List[Union[dict, BadlistModel]]) -> Dict[str, Any]:
         return hashlist_add_update_many(self, self.datastore.badlist, data)
@@ -26,6 +27,7 @@ class SyncableSafelistClient(SafelistClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
+        self.classification_override = None
 
     def add_update_many(self, data: List[Union[dict, SafelistModel]]) -> Dict[str, Any]:
         return hashlist_add_update_many(self, self.datastore.safelist, data)
@@ -55,9 +57,14 @@ def hashlist_add_update_many(client: Union[SyncableBadlistClient, SyncableSafeli
             # Set the source name
             source = d["sources"][0]["name"]
 
+        if client.classification_override:
+            # Override the classification of the signature based on what's assigned to the client
+            d["sources"][0]["classification"] = client.classification_override
+
         if client.sync:
             # Compute the expected ID and add it to the list
             current_ids.add(client._preprocess_object(d))
+
         # Update with JSON-friendly version of data to be sent to API
         data[i] = d
 
@@ -122,6 +129,7 @@ class SyncableSignatureClient(SignatureClient):
     def __init__(self, datastore, config=None):
         super().__init__(datastore, config)
         self.sync = False
+        self.classification_override = None
 
     def add_update_many(self, source: str, sig_type: str, data: List[Union[dict, SignatureModel]],
                         dedup_name: bool = True) -> Dict[str, Any]:
@@ -144,6 +152,10 @@ class SyncableSignatureClient(SignatureClient):
                 d['status'] = sig_exists['status']
                 d['state_change_user'] = sig_exists['state_change_user']
 
+            if self.classification_override:
+                # Override the classification of the signature based on what's assigned to the client
+                d['classification'] = self.classification_override
+
             if self.sync:
                 # Add signature ID to the list
                 current_signature_ids.add(sig_id)
@@ -208,6 +220,7 @@ class UpdaterClient(object):
     def __init__(self, datastore) -> None:
         self.datastore = datastore
         self._sync = False
+        self._classification_override = None
         self.badlist = SyncableBadlistClient(datastore)
         self.safelist = SyncableSafelistClient(datastore)
         self.signature = SyncableSignatureClient(datastore)
@@ -223,3 +236,14 @@ class UpdaterClient(object):
         self.safelist.sync = value
         self.signature.sync = value
         self._sync = value
+
+    @property
+    def classification_override(self):
+        return self._classification_override
+    @classification_override.setter
+    def classification_override(self, value: str):
+        # Set the classification override
+        self.badlist.classification_override = value
+        self.safelist.classification_override = value
+        self.signature.classification_override = value
+        self._classification_override = value

assemblyline_v4_service/updater/helper.py

@@ -88,6 +88,7 @@ def url_download(source: Dict[str, Any], previous_update: int, logger: Logger, o
     ca_cert = source.get('ca_cert', None)
     ignore_ssl_errors = source.get('ssl_ignore_errors', False)
     auth = (username, password) if username and password else None
+    fetch_method = source.get('fetch_method', 'GET').lower()
 
     proxy = source.get('proxy', None)
     headers_list = source.get('headers', [])
@@ -134,7 +135,10 @@ def url_download(source: Dict[str, Any], previous_update: int, logger: Logger, o
                 else:
                     headers = {'If-Modified-Since': previous_update}
 
-            response = session.get(uri, auth=auth, headers=headers, proxies=proxies, stream=True)
+            if fetch_method in ['get', 'post']:
+                response = getattr(session, fetch_method)(uri, auth=auth, headers=headers, proxies=proxies, stream=True)
+            else:
+                raise ValueError(f"Unknown fetch method: {fetch_method}")
 
         # Check the response code
         if response.status_code == requests.codes['not_modified']:

assemblyline_v4_service/updater/updater.py

@@ -378,14 +378,37 @@ class ServiceUpdater(ThreadedCoreBase):
                     source_obj = sources[source_name]
                     old_update_time = self.get_source_update_time()
 
-                    self.push_status("UPDATING", "Starting..")
-                    source = source_obj.as_primitives()
-                    uri: str = source['uri']
-                    default_classification = source.get('default_classification', classification.UNRESTRICTED)
-                    # Enable syncing if the source specifies it
-                    self.client.sync = source.get('sync', False)
-
+                    # Are we ignoring the cache for this source?
+                    if source_obj.ignore_cache:
+                        old_update_time = 0
                     try:
+
+                        source = source_obj.as_primitives()
+                        uri: str = source_obj.uri
+
+                        # If source is not currently enabled/active, skip..
+                        if not source_obj.enabled:
+                            raise SkipSource
+
+                        # Is it time for this source to run?
+                        elapsed_time = time.time() - old_update_time
+                        if elapsed_time < source.get('update_interval', service.update_config.update_interval_seconds):
+                            # Too early to run the update for this particular source, skip for now
+                            raise SkipSource
+
+
+                        self.push_status("UPDATING", "Starting..")
+                        fetch_method = source.get('fetch_method', 'GET')
+                        default_classification = source.get('default_classification', classification.UNRESTRICTED)
+
+                        # Configure the client as necessary
+
+                        # Enable syncing if the source specifies it
+                        self.client.sync = source.get('sync', False)
+                        # Override classfication of signatures if specified
+                        self.client.classification_override = default_classification \
+                            if source.get('override_classification', False) else None
+
                         self.push_status("UPDATING", "Pulling..")
                         output = None
                         seen_fetch = seen_fetches.get(uri)
@@ -397,21 +420,20 @@ class ServiceUpdater(ThreadedCoreBase):
                             self.log.info(f'Already visited {uri} in this run. Using cached download path..')
                             output = seen_fetches[uri]
                         else:
-                            # Pull sources from external locations (method depends on the URL)
-                            try:
+                            self.log.info(f"Fetching {source_name} using {fetch_method}")
+                            # Pull sources from external locations
+                            if uri.startswith("file://"):
+                                # Perform an update using a local mount
+                                output = uri.split("file://", 1)[1]
+                                if not os.path.exists(output):
+                                    raise FileNotFoundError(f"{output} doesn't exist within container.")
+                            elif fetch_method == "GIT" or uri.endswith('.git'):
                                 # First we'll attempt by performing a Git clone
                                 # (since not all services hint at being a repository in their URL),
                                 output = git_clone_repo(source, old_update_time, self.log, update_dir)
-                            except SkipSource:
-                                raise
-                            except Exception as git_ex:
-                                # Should that fail, we'll attempt a direct-download using Python Requests
-                                if not uri.endswith('.git'):
-                                    # Proceed with direct download, raise exception as required if necessary
-                                    output = url_download(source, old_update_time, self.log, update_dir)
-                                else:
-                                    # Raise Git Exception
-                                    raise git_ex
+                            else:
+                                # Other fetch methods are meant for URL downloads using Requests
+                                output = url_download(source, old_update_time, self.log, update_dir)
                             # Add output path to the list of seen fetches in this run
                             seen_fetches[uri] = output
 
@@ -430,7 +452,8 @@ class ServiceUpdater(ThreadedCoreBase):
 
                         self.push_status("UPDATING", "Importing..")
                         # Import into Assemblyline
-                        self.import_update(validated_files, source_name, default_classification)
+                        self.import_update(validated_files, source_name, default_classification,
+                                           source.get('configuration') or {})
                         self.push_status("DONE", "Signature(s) Imported.")
                     except SkipSource:
                         # This source hasn't changed, no need to re-import into Assemblyline
@@ -457,7 +480,8 @@ class ServiceUpdater(ThreadedCoreBase):
         return True
 
     # Define how your source update gets imported into Assemblyline
-    def import_update(self, files_sha256: List[Tuple[str, str]], source_name: str, default_classification=None):
+    def import_update(self, files_sha256: List[Tuple[str, str]], source_name: str, default_classification=None,
+                      configuration: dict = {}, *args, **kwargs):
         raise NotImplementedError()
 
     # Define how to prepare the output directory before being served, must return the path of the directory to serve.
@@ -486,7 +510,10 @@ class ServiceUpdater(ThreadedCoreBase):
         while self.running:
             # Stringify and hash the the current update configuration
             service = self._service
-            update_interval = service.update_config.update_interval_seconds
+
+            # The update interval (or sleep interval) will be based on the smallest interval across sources
+            update_interval = min([service.update_config.update_interval_seconds] +
+                                  [s.update_interval for s in service.update_config.sources if s.update_interval])
 
             # Is it time to update yet?
             if time.time() - self.get_scheduled_update_time() < update_interval \

{assemblyline_v4_service-4.5.1.dev379.dist-info → assemblyline_v4_service-4.5.1.dev381.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.5.1.dev379
+Version: 4.5.1.dev381
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.5.1.dev379.dist-info → assemblyline_v4_service-4.5.1.dev381.dist-info}/RECORD

@@ -1,4 +1,4 @@
-assemblyline_v4_service/VERSION,sha256=Iq2Hrx8-P0h74nvnG6J62t2ZLiDciQqSYHKDQLrr59o,13
+assemblyline_v4_service/VERSION,sha256=GoI_N3ukyAbYAnqbnnVvNHDAsRDMZ6Ex_3AyNiQOTQ0,13
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=sS1cFkDLw8hUPMpj7tbHXFv8ZmHcazrwZ0l6oQDwwkQ,1575
 assemblyline_v4_service/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,10 +19,10 @@ assemblyline_v4_service/dev/run_service_once.py,sha256=iEWkLnF4cWN0V3Mc6GhMKjDAP
 assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
 assemblyline_v4_service/updater/app.py,sha256=Mtmx4bkXfP4nFqqa5q15jW8QIXr4JK84lCovxAVyvPs,3317
-assemblyline_v4_service/updater/client.py,sha256=7_56Ie-2073FlrVWscS-3lIIkg8F-81u97FVNqEJLyQ,9798
+assemblyline_v4_service/updater/client.py,sha256=mmCwNuGSDymbvrMuEXLgTR13vqaUEjs0Jjew-9QkQtM,10848
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=p3j2KPBeD5jvMw9O5i7vAtlRgPSVVxIG9AO0DfN82J8,1247
-assemblyline_v4_service/updater/helper.py,sha256=-B35wdjpeY4t1R9SPDrTFHFKHwE3uzy9N69mV6mHy-g,9532
-assemblyline_v4_service/updater/updater.py,sha256=knl8X1xG5wF9nl8j--0WC3BmzQ0PO1Zq7RW58euVjn0,29687
+assemblyline_v4_service/updater/helper.py,sha256=f0yB6iKExQAqCMkMs0aWPkzcaSy_z9sycgunoCOJlU4,9756
+assemblyline_v4_service/updater/updater.py,sha256=3mKYnAwiEVxmcoa3158HSs_X-xPwn1hKCEnphmnF1H0,31239
 test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/conftest.py,sha256=W3SieQpZsZpGEmtLqY4aIlxREDSsHceyCrFcFsWUM0U,1851
 test/test_healthz.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
@@ -38,8 +38,8 @@ test/test_common/test_request.py,sha256=Ceyds8BNO1O0f1kH1VEb84faJcaupvSjVKIrGdHe
 test/test_common/test_result.py,sha256=6BiOKxEPrKBjOY44jv3TY-yiXm0qI1ok_CZBnjP9TM4,45447
 test/test_common/test_task.py,sha256=P44mNcSe-3tJgDk9ppN3KbM7oN4LBVIuhONG-Gveh74,19007
 test/test_common/test_utils.py,sha256=TbnBxqpS_ZC5ptXR9XJX3xtbItD0mTbtiBxxdyP8J5k,5904
-assemblyline_v4_service-4.5.1.dev379.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.5.1.dev379.dist-info/METADATA,sha256=dQ9XH-JPcNBIyppzYK1xBz4ZzDCzLcc05M0y5ByoZXI,9493
-assemblyline_v4_service-4.5.1.dev379.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-assemblyline_v4_service-4.5.1.dev379.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
-assemblyline_v4_service-4.5.1.dev379.dist-info/RECORD,,
+assemblyline_v4_service-4.5.1.dev381.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.5.1.dev381.dist-info/METADATA,sha256=LSLNRHFtVgToOu0Kow7Vb9I3kmTW7HMT_3WqUGWX78s,9493
+assemblyline_v4_service-4.5.1.dev381.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+assemblyline_v4_service-4.5.1.dev381.dist-info/top_level.txt,sha256=LpTOEaVCatkrvbVq3EZseMSIa2PQZU-2rhuO_FTpZgY,29
+assemblyline_v4_service-4.5.1.dev381.dist-info/RECORD,,