assemblyline-v4-service 4.4.1.dev3__py3-none-any.whl → 4.4.1.dev5__py3-none-any.whl

assemblyline_v4_service/VERSION

@@ -1 +1 @@
-4.4.1.dev3
+4.4.1.dev5

assemblyline_v4_service/testing/helper.py

@@ -107,7 +107,7 @@ class TestHelper:
                                    for param in self.submission_params},
                 "fileinfo": {k: v for k, v in self.identify.fileinfo(file_path).items() if k in fileinfo_keys},
                 "filename": filename,
-                "min_classification": "TLP:W",
+                "min_classification": "TLP:C",
                 "max_files": 501,
                 "ttl": 3600,
                 "temporary_submission_data": [

assemblyline_v4_service/updater/helper.py

@@ -56,8 +56,7 @@ def filter_downloads(update_directory, pattern, default_pattern=".*") -> List[Tu
     return f_files
 
 
-def url_download(source: Dict[str, Any], previous_update: int = None,
-                 logger=None, output_dir: str = None) -> List[Tuple[str, str]]:
+def url_download(source: Dict[str, Any], previous_update: int = None, logger=None, output_dir: str = None) -> str:
     """
 
     :param source:
@@ -66,7 +65,6 @@ def url_download(source: Dict[str, Any], previous_update: int = None,
     """
     name = source['name']
     uri = source['uri']
-    pattern = source.get('pattern', None)
     username = source.get('username', None)
     password = source.get('password', None)
     ca_cert = source.get('ca_cert', None)
@@ -143,9 +141,9 @@ def url_download(source: Dict[str, Any], previous_update: int = None,
                 format = format if format in ["zip", "tar"] else None
                 shutil.unpack_archive(file_path, extract_dir=extract_dir, format=format)
 
-                return filter_downloads(extract_dir, pattern)
+                return extract_dir
             else:
-                return [(file_path, get_sha256_for_file(file_path))]
+                return file_path
         else:
             logger.warning(f"Download not successful: {response.content}")
             return []
@@ -161,11 +159,9 @@ def url_download(source: Dict[str, Any], previous_update: int = None,
         session.close()
 
 
-def git_clone_repo(source: Dict[str, Any], previous_update: int = None, default_pattern: str = "*",
-                   logger=None, output_dir: str = None) -> List[Tuple[str, str]]:
+def git_clone_repo(source: Dict[str, Any], previous_update: int = None, logger=None, output_dir: str = None) -> str:
     name = source['name']
     url = source['uri']
-    pattern = source.get('pattern', None)
     key = source.get('private_key', None)
     username = source.get('username', None)
     password = source.get('password', None)
@@ -225,7 +221,7 @@ def git_clone_repo(source: Dict[str, Any], previous_update: int = None, default_
                         raise SkipSource()
                     break
 
-        return filter_downloads(clone_dir, pattern, default_pattern)
+        return clone_dir
     except SkipSource:
         # Raise to calling function for handling
         raise

assemblyline_v4_service/updater/updater.py

@@ -32,7 +32,7 @@ from assemblyline.remote.datatypes.lock import Lock
 from assemblyline.odm.models.user import User
 from assemblyline.odm.models.user_settings import UserSettings
 
-from assemblyline_v4_service.updater.helper import url_download, git_clone_repo, SkipSource
+from assemblyline_v4_service.updater.helper import url_download, git_clone_repo, SkipSource, filter_downloads
 
 
 if typing.TYPE_CHECKING:
@@ -369,6 +369,9 @@ class ServiceUpdater(ThreadedCoreBase):
                 sources: dict[str, UpdateSource] = {_s['name']: _s for _s in service.update_config.sources}
                 files_sha256: dict[str, dict[str, str]] = {}
 
+                # Map already visited URIs to download paths (avoid re-cloning/re-downloads)
+                seen_fetches = dict()
+
                 # Go through each source and download file
                 for source_name, source_obj in sources.items():
                     # Set current source for pushing state to UI
@@ -385,20 +388,29 @@ class ServiceUpdater(ThreadedCoreBase):
                     default_classification = source.get('default_classification', classification.UNRESTRICTED)
                     try:
                         self.push_status("UPDATING", "Pulling..")
-
-                        # Pull sources from external locations (method depends on the URL)
-                        try:
-                            # First we'll attempt by performing a Git clone
-                            # (since not all services hint at being a repository in their URL),
-                            files = git_clone_repo(source, old_update_time, self.default_pattern, self.log, update_dir)
-                        except Exception as git_ex:
-                            # Should that fail, we'll attempt a direct-download using Python Requests
-                            if not uri.endswith('.git'):
-                                # Proceed with direct download, raise exception as required if necessary
-                                files = url_download(source, old_update_time, self.log, update_dir)
-                            else:
-                                # Raise Git Exception
-                                raise git_ex
+                        output = None
+                        if uri in seen_fetches:
+                            # We've already fetched something from the same URI, re-use downloaded path
+                            self.log.info(f'Already visited {uri} in this run. Using cached download path..')
+                            output = seen_fetches[uri]
+                        else:
+                            # Pull sources from external locations (method depends on the URL)
+                            try:
+                                # First we'll attempt by performing a Git clone
+                                # (since not all services hint at being a repository in their URL),
+                                output = git_clone_repo(source, old_update_time, self.log, update_dir)
+                            except Exception as git_ex:
+                                # Should that fail, we'll attempt a direct-download using Python Requests
+                                if not uri.endswith('.git'):
+                                    # Proceed with direct download, raise exception as required if necessary
+                                    output = url_download(source, old_update_time, self.log, update_dir)
+                                else:
+                                    # Raise Git Exception
+                                    raise git_ex
+                            # Add output path to the list of seen fetches in this run
+                            seen_fetches[uri] = output
+
+                        files = filter_downloads(output, source['pattern'], self.default_pattern)
 
                         # Add to collection of sources for caching purposes
                         self.log.info(f"Found new {self.updater_type} rule files to process for {source_name}!")

{assemblyline_v4_service-4.4.1.dev3.dist-info → assemblyline_v4_service-4.4.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-v4-service
-Version: 4.4.1.dev3
+Version: 4.4.1.dev5
 Summary: Assemblyline 4 - Service base
 Home-page: https://github.com/CybercentreCanada/assemblyline-v4-service/
 Author: CCCS Assemblyline development team

{assemblyline_v4_service-4.4.1.dev3.dist-info → assemblyline_v4_service-4.4.1.dev5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-assemblyline_v4_service/VERSION,sha256=wYfUWkgn81gW-y16MRL4vOfEnNDzfQOAq0wnhDXxLak,11
+assemblyline_v4_service/VERSION,sha256=TA5K6_WV8uRs4Zs_JwC03uQVzUZ_CjaprBvWthjQA8w,11
 assemblyline_v4_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/healthz.py,sha256=3QGBg0EZuXC6UN411HFwpLNEop9UvS9feFhvBUTP-k4,1576
 assemblyline_v4_service/run_privileged_service.py,sha256=9uTfHetXR5G-EDKMDrgfWUOw34yr64-cj6Cm9eZaCbQ,14547
@@ -39,16 +39,16 @@ assemblyline_v4_service/common/pestudio/xml/strings.xml,sha256=kRU8WbCcU1RckM6oC
 assemblyline_v4_service/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/dev/run_service_once.py,sha256=4gnb09WeKXlyWQKCQdH4SoL4xtfIRWq_9nyIiECrJ7g,10592
 assemblyline_v4_service/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-assemblyline_v4_service/testing/helper.py,sha256=tVfTirHeckUxVAN6cRUZvnH8WG3ccp5IM4sB8n8W0nc,19711
+assemblyline_v4_service/testing/helper.py,sha256=f0-qBtgR0vWZBpEV9sPfcworLtdh4h_CcoAofHlOtZE,19711
 assemblyline_v4_service/testing/regenerate_results.py,sha256=Cbp2CMAxbF3kz5vxEPPCxrgUp1Vl3Tz6e46aUhg_I4U,1101
 assemblyline_v4_service/updater/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 assemblyline_v4_service/updater/__main__.py,sha256=9Os-u8Tf7MD73JSrUSPmOaErTgfvesNLiEeszU4ujXA,133
 assemblyline_v4_service/updater/app.py,sha256=IB9UrfUesu88ixC_VKk8IbEVvhYFI7c7gla_IfS_g2g,2780
 assemblyline_v4_service/updater/gunicorn_config.py,sha256=8Qulsnw9pcsol4x_3oeyRkvovFEe0nEJQnPPemuOl-I,1155
-assemblyline_v4_service/updater/helper.py,sha256=I96U4y2RX9NLEJghziB3bn56pfB5jwU0kH3T2zeyFTA,9222
-assemblyline_v4_service/updater/updater.py,sha256=1MvpoA4uvxOd0F747rnQOaSI8oTUTtB5_5AV7plbbkA,27248
-assemblyline_v4_service-4.4.1.dev3.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
-assemblyline_v4_service-4.4.1.dev3.dist-info/METADATA,sha256=V6--cSxBapac8DC3HgbmT31Qs182UWtTcgdEeSnEWPo,9358
-assemblyline_v4_service-4.4.1.dev3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-assemblyline_v4_service-4.4.1.dev3.dist-info/top_level.txt,sha256=Ut5IqePObcxlJ8rv2--dOAzYbxzqlllfiV_51cbqjbA,24
-assemblyline_v4_service-4.4.1.dev3.dist-info/RECORD,,
+assemblyline_v4_service/updater/helper.py,sha256=gsJwmdDz1hJYstRfg5jdkq7nH45ppJ0yPqEiH3IDsZE,8931
+assemblyline_v4_service/updater/updater.py,sha256=MDKV_pvtmsJVv5qUH_qlhnWKW0oC-uXsoY2_bWqP9pI,28054
+assemblyline_v4_service-4.4.1.dev5.dist-info/LICENCE.md,sha256=NSkYo9EH8h5oOkzg4VhjAHF4339MqPP2cQ8msTPgl-c,1396
+assemblyline_v4_service-4.4.1.dev5.dist-info/METADATA,sha256=q1lJ0Vv_Eg3mA9tCdr_wHV7wXVAVQF3hdc4OyBaVj-o,9358
+assemblyline_v4_service-4.4.1.dev5.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+assemblyline_v4_service-4.4.1.dev5.dist-info/top_level.txt,sha256=Ut5IqePObcxlJ8rv2--dOAzYbxzqlllfiV_51cbqjbA,24
+assemblyline_v4_service-4.4.1.dev5.dist-info/RECORD,,