assemblyline-core 4.5.1.dev182__tar.gz → 4.5.1.dev184__tar.gz

{assemblyline-core-4.5.1.dev182 → assemblyline-core-4.5.1.dev184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev182
+Version: 4.5.1.dev184
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev184/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev184

{assemblyline-core-4.5.1.dev182 → assemblyline-core-4.5.1.dev184}/assemblyline_core/ingester/ingester.py

@@ -36,6 +36,7 @@ from assemblyline.remote.datatypes.queues.priority import PriorityQueue
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
 from assemblyline.remote.datatypes.queues.multi import MultiQueue
 from assemblyline.remote.datatypes.hash import Hash
+from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
 from assemblyline import odm
 from assemblyline.odm.models.submission import SubmissionParams, Submission as DatabaseSubmission
 from assemblyline.odm.models.alert import EXTENDED_SCAN_VALUES
@@ -180,6 +181,9 @@ class Ingester(ThreadedCoreBase):
         # Utility object to handle post-processing actions
         self.postprocess_worker = ActionWorker(cache=True, config=self.config, datastore=self.datastore,
                                                redis_persist=self.redis_persist)
+        # Async Submission quota tracker
+        self.async_submission_tracker = UserQuotaTracker('async_submissions', timeout=24 * 60 * 60,  # 1 day timeout
+                                                         redis=self.redis_persist)
 
         if self.config.core.metrics.apm_server.server_url is not None:
             self.log.info(f"Exporting application metrics to: {self.config.core.metrics.apm_server.server_url}")
@@ -640,7 +644,7 @@ class Ingester(ThreadedCoreBase):
                     'archived': False,
                     'archive_ts': None,
                     'classification': task.params.classification,
-                    'expiry_ts':now_as_iso(task.params.ttl * 24 * 60 * 60),
+                    'expiry_ts': now_as_iso(task.params.ttl * 24 * 60 * 60),
                     'from_archive': False,
                     'metadata': task.submission.metadata,
                     'params': task.params.as_primitives(),
@@ -863,6 +867,9 @@ class Ingester(ThreadedCoreBase):
         return True
 
     def _notify_drop(self, task: IngestTask):
+        if self.config.ui.enforce_quota:
+            self.async_submission_tracker.end(task.params.submitter)
+
         self.send_notification(task)
 
         c12n = task.params.classification
@@ -936,4 +943,7 @@ class Ingester(ThreadedCoreBase):
                 task.extended_scan = 'submitted'
                 task.params.psid = None
 
+        if self.config.ui.enforce_quota:
+            self.async_submission_tracker.end(task.params.submitter)
+
         self.send_notification(task)

{assemblyline-core-4.5.1.dev182 → assemblyline-core-4.5.1.dev184}/assemblyline_core/updater/helper.py

@@ -87,9 +87,6 @@ class AzureContainerRegistry(ContainerRegistry):
                 headers["Authorization"] = f"Bearer {token}"
 
                 resp = self._perform_request(url, headers, verify, proxies)
-                if not resp:
-                    # Fetching tag list failed
-                    return []
 
         # At this point, we should have a response from the API
         if resp and resp.ok:
@@ -218,7 +215,7 @@ def get_latest_tag_for_service(
 
     # Pre-filter tags to only consider 'compatible' tags relative to the running system
     tags = [t for t in tags
-            if re.match(f"({FRAMEWORK_VERSION})[.]({SYSTEM_VERSION})[.]\\d+[.]({update_channel})\\d+", t)]
+            if re.match(f"({FRAMEWORK_VERSION})\.({SYSTEM_VERSION})\.\\d+\.({update_channel})\\d+", t)]
 
     if not tags:
         logger.warning(f"{prefix}Cannot fetch latest tag for service {service_name} - {image_name}"
@@ -253,14 +250,28 @@ def _get_dockerhub_tags(image_name, update_channel, proxies=None):
     # Find latest tag for each types
     rv = []
     namespace, repository = image_name.split('/', 1)
-    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/namespaces/{namespace}/repositories/{repository}/tags"
-    f"?page_size=50&page=1&name={update_channel}"
+    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/namespaces/{namespace}/repositories/{repository}/tags" \
+        f"?page_size=50&page=1&name={update_channel}&ordering=last_updated"
 
     while True:
         resp = requests.get(url, proxies=proxies)
         if resp.ok:
             resp_data = resp.json()
-            rv.extend([x['name'] for x in resp_data['results']])
+            if namespace == "cccs":
+                # The tags are coming from a repository managed by the Assemblyline team
+                for x in resp_data['results']:
+                    tag_name = x['name']
+                    if tag_name == f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.{update_channel}":
+                        # Ignore tag aliases containing the update_channel
+                        continue
+
+                    if tag_name.startswith(f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}"):
+                        # Because the order of paging is based on `last_updated`,
+                        # we can return the first valid candidate
+                        return [tag_name]
+            else:
+                # The tags are coming from another repository, so we're don't know the ordering of tags
+                rv.extend([x['name'] for x in resp_data['results']])
             # Page until there are no results left
             url = resp_data.get('next', None)
             if url is None:

{assemblyline-core-4.5.1.dev182 → assemblyline-core-4.5.1.dev184}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev182
+Version: 4.5.1.dev184
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev182/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev182