assemblyline-core 4.5.0.33__tar.gz → 4.5.0.34__tar.gz

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.33
+Version: 4.5.0.34
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.34/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.34

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/assemblyline_core/dispatching/client.py

@@ -254,7 +254,8 @@ class DispatchClient:
 
     @elasticapm.capture_span(span_type='dispatch_client')
     def service_finished(self, sid: str, result_key: str, result: Result,
-                         temporary_data: Optional[dict[str, Any]] = None):
+                         temporary_data: Optional[dict[str, Any]] = None,
+                         version="create"):
         """Notifies the dispatcher of service completion, and possible new files to dispatch."""
         # Make sure the dispatcher knows we were working on this task
         task_key = ServiceTask.make_key(sid=sid, service_name=result.response.service_name, sha=result.sha256)
@@ -272,28 +273,22 @@ class DispatchClient:
             self.ds.emptyresult.save(result_key, {"expiry_ts": result.expiry_ts})
         else:
             while True:
-                old, version = self.ds.result.get_if_exists(result_key, version=True)
-                if old:
-                    if old.expiry_ts and result.expiry_ts:
-                        result.expiry_ts = max(result.expiry_ts, old.expiry_ts)
-                    else:
-                        result.expiry_ts = None
                 try:
-                    if self.ds.result.exists(result_key):
-                        # A result already exists for this key
-                        # Regenerate entire result key based on result and modified task (ignore caching)
-                        task.ignore_cache = True
-                        result_key = Result.help_build_key(sha256=task.fileinfo.sha256,
-                                                           service_name=result.response.service_name,
-                                                           service_version=result.response.service_version,
-                                                           service_tool_version=result.response.service_tool_version,
-                                                           is_empty=False,
-                                                           task=task)
-                        version = "create"
                     self.ds.result.save(result_key, result, version=version)
                     break
                 except VersionConflictException as vce:
                     self.log.info(f"Retrying to save results due to version conflict: {str(vce)}")
+                    # A result already exists for this key
+                    # Regenerate entire result key based on result and modified task (ignore caching)
+                    version = "create"
+                    result.created = now_as_iso()
+                    task.ignore_cache = True
+                    result_key = Result.help_build_key(sha256=task.fileinfo.sha256,
+                                                       service_name=result.response.service_name,
+                                                       service_version=result.response.service_version,
+                                                       service_tool_version=result.response.service_tool_version,
+                                                       is_empty=False,
+                                                       task=task)
 
         # Send the result key to any watching systems
         msg = {'status': 'OK', 'cache_key': result_key}

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/assemblyline_core/ingester/ingester.py

@@ -36,6 +36,7 @@ from assemblyline.remote.datatypes.queues.priority import PriorityQueue
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
 from assemblyline.remote.datatypes.queues.multi import MultiQueue
 from assemblyline.remote.datatypes.hash import Hash
+from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
 from assemblyline import odm
 from assemblyline.odm.models.submission import SubmissionParams, Submission as DatabaseSubmission
 from assemblyline.odm.models.alert import EXTENDED_SCAN_VALUES
@@ -180,6 +181,9 @@ class Ingester(ThreadedCoreBase):
         # Utility object to handle post-processing actions
         self.postprocess_worker = ActionWorker(cache=True, config=self.config, datastore=self.datastore,
                                                redis_persist=self.redis_persist)
+        # Async Submission quota tracker
+        self.async_submission_tracker = UserQuotaTracker('async_submissions', timeout=24 * 60 * 60,  # 1 day timeout
+                                                         redis=self.redis_persist)
 
         if self.config.core.metrics.apm_server.server_url is not None:
             self.log.info(f"Exporting application metrics to: {self.config.core.metrics.apm_server.server_url}")
@@ -640,7 +644,7 @@ class Ingester(ThreadedCoreBase):
                     'archived': False,
                     'archive_ts': None,
                     'classification': task.params.classification,
-                    'expiry_ts':now_as_iso(task.params.ttl * 24 * 60 * 60),
+                    'expiry_ts': now_as_iso(task.params.ttl * 24 * 60 * 60),
                     'from_archive': False,
                     'metadata': task.submission.metadata,
                     'params': task.params.as_primitives(),
@@ -863,6 +867,9 @@ class Ingester(ThreadedCoreBase):
         return True
 
     def _notify_drop(self, task: IngestTask):
+        if self.config.ui.enforce_quota:
+            self.async_submission_tracker.end(task.params.submitter)
+
         self.send_notification(task)
 
         c12n = task.params.classification
@@ -936,4 +943,7 @@ class Ingester(ThreadedCoreBase):
                 task.extended_scan = 'submitted'
                 task.params.psid = None
 
+        if self.config.ui.enforce_quota:
+            self.async_submission_tracker.end(task.params.submitter)
+
         self.send_notification(task)

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/assemblyline_core/tasking_client.py

@@ -24,6 +24,7 @@ from assemblyline.remote.datatypes.events import EventSender, EventWatcher
 from assemblyline.remote.datatypes.hash import ExpiringHash
 from assemblyline_core.dispatching.client import DispatchClient
 
+
 class TaskingClientException(Exception):
     pass
 
@@ -252,7 +253,7 @@ class TaskingClient:
         # If we are allowed, try to see if the result has been cached
         if not task.ignore_cache and not service_data.disable_cache:
             # Checking for previous results for this key
-            result = self.datastore.result.get_if_exists(result_key)
+            result, version = self.datastore.result.get_if_exists(result_key, version=True)
             if result:
                 metric_factory.increment('cache_hit')
                 if result.result.score:
@@ -262,8 +263,8 @@ class TaskingClient:
 
                 result.archive_ts = None
 
-                if task.ttl:
-                    result.expiry_ts = now_as_iso(task.ttl * 24 * 60 * 60)
+                if task.ttl and result.expiry_ts:
+                    result.expiry_ts = max(result.expiry_ts, now_as_iso(task.ttl * 24 * 60 * 60))
 
                 # Create a list of files to freshen
                 freshen_hashes = [task.fileinfo.sha256]
@@ -286,7 +287,7 @@ class TaskingClient:
                 for sha256 in freshen_hashes:
                     self.datastore.save_or_freshen_file(sha256, {}, result.expiry_ts, result.classification)
 
-                self.dispatch_client.service_finished(task.sid, result_key, result)
+                self.dispatch_client.service_finished(task.sid, result_key, result, version=version)
                 cache_found = True
 
             if not cache_found:

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/assemblyline_core/updater/helper.py

@@ -217,7 +217,7 @@ def get_latest_tag_for_service(
 
     # Pre-filter tags to only consider 'compatible' tags relative to the running system
     tags = [t for t in tags
-            if re.match(f"({FRAMEWORK_VERSION})[.]({SYSTEM_VERSION})[.]\\d+[.]({update_channel})\\d+", t)]
+            if re.match(f"({FRAMEWORK_VERSION})\.({SYSTEM_VERSION})\.\\d+\.({update_channel})\\d+", t)]
 
     if not tags:
         logger.warning(f"{prefix}Cannot fetch latest tag for service {service_name} - {image_name}"
@@ -252,14 +252,28 @@ def _get_dockerhub_tags(image_name, update_channel, proxies=None):
     # Find latest tag for each types
     rv = []
     namespace, repository = image_name.split('/', 1)
-    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/namespaces/{namespace}/repositories/{repository}/tags"
-    f"?page_size=50&page=1&name={update_channel}"
+    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/namespaces/{namespace}/repositories/{repository}/tags" \
+        f"?page_size=50&page=1&name={update_channel}&ordering=last_updated"
 
     while True:
         resp = requests.get(url, proxies=proxies)
         if resp.ok:
             resp_data = resp.json()
-            rv.extend([x['name'] for x in resp_data['results']])
+            if namespace == "cccs":
+                # The tags are coming from a repository managed by the Assemblyline team
+                for x in resp_data['results']:
+                    tag_name = x['name']
+                    if tag_name == f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.{update_channel}":
+                        # Ignore tag aliases containing the update_channel
+                        continue
+
+                    if tag_name.startswith(f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}"):
+                        # Because the order of paging is based on `last_updated`,
+                        # we can return the first valid candidate
+                        return [tag_name]
+            else:
+                # The tags are coming from another repository, so we're don't know the ordering of tags
+                rv.extend([x['name'] for x in resp_data['results']])
             # Page until there are no results left
             url = resp_data.get('next', None)
             if url is None:

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/assemblyline_core/workflow/run_workflow.py

@@ -175,7 +175,7 @@ class WorkflowManager(ServerBase):
 
                     except SearchException:
                         self.log.warning(f"Invalid query '{safe_str(workflow.query or '')}' in workflow "
-                                         f"'{workflow.name or 'unknown'}' by '{workflow.created_by or 'unknown'}'")
+                                         f"'{workflow.name or 'unknown'}' by '{workflow.creator or 'unknown'}'")
 
                         # End of transaction
                         if self.apm_client:

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.34}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.33
+Version: 4.5.0.34
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.33/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.33