assemblyline-core 4.5.0.19__tar.gz → 4.5.0.21__tar.gz

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.19
+Version: 4.5.0.21
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.21/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.21

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/assemblyline_core/archiver/run_archiver.py

@@ -7,8 +7,8 @@ from assemblyline.common import forge
 from assemblyline.common.archiving import ARCHIVE_QUEUE_NAME
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.datastore.collection import ESCollection, Index
+from assemblyline.datastore.exceptions import VersionConflictException
 from assemblyline.odm.messages.archive_heartbeat import Metrics
-from assemblyline.odm.models.submission import Submission
 from assemblyline.remote.datatypes import get_client
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 
@@ -62,7 +62,16 @@ class Archiver(ServerBase):
             return
         else:
             try:
-                archive_type, type_id, delete_after = message
+                if len(message) == 3:
+                    archive_type, type_id, delete_after = message
+                    metadata = None
+                    use_alternate_dtl = False
+                elif len(message) == 4:
+                    archive_type, type_id, delete_after, metadata = message
+                    use_alternate_dtl = False
+                else:
+                    archive_type, type_id, delete_after, metadata, use_alternate_dtl = message
+
                 self.counter.increment('received')
             except Exception:
                 self.log.error(f"Invalid message received: {message}")
@@ -76,11 +85,24 @@ class Archiver(ServerBase):
             if archive_type == "submission":
                 self.counter.increment('submission')
                 # Load submission
-                submission: Submission = self.datastore.submission.get_if_exists(type_id)
+                while True:
+                    try:
+                        submission, version = self.datastore.submission.get_if_exists(type_id, version=True)
+
+                        # If we have metadata passed in the message, we need to apply it before archiving the submission
+                        if metadata and self.config.core.archiver.use_metadata:
+                            submission.metadata.update({f"archive.{k}": v for k, v in metadata.items()})
+                            self.datastore.submission.save(type_id, submission, version=version)
+
+                        break
+                    except VersionConflictException as vce:
+                        self.log.info(f"Retrying saving metadata due to version conflict: {str(vce)}")
+
                 if not submission:
                     raise SubmissionNotFound(type_id)
 
-                self.datastore.submission.archive(type_id, delete_after=delete_after)
+                self.datastore.submission.archive(type_id, delete_after=delete_after,
+                                                  use_alternate_dtl=use_alternate_dtl)
                 if not delete_after:
                     self.datastore.submission.update(type_id, [(ESCollection.UPDATE_SET, 'archived', True)],
                                                      index_type=Index.HOT)
@@ -100,7 +122,8 @@ class Archiver(ServerBase):
                     infos = infos.union({'password' for x in tags if x['type'] == 'info.password'})
 
                     # Create the archive file
-                    self.datastore.file.archive(sha256, delete_after=delete_after, allow_missing=True)
+                    self.datastore.file.archive(sha256, delete_after=delete_after,
+                                                allow_missing=True, use_alternate_dtl=use_alternate_dtl)
 
                     # Auto-Labelling
                     operations = []
@@ -142,7 +165,8 @@ class Archiver(ServerBase):
                 for r in submission.results:
                     if not r.endswith(".e"):
                         self.counter.increment('result')
-                        self.datastore.result.archive(r, delete_after=delete_after, allow_missing=True)
+                        self.datastore.result.archive(r, delete_after=delete_after,
+                                                      allow_missing=True, use_alternate_dtl=use_alternate_dtl)
 
                 # End of process alert transaction (success)
                 self.log.info(f"Successfully archived submission '{type_id}'.")

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/assemblyline_core/scaler/controllers/docker_ctl.py

@@ -191,7 +191,7 @@ class DockerController(ControllerInterface):
         # Define environment variables
         env = [f'{_e.name}={_e.value}' for _e in cfg.environment]
         env += [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
-        env += [f'LOG_LEVEL={self.log_level}']
+        env += [f'LOG_LEVEL={self.log_level}', f'AL_SERVICE_NAME={service_name}']
         env += [f'{_n}={_v}' for _n, _v in self._service_limited_env[service_name].items()]
         if prof.privileged:
             env.append('PRIVILEGED=true')
@@ -259,6 +259,15 @@ class DockerController(ControllerInterface):
         env += [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
         env += [f'LOG_LEVEL={self.log_level}', f'AL_SERVICE_NAME={service_name}']
 
+        healthcheck = {
+            'test': ["CMD", "python3", "-m", "assemblyline_v4_service.healthz"],
+            'interval': SERVICE_LIVENESS_PERIOD,
+            'timeout': SERVICE_LIVENESS_TIMEOUT
+        }
+
+        if 'assemblyline' not in cfg.image:
+            healthcheck = None
+
         container = self.client.containers.run(
             image=cfg.image,
             name=name,
@@ -274,12 +283,9 @@ class DockerController(ControllerInterface):
             environment=env,
             detach=True,
             # ports=ports,
-            healthcheck={
-                'test': ["CMD", "python3", "-m", "assemblyline_v4_service.healthz"],
-                'interval': SERVICE_LIVENESS_PERIOD,
-                'timeout': SERVICE_LIVENESS_TIMEOUT
-            }
+            healthcheck=healthcheck
         )
+
         if core_container:
             self._connect_to_network(container, self.core_network, aliases=[hostname])
 

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/assemblyline_core/scaler/scaler_server.py

@@ -448,11 +448,30 @@ class ScalerServer(ThreadedCoreBase):
                 self._sync_service(service)
 
     def sync_services(self):
+        last_synced_profiles = None
         while self.running:
             with apm_span(self.apm_client, 'sync_services'):
                 self.log.info('Synchronizing service configuration')
                 with self.profiles_lock:
                     current_services = set(self.profiles.keys())
+
+                    # Check to see if the service is progressing since it's last sync
+                    if last_synced_profiles:
+                        for service, profile in self.profiles.items():
+                            # Assume there was no backlog initially if the service is new since last sync
+                            last_synced_backlog = 0
+                            if last_synced_profiles.get(service):
+                                 last_synced_backlog = last_synced_profiles[service].backlog
+
+                            # Check to see if the backlog has increased and if the service has been running since
+                            if profile.backlog and profile.backlog >= last_synced_backlog and \
+                                profile.running_instances == 0 and profile.target_instances > 0:
+                                # Restart the service in an attempt to resolve intermittent issues with container/pod
+                                self.controller.restart(profile)
+
+                    # Update the last synced profiles for next time
+                    last_synced_profiles = self.profiles
+
                 discovered_services: list[str] = []
 
                 # Get all the service data

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/assemblyline_core/updater/helper.py

@@ -21,6 +21,17 @@ class ContainerRegistry():
     def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
         raise NotImplementedError()
 
+    def _perform_request(self, url, headers, verify, proxies):
+        resp = None
+        try:
+            resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
+        except requests.exceptions.SSLError:
+            # Connect to insecure registry over HTTP (development only)
+            if not verify:
+                url = url.replace('https://','http://')
+                resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
+        return resp
+
 
 class DockerRegistry(ContainerRegistry):
     def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
@@ -37,20 +48,13 @@ class DockerRegistry(ContainerRegistry):
             # Assume the token server is the same as the container image registry host if not explicitly set
             token_server = token_server if token_server else server
             token_url = f"https://{token_server}/token?scope=repository:{image_name}:pull"
-            resp = requests.get(token_url)
-            if resp.ok:
+            resp = self._perform_request(token_url, headers, verify, proxies)
+            if resp and resp.ok:
                 # Request to obtain token was successful, set Authorization header for registry API
                 token = resp.json().get('token')
                 headers["Authorization"] = f"Bearer {token}"
 
-        resp = None
-        try:
-            resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
-        except requests.exceptions.SSLError:
-            # Connect to insecure registry over HTTP (development only)
-            if not verify:
-                url = f"http://{server}/v2/{image_name}/tags/list"
-                resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
+        resp = self._perform_request(url, headers, verify, proxies)
 
         # Test for valid response
         if resp and resp.ok:
@@ -76,20 +80,13 @@ class HarborRegistry(ContainerRegistry):
             # Assume the token server is the same as the container image registry host if not explicitly set
             token_server = token_server if token_server else server
             token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
-            resp = requests.get(token_url)
-            if resp.ok:
+            resp = self._perform_request(token_url, headers, verify, proxies)
+            if resp and resp.ok:
                 # Request to obtain token was successful, set Authorization header for registry API
                 token = resp.json().get('token')
                 headers["Authorization"] = f"Bearer {token}"
-        resp = None
-        try:
-            resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
-        except requests.exceptions.SSLError:
-            # Connect to insecure registry over HTTP (development only)
-            if not verify:
-                url = f"http://{server}/api/v2.0/projects/{project_id}/repositories/{repo_id}/artifacts"
-                resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
 
+        resp = self._perform_request(url, headers, verify, proxies)
         if resp and resp.ok:
             return [tag['name'] for image in resp.json() if image['tags'] for tag in image['tags']]
         return []

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/assemblyline_core/updater/run_updater.py

@@ -357,10 +357,24 @@ class KubernetesUpdateInterface:
 
         if blocking:
             try:
+                # Obtain the name of the pod that spawned from the Job
+                pod_name = self.api.list_namespaced_pod(namespace=self.namespace,
+                                                        label_selector=",".join([f"{k}={v}"
+                                                                                 for k, v in labels.items()]),
+                                                        limit=1).items[0].metadata.name
                 while not (status.failed or status.succeeded):
                     time.sleep(3)
                     status = self.batch_api.read_namespaced_job(namespace=self.namespace, name=name,
                                                                 _request_timeout=API_TIMEOUT).status
+                    # Monitor container's waiting status state
+                    pod_waiting_state = self.api.read_namespaced_pod(name=pod_name, namespace=self.namespace,
+                                                               _request_timeout=API_TIMEOUT).status.container_statuses[0].state.waiting
+                    # Check to see if we've encountered an issue before the container starts
+                    if pod_waiting_state and pod_waiting_state.reason == "ImagePullBackOff":
+                        # Delete job and raise exception
+                        self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace,
+                                                     propagation_policy='Background', _request_timeout=API_TIMEOUT)
+                        raise Exception(pod_waiting_state.message)
 
                 self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace,
                                                      propagation_policy='Background', _request_timeout=API_TIMEOUT)
@@ -479,6 +493,13 @@ class ServiceUpdater(ThreadedCoreBase):
                     image_name, tag_name, auth = get_latest_tag_for_service(
                         service,  self.config, self.log, prefix="[CI] ")
 
+                    if not tag_name:
+                        # Fallback to tag alias if we can't find a suitable versioned tag
+                        self.log.warning("Unable to find a versioned tag for "
+                                         f"{self.config.services.preferred_update_channel} channel. "
+                                         f"Defaulting to '{tag}' tag...")
+                        tag_name = tag
+
                     docker_config = dict(image=f"{image_name}:{tag_name}")
                     if auth:
                         docker_config.update(dict(registry_username=auth['username'],
@@ -507,12 +528,10 @@ class ServiceUpdater(ThreadedCoreBase):
                         blocking=True
                     )
 
-                    service_key = f"{service_name}_{tag_name.replace('stable', '')}"
-
                 except Exception as e:
                     self.log.error(
                         f"[CI] Service {service_name} has failed to install. Install procedure cancelled... [{str(e)}]")
-                return service_key
+                return f"{service_name}_{str(tag_name).replace('stable', '')}"
 
             # Start up installs for services in parallel
             install_threads = []

{assemblyline-core-4.5.0.19 → assemblyline-core-4.5.0.21}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.19
+Version: 4.5.0.21
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.19/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.19