assemblyline-core 4.5.1.dev108__tar.gz → 4.5.1.dev109__tar.gz

{assemblyline-core-4.5.1.dev108 → assemblyline-core-4.5.1.dev109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev108
+Version: 4.5.1.dev109
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev109/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev109

{assemblyline-core-4.5.1.dev108 → assemblyline-core-4.5.1.dev109}/assemblyline_core/scaler/controllers/docker_ctl.py

@@ -191,7 +191,7 @@ class DockerController(ControllerInterface):
         # Define environment variables
         env = [f'{_e.name}={_e.value}' for _e in cfg.environment]
         env += [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
-        env += [f'LOG_LEVEL={self.log_level}']
+        env += [f'LOG_LEVEL={self.log_level}', f'AL_SERVICE_NAME={service_name}']
         env += [f'{_n}={_v}' for _n, _v in self._service_limited_env[service_name].items()]
         if prof.privileged:
             env.append('PRIVILEGED=true')
@@ -259,6 +259,15 @@ class DockerController(ControllerInterface):
         env += [f'{name}={os.environ[name]}' for name in INHERITED_VARIABLES if name in os.environ]
         env += [f'LOG_LEVEL={self.log_level}', f'AL_SERVICE_NAME={service_name}']
 
+        healthcheck = {
+            'test': ["CMD", "python3", "-m", "assemblyline_v4_service.healthz"],
+            'interval': SERVICE_LIVENESS_PERIOD,
+            'timeout': SERVICE_LIVENESS_TIMEOUT
+        }
+
+        if 'assemblyline' not in cfg.image:
+            healthcheck = None
+
         container = self.client.containers.run(
             image=cfg.image,
             name=name,
@@ -274,12 +283,9 @@ class DockerController(ControllerInterface):
             environment=env,
             detach=True,
             # ports=ports,
-            healthcheck={
-                'test': ["CMD", "python3", "-m", "assemblyline_v4_service.healthz"],
-                'interval': SERVICE_LIVENESS_PERIOD,
-                'timeout': SERVICE_LIVENESS_TIMEOUT
-            }
+            healthcheck=healthcheck
         )
+
         if core_container:
             self._connect_to_network(container, self.core_network, aliases=[hostname])
 

{assemblyline-core-4.5.1.dev108 → assemblyline-core-4.5.1.dev109}/assemblyline_core/scaler/scaler_server.py

@@ -452,11 +452,30 @@ class ScalerServer(ThreadedCoreBase):
                 self._sync_service(service)
 
     def sync_services(self):
+        last_synced_profiles = None
         while self.running:
             with apm_span(self.apm_client, 'sync_services'):
                 self.log.info('Synchronizing service configuration')
                 with self.profiles_lock:
                     current_services = set(self.profiles.keys())
+
+                    # Check to see if the service is progressing since it's last sync
+                    if last_synced_profiles:
+                        for service, profile in self.profiles.items():
+                            # Assume there was no backlog initially if the service is new since last sync
+                            last_synced_backlog = 0
+                            if last_synced_profiles.get(service):
+                                 last_synced_backlog = last_synced_profiles[service].backlog
+
+                            # Check to see if the backlog has increased and if the service has been running since
+                            if profile.backlog and profile.backlog >= last_synced_backlog and \
+                                profile.running_instances == 0 and profile.target_instances > 0:
+                                # Restart the service in an attempt to resolve intermittent issues with container/pod
+                                self.controller.restart(profile)
+
+                    # Update the last synced profiles for next time
+                    last_synced_profiles = self.profiles
+
                 discovered_services: list[str] = []
 
                 # Get all the service data

{assemblyline-core-4.5.1.dev108 → assemblyline-core-4.5.1.dev109}/assemblyline_core/updater/run_updater.py

@@ -357,10 +357,24 @@ class KubernetesUpdateInterface:
 
         if blocking:
             try:
+                # Obtain the name of the pod that spawned from the Job
+                pod_name = self.api.list_namespaced_pod(namespace=self.namespace,
+                                                        label_selector=",".join([f"{k}={v}"
+                                                                                 for k, v in labels.items()]),
+                                                        limit=1).items[0].metadata.name
                 while not (status.failed or status.succeeded):
                     time.sleep(3)
                     status = self.batch_api.read_namespaced_job(namespace=self.namespace, name=name,
                                                                 _request_timeout=API_TIMEOUT).status
+                    # Monitor container's waiting status state
+                    pod_waiting_state = self.api.read_namespaced_pod(name=pod_name, namespace=self.namespace,
+                                                               _request_timeout=API_TIMEOUT).status.container_statuses[0].state.waiting
+                    # Check to see if we've encountered an issue before the container starts
+                    if pod_waiting_state and pod_waiting_state.reason == "ImagePullBackOff":
+                        # Delete job and raise exception
+                        self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace,
+                                                     propagation_policy='Background', _request_timeout=API_TIMEOUT)
+                        raise Exception(pod_waiting_state.message)
 
                 self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace,
                                                      propagation_policy='Background', _request_timeout=API_TIMEOUT)
@@ -479,6 +493,13 @@ class ServiceUpdater(ThreadedCoreBase):
                     image_name, tag_name, auth = get_latest_tag_for_service(
                         service,  self.config, self.log, prefix="[CI] ")
 
+                    if not tag_name:
+                        # Fallback to tag alias if we can't find a suitable versioned tag
+                        self.log.warning("Unable to find a versioned tag for "
+                                         f"{self.config.services.preferred_update_channel} channel. "
+                                         f"Defaulting to '{tag}' tag...")
+                        tag_name = tag
+
                     docker_config = dict(image=f"{image_name}:{tag_name}")
                     if auth:
                         docker_config.update(dict(registry_username=auth['username'],
@@ -507,12 +528,10 @@ class ServiceUpdater(ThreadedCoreBase):
                         blocking=True
                     )
 
-                    service_key = f"{service_name}_{tag_name.replace('stable', '')}"
-
                 except Exception as e:
                     self.log.error(
                         f"[CI] Service {service_name} has failed to install. Install procedure cancelled... [{str(e)}]")
-                return service_key
+                return f"{service_name}_{str(tag_name).replace('stable', '')}"
 
             # Start up installs for services in parallel
             install_threads = []

{assemblyline-core-4.5.1.dev108 → assemblyline-core-4.5.1.dev109}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev108
+Version: 4.5.1.dev109
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev108/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev108