assemblyline-core 4.5.1.dev68__tar.gz → 4.5.1.dev71__tar.gz

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev68
+Version: 4.5.1.dev71
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev71/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev71

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/assemblyline_core/archiver/run_archiver.py

@@ -7,8 +7,8 @@ from assemblyline.common import forge
 from assemblyline.common.archiving import ARCHIVE_QUEUE_NAME
 from assemblyline.common.metrics import MetricsFactory
 from assemblyline.datastore.collection import ESCollection, Index
+from assemblyline.datastore.exceptions import VersionConflictException
 from assemblyline.odm.messages.archive_heartbeat import Metrics
-from assemblyline.odm.models.submission import Submission
 from assemblyline.remote.datatypes import get_client
 from assemblyline.remote.datatypes.queues.named import NamedQueue
 
@@ -62,7 +62,12 @@ class Archiver(ServerBase):
             return
         else:
             try:
-                archive_type, type_id, delete_after = message
+                if len(message) == 3:
+                    archive_type, type_id, delete_after = message
+                    metadata = None
+                else:
+                    archive_type, type_id, delete_after, metadata = message
+
                 self.counter.increment('received')
             except Exception:
                 self.log.error(f"Invalid message received: {message}")
@@ -76,7 +81,19 @@ class Archiver(ServerBase):
             if archive_type == "submission":
                 self.counter.increment('submission')
                 # Load submission
-                submission: Submission = self.datastore.submission.get_if_exists(type_id)
+                while True:
+                    try:
+                        submission, version = self.datastore.submission.get_if_exists(type_id, version=True)
+
+                        # If we have metadata passed in the message, we need to apply it before archiving the submission
+                        if metadata and self.config.core.archiver.use_metadata:
+                            submission.metadata.update({f"archive.{k}": v for k, v in metadata.items()})
+                            self.datastore.submission.save(type_id, submission, version=version)
+
+                        break
+                    except VersionConflictException as vce:
+                        self.log.info(f"Retrying saving metadata due to version conflict: {str(vce)}")
+
                 if not submission:
                     raise SubmissionNotFound(type_id)
 

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/assemblyline_core/scaler/controllers/docker_ctl.py

@@ -158,6 +158,14 @@ class DockerController(ControllerInterface):
             if 'already exists' in str(e):
                 return
             raise e
+        except docker.errors.NotFound as e:
+            if aliases == ['service-server']:
+                # We've lost our service-server container, time to find another
+                self.service_server = self.find_service_server()
+                network.connect(self.service_server, aliases=aliases)
+                return
+            raise e
+
 
     def _start(self, service_name):
         """Launch a docker container in a manner suitable for Assemblyline."""

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/assemblyline_core/scaler/controllers/kubernetes_ctl.py

@@ -21,7 +21,7 @@ from assemblyline.odm.models.config import Selector
 
 from kubernetes import client, config, watch
 from kubernetes.client import V1Deployment, V1DeploymentSpec, V1PodTemplateSpec, V1DeploymentStrategy, \
-    V1PodSpec, V1PodOS, V1ObjectMeta, V1Volume, V1Container, V1VolumeMount, V1EnvVar, V1ConfigMapVolumeSource, \
+    V1PodSpec, V1ObjectMeta, V1Volume, V1Container, V1VolumeMount, V1EnvVar, V1ConfigMapVolumeSource, \
     V1PersistentVolumeClaimVolumeSource, V1LabelSelector, V1ResourceRequirements, V1PersistentVolumeClaim, \
     V1PersistentVolumeClaimSpec, V1NetworkPolicy, V1NetworkPolicySpec, V1NetworkPolicyEgressRule, V1NetworkPolicyPeer, \
     V1NetworkPolicyIngressRule, V1Secret, V1SecretVolumeSource, V1LocalObjectReference, V1Service, \
@@ -839,25 +839,14 @@ class KubernetesController(ControllerInterface):
                 volume_mounts=chown_mounts
             ))
 
-        pod_os = None
-        security_context = V1PodSecurityContext(fs_group=1000)
-        if docker_config.operating_system:
-            #  Allow Kubernetes to schedule the pod to a compatible node
-            pod_os = V1PodOS(name=docker_config.operating_system)
-
-        if docker_config.operating_system == 'windows':
-            security_context = None
-
-
         pod = V1PodSpec(
             init_containers=init_containers,
             volumes=all_volumes,
             containers=self._create_containers(service_name, deployment_name, docker_config,
                                                all_mounts, core_container=core_mounts),
-            os=pod_os,
             priority_class_name=self.dependency_priority if high_priority else self.priority,
             termination_grace_period_seconds=shutdown_seconds,
-            security_context=security_context,
+            security_context=V1PodSecurityContext(fs_group=1000),
             service_account_name=service_account,
             affinity=selector_to_node_affinity(self.linux_node_selector),
         )

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/assemblyline_core/updater/helper.py

@@ -12,145 +12,95 @@ from base64 import b64encode
 from collections import defaultdict
 from logging import Logger
 from packaging.version import parse, Version
-from typing import Any, Dict, List, Tuple
 
 DEFAULT_DOCKER_REGISTRY = "registry.hub.docker.com"
 
 
 class ContainerRegistry():
-    def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
-                 proxies: Dict[str, str] = None, *args, **kwargs):
-        self.server = server
-        self.session = requests.Session()
-        self.session.headers = headers
-        self.session.verify = verify
-        self.session.proxies = proxies
-
-    def _make_request(self, path: str) -> Dict[str, Any]:
-        request_path = f"{self.server}{path}"
-        resp = None
-        try:
-            resp = self.session.get(f"https://{request_path}")
-        except requests.exceptions.SSLError:
-            # Connect to insecure registry over HTTP (development only)
-            if not self.session.verify:
-                resp = self.session.get(f"http://{request_path}")
-        # Test for valid response
-        if resp and resp.ok:
-            return resp.json()
-        return None
-
     # Provide a means of obtaining a list of tags from a container registry
-    def get_image_tags(self, image_name) -> List[str]:
+    def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
         raise NotImplementedError()
 
-    # Provide a means of obtaining the compatible operating system for the container image
-    def get_image_os(self, image_name, image_tag) -> str:
-        raise NotImplementedError()
-
-    def get_token(self, image_name) -> None: ...
-
-        
-class DockerHub(ContainerRegistry):
-    def __init__(self, update_channel, proxies: Dict[str, str] = None, *args, **kwargs):
-        super().__init__(DEFAULT_DOCKER_REGISTRY, None, True, proxies)
-        self.update_channel = update_channel
-
-    def get_image_tags(self, image_name) -> List[str]:
-        resp = self._make_request(f"/v2/repositories/{image_name}/tags?page_size=5&page=1&name={self.update_channel}")
-        if resp:
-            return [x['name'] for x in resp['results']]
-        return []
-
-    def get_image_os(self, image_name, image_tag) -> str:
-        resp = self._make_request(f"/v2/repositories/{image_name}/tags/{image_tag}")
-        if resp:
-            return resp['images'][0]['os']
-        return None
 
-
-# Ref: https://docs.docker.com/registry/spec/api/#detail
 class DockerRegistry(ContainerRegistry):
-    def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
-                 proxies: Dict[str, str] = None, token_server: str = None, *args, **kwargs):
-        super().__init__(server, headers, verify, proxies, *args, **kwargs)
-        self.token_server = token_server      
+    def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
+        # Find latest tag for each types
+        url = f"https://{server}/v2/{image_name}/tags/list"
 
-    def get_token(self, image_name) -> None:
-        if not self.session.headers.get('Authorization'):
+        # Get tag list
+        headers = {}
+        if auth:
+            headers["Authorization"] = auth
+        else:
             # Retrieve token for authentication: https://distribution.github.io/distribution/spec/auth/token/
-            
+
             # Assume the token server is the same as the container image registry host if not explicitly set
-            token_server = self.token_server if self.token_server else self.server
+            token_server = token_server if token_server else server
             token_url = f"https://{token_server}/token?scope=repository:{image_name}:pull"
-            token = requests.get(token_url).json().get('token')
-            self.session.headers["Authorization"] = f"Bearer {token}"         
-    
-    def get_image_tags(self, image_name) -> List[str]:
-        # Find latest tag for each types
-        resp = self._make_request(f"/v2/{image_name}/tags/list")
-        if resp:
-            return resp['tags'] or []
-        return []
-
-    def get_image_os(self, image_name, image_tag) -> str:
-        resp = self._make_request(f"/v2/{image_name}/manifests/{image_tag}")
-        if resp:
-            # Retrieve OS compatibilty from historical record
-            return json.loads(resp['history'][0]['v1Compatibility'])['os']
-
-        # Unable to determine the OS compatibility
-        return None
+            resp = requests.get(token_url)
+            if resp.ok:
+                # Request to obtain token was successful, set Authorization header for registry API
+                token = resp.json().get('token')
+                headers["Authorization"] = f"Bearer {token}"
 
+        resp = None
+        try:
+            resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
+        except requests.exceptions.SSLError:
+            # Connect to insecure registry over HTTP (development only)
+            if not verify:
+                url = f"http://{server}/v2/{image_name}/tags/list"
+                resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
+        # Test for valid response
+        if resp and resp.ok:
+            # Test for positive list of tags
+            resp_data = resp.json()
+            return resp_data['tags'] or []
+        return []
 
-# Ref: https://github.com/goharbor/harbor/blob/main/api/v2.0/swagger.yaml
 class HarborRegistry(ContainerRegistry):
-    def __init__(self, server, headers: Dict[str, str] = None, verify: bool = True,
-                 proxies: Dict[str, str] = None, token_server: str = None, *args, **kwargs):
-        super().__init__(server, headers, verify, proxies, *args, **kwargs)
-                     
-    def get_token(self, image_name) -> None:
-        if not self.session.headers.get('Authorization'):
-            # Retrieve token for authentication: https://github.com/goharbor/harbor/wiki/Harbor-FAQs#api
-
-            # Assume the token server is the same as the container image registry host if not explicitly set
-            token_server = self.token_server if self.token_server else self.server
-            token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
-            token = requests.get(token_url).json().get('token')
-            headers["Authorization"] = f"Bearer {token}"
-      
-    def _get_project_repo_ids(self, image_name) -> Tuple[str, str]:
+    def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
         # Determine project/repo IDs from image name
         project_id, repo_id = image_name.split('/', 1)
         repo_id = repo_id.replace('/', "%2F")
-        return project_id, repo_id
+        url = f"https://{server}/api/v2.0/projects/{project_id}/repositories/{repo_id}/artifacts?page_size=0"
 
-    def get_image_tags(self, image_name) -> List[str]:
-        project_id, repo_id = self._get_project_repo_ids(image_name)
-        resp = self._make_request(f"/api/v2.0/projects/{project_id}/repositories/{repo_id}/artifacts?page_size=0")
-        if resp:
-            return [tag['name'] for image in resp if image['tags'] for tag in image['tags']]
-        return []
+        headers = {}
+        if auth:
+            headers["Authorization"] = auth
+        else:
+            # Retrieve token for authentication: https://github.com/goharbor/harbor/wiki/Harbor-FAQs#api
 
-    def get_image_os(self, image_name, image_tag) -> str:
-        project_id, repo_id = self._get_project_repo_ids(image_name)
-        resp = self._make_request(f"/api/v2.0/projects/{project_id}/repositories/{repo_id}/artifacts/{image_tag}")
-        if resp:
-            # Retrieve OS compatibilty from reference
-            return resp['references'][0]['platform']['os']
+            # Assume the token server is the same as the container image registry host if not explicitly set
+            token_server = token_server if token_server else server
+            token_url = f"https://{server}/service/token?scope=repository:{image_name}:pull"
+            resp = requests.get(token_url)
+            if resp.ok:
+                # Request to obtain token was successful, set Authorization header for registry API
+                token = resp.json().get('token')
+                headers["Authorization"] = f"Bearer {token}"
+        resp = None
+        try:
+            resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
+        except requests.exceptions.SSLError:
+            # Connect to insecure registry over HTTP (development only)
+            if not verify:
+                url = f"http://{server}/api/v2.0/projects/{project_id}/repositories/{repo_id}/artifacts"
+                resp = requests.get(url, headers=headers, verify=verify, proxies=proxies)
 
-        # Unable to determine the OS compatibility
-        return None
+        if resp and resp.ok:
+            return [tag['name'] for image in resp.json() if image['tags'] for tag in image['tags']]
+        return []
 
 
 REGISTRY_TYPE_MAPPING = {
-    'dockerhub': DockerHub,
-    'docker': DockerRegistry,
-    'harbor': HarborRegistry
+    'docker': DockerRegistry(),
+    'harbor': HarborRegistry()
 }
 
 
-def get_latest_tag_for_service(service_config: ServiceConfig, system_config: SystemConfig, logger: Logger, prefix: str = ""):
+def get_latest_tag_for_service(
+        service_config: ServiceConfig, system_config: SystemConfig, logger: Logger, prefix: str = ""):
     def process_image(image):
         # Find which server to search in
         server = image.split("/")[0]
@@ -211,6 +161,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
         # We're assuming that if only a password is given, then this is a token
         auth = f"Bearer {service_config.docker_config.registry_password}"
 
+    registry = REGISTRY_TYPE_MAPPING[service_config.docker_config.registry_type]
     token_server = None
     proxies = None
     for reg_conf in system_config.core.updater.registry_configs:
@@ -219,21 +170,14 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
             token_server = reg_conf.token_server or None
             break
 
-    registry_type = 'dockerhub' if server == DEFAULT_DOCKER_REGISTRY else service_config.docker_config.registry_type
-    registry_args = {
-        'server': server,
-        'headers': {'Authorization': auth},
-        'verify': not system_config.services.allow_insecure_registry,
-        'proxies': proxies,
-        'update_channel': update_channel,
-        'token_server': token_server
-    }
-
-    registry: ContainerRegistry = REGISTRY_TYPE_MAPPING[registry_type](**registry_args)
-    registry.get_token(image_name)
-    tags = registry.get_image_tags(image_name)
-
+    if server == DEFAULT_DOCKER_REGISTRY:
+        tags = _get_dockerhub_tags(image_name, update_channel, proxies)
+    else:
+        tags = registry._get_proprietary_registry_tags(server, image_name, auth,
+                                                       not system_config.services.allow_insecure_registry,
+                                                       proxies, token_server)
     tag_name = None
+
     # Pre-filter tags to only consider 'compatible' tags relative to the running system
     tags = [t for t in tags
             if re.match(f"({FRAMEWORK_VERSION})[.]({SYSTEM_VERSION})[.]\\d+[.]({update_channel})\\d+", t)]
@@ -257,10 +201,28 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
     image_variables = defaultdict(str)
     image_variables.update(system_config.services.image_variables)
     image = string.Template(image).safe_substitute(image_variables)
-    os = registry.get_image_os(image_name, tag_name)
+    server, image_name = process_image(image)
 
     # Append server to image if not the default server
-    if server != DEFAULT_DOCKER_REGISTRY:
+    if server != "registry.hub.docker.com":
         image_name = "/".join([server, image_name])
 
-    return image_name, tag_name, auth_config, os
+    return image_name, tag_name, auth_config
+
+
+# Default for obtaining tags from DockerHub
+def _get_dockerhub_tags(image_name, update_channel, proxies=None):
+    # Find latest tag for each types
+    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/repositories/{image_name}/tags" \
+        f"?page_size=5&page=1&name={update_channel}"
+
+    # Get tag list
+    resp = requests.get(url, proxies=proxies)
+
+    # Test for valid response
+    if resp.ok:
+        # Test for positive list of tags
+        resp_data = resp.json()
+        return [x['name'] for x in resp_data['results']]
+
+    return []

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/assemblyline_core/updater/run_updater.py

@@ -476,10 +476,10 @@ class ServiceUpdater(ThreadedCoreBase):
                          'version': tag,
                          'docker_config': {'image': install_data.get('image')}})
 
-                    image_name, tag_name, auth, os = get_latest_tag_for_service(
+                    image_name, tag_name, auth = get_latest_tag_for_service(
                         service,  self.config, self.log, prefix="[CI] ")
 
-                    docker_config = dict(image=f"{image_name}:{tag_name}", operating_system=os)
+                    docker_config = dict(image=f"{image_name}:{tag_name}")
                     if auth:
                         docker_config.update(dict(registry_username=auth['username'],
                                                   registry_password=auth['password']))
@@ -675,7 +675,7 @@ class ServiceUpdater(ThreadedCoreBase):
 
             for service in self.datastore.list_all_services(full=True):
                 discovered_services.append(service.name)
-                image_name, tag_name, auth, _ = get_latest_tag_for_service(service, self.config, self.log, prefix="[CV] ")
+                image_name, tag_name, auth = get_latest_tag_for_service(service, self.config, self.log, prefix="[CV] ")
                 self.latest_service_tags.set(service.name,
                                              {'auth': auth, 'image': image_name, service.update_channel: tag_name})
 

{assemblyline-core-4.5.1.dev68 → assemblyline-core-4.5.1.dev71}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev68
+Version: 4.5.1.dev71
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev68/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev68