assemblyline-core 4.5.0.33__tar.gz → 4.5.0.35__tar.gz

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.33
+Version: 4.5.0.35
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.35/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.0.35

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core/dispatching/client.py

@@ -254,7 +254,8 @@ class DispatchClient:
 
     @elasticapm.capture_span(span_type='dispatch_client')
     def service_finished(self, sid: str, result_key: str, result: Result,
-                         temporary_data: Optional[dict[str, Any]] = None):
+                         temporary_data: Optional[dict[str, Any]] = None,
+                         version="create"):
         """Notifies the dispatcher of service completion, and possible new files to dispatch."""
         # Make sure the dispatcher knows we were working on this task
         task_key = ServiceTask.make_key(sid=sid, service_name=result.response.service_name, sha=result.sha256)
@@ -272,28 +273,22 @@ class DispatchClient:
             self.ds.emptyresult.save(result_key, {"expiry_ts": result.expiry_ts})
         else:
             while True:
-                old, version = self.ds.result.get_if_exists(result_key, version=True)
-                if old:
-                    if old.expiry_ts and result.expiry_ts:
-                        result.expiry_ts = max(result.expiry_ts, old.expiry_ts)
-                    else:
-                        result.expiry_ts = None
                 try:
-                    if self.ds.result.exists(result_key):
-                        # A result already exists for this key
-                        # Regenerate entire result key based on result and modified task (ignore caching)
-                        task.ignore_cache = True
-                        result_key = Result.help_build_key(sha256=task.fileinfo.sha256,
-                                                           service_name=result.response.service_name,
-                                                           service_version=result.response.service_version,
-                                                           service_tool_version=result.response.service_tool_version,
-                                                           is_empty=False,
-                                                           task=task)
-                        version = "create"
                     self.ds.result.save(result_key, result, version=version)
                     break
                 except VersionConflictException as vce:
                     self.log.info(f"Retrying to save results due to version conflict: {str(vce)}")
+                    # A result already exists for this key
+                    # Regenerate entire result key based on result and modified task (ignore caching)
+                    version = "create"
+                    result.created = now_as_iso()
+                    task.ignore_cache = True
+                    result_key = Result.help_build_key(sha256=task.fileinfo.sha256,
+                                                       service_name=result.response.service_name,
+                                                       service_version=result.response.service_version,
+                                                       service_tool_version=result.response.service_tool_version,
+                                                       is_empty=False,
+                                                       task=task)
 
         # Send the result key to any watching systems
         msg = {'status': 'OK', 'cache_key': result_key}

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core/ingester/ingester.py

@@ -36,6 +36,7 @@ from assemblyline.remote.datatypes.queues.priority import PriorityQueue
 from assemblyline.remote.datatypes.queues.comms import CommsQueue
 from assemblyline.remote.datatypes.queues.multi import MultiQueue
 from assemblyline.remote.datatypes.hash import Hash
+from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
 from assemblyline import odm
 from assemblyline.odm.models.submission import SubmissionParams, Submission as DatabaseSubmission
 from assemblyline.odm.models.alert import EXTENDED_SCAN_VALUES
@@ -180,6 +181,9 @@ class Ingester(ThreadedCoreBase):
         # Utility object to handle post-processing actions
         self.postprocess_worker = ActionWorker(cache=True, config=self.config, datastore=self.datastore,
                                                redis_persist=self.redis_persist)
+        # Async Submission quota tracker
+        self.async_submission_tracker = UserQuotaTracker('async_submissions', timeout=24 * 60 * 60,  # 1 day timeout
+                                                         redis=self.redis_persist)
 
         if self.config.core.metrics.apm_server.server_url is not None:
             self.log.info(f"Exporting application metrics to: {self.config.core.metrics.apm_server.server_url}")
@@ -640,7 +644,7 @@ class Ingester(ThreadedCoreBase):
                     'archived': False,
                     'archive_ts': None,
                     'classification': task.params.classification,
-                    'expiry_ts':now_as_iso(task.params.ttl * 24 * 60 * 60),
+                    'expiry_ts': now_as_iso(task.params.ttl * 24 * 60 * 60),
                     'from_archive': False,
                     'metadata': task.submission.metadata,
                     'params': task.params.as_primitives(),
@@ -863,6 +867,9 @@ class Ingester(ThreadedCoreBase):
         return True
 
     def _notify_drop(self, task: IngestTask):
+        if self.config.ui.enforce_quota:
+            self.async_submission_tracker.end(task.params.submitter)
+
         self.send_notification(task)
 
         c12n = task.params.classification
@@ -936,4 +943,7 @@ class Ingester(ThreadedCoreBase):
                 task.extended_scan = 'submitted'
                 task.params.psid = None
 
+        if self.config.ui.enforce_quota:
+            self.async_submission_tracker.end(task.params.submitter)
+
         self.send_notification(task)

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core/scaler/scaler_server.py

@@ -35,6 +35,7 @@ from assemblyline.common.uid import get_id_from_data
 from assemblyline.common.forge import get_classification, get_service_queue, get_apm_client
 from assemblyline.common.constants import SCALER_TIMEOUT_QUEUE, SERVICE_STATE_HASH, ServiceStatus
 from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
+from assemblyline_core.updater.helper import get_registry_config
 from assemblyline_core.scaler.controllers import KubernetesController
 from assemblyline_core.scaler.controllers.interface import ServiceControlError
 from assemblyline_core.server_base import ServiceStage, ThreadedCoreBase
@@ -514,6 +515,12 @@ class ScalerServer(ThreadedCoreBase):
             for var in default_settings.environment:
                 if var.name not in set_keys:
                     docker_config.environment.append(var)
+
+            # Set authentication to registry to pull the image
+            auth_config = get_registry_config(docker_config, self.config)
+            docker_config.registry_username = auth_config['username']
+            docker_config.registry_password = auth_config['password']
+
             return docker_config
 
         # noinspection PyBroadException
@@ -533,7 +540,7 @@ class ScalerServer(ThreadedCoreBase):
             if not service.version.startswith(system_spec):
                 # If FW and SYS version don't prefix in the service version, we can't guarantee the
                 # service is compatible. Disable and treat it as incompatible due to service version.
-                self.log.warning("Disabling service with incompatible version. "
+                self.log.warning(f"Disabling {service.name} with incompatible version. "
                                  f"[{service.version} != '{system_spec}.X.{service.update_channel}Y'].")
                 disable_incompatible_service()
             elif service.update_config and service.update_config.wait_for_update and not service.update_config.sources:
@@ -648,7 +655,7 @@ class ScalerServer(ThreadedCoreBase):
                         profile.privileged = service.privileged
 
                         for dependency_name, dependency_blob in dependency_blobs.items():
-                            if profile.dependency_blobs[dependency_name] != dependency_blob:
+                            if profile.dependency_blobs.get(dependency_name, '') != dependency_blob:
                                 self.log.info(f"Updating deployment information for {name}/{dependency_name}")
                                 profile.dependency_blobs[dependency_name] = dependency_blob
                                 self.controller.start_stateful_container(

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core/tasking_client.py

@@ -24,6 +24,7 @@ from assemblyline.remote.datatypes.events import EventSender, EventWatcher
 from assemblyline.remote.datatypes.hash import ExpiringHash
 from assemblyline_core.dispatching.client import DispatchClient
 
+
 class TaskingClientException(Exception):
     pass
 
@@ -252,7 +253,7 @@ class TaskingClient:
         # If we are allowed, try to see if the result has been cached
         if not task.ignore_cache and not service_data.disable_cache:
             # Checking for previous results for this key
-            result = self.datastore.result.get_if_exists(result_key)
+            result, version = self.datastore.result.get_if_exists(result_key, version=True)
             if result:
                 metric_factory.increment('cache_hit')
                 if result.result.score:
@@ -262,8 +263,8 @@ class TaskingClient:
 
                 result.archive_ts = None
 
-                if task.ttl:
-                    result.expiry_ts = now_as_iso(task.ttl * 24 * 60 * 60)
+                if task.ttl and result.expiry_ts:
+                    result.expiry_ts = max(result.expiry_ts, now_as_iso(task.ttl * 24 * 60 * 60))
 
                 # Create a list of files to freshen
                 freshen_hashes = [task.fileinfo.sha256]
@@ -286,7 +287,7 @@ class TaskingClient:
                 for sha256 in freshen_hashes:
                     self.datastore.save_or_freshen_file(sha256, {}, result.expiry_ts, result.classification)
 
-                self.dispatch_client.service_finished(task.sid, result_key, result)
+                self.dispatch_client.service_finished(task.sid, result_key, result, version=version)
                 cache_found = True
 
             if not cache_found:

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core/updater/helper.py

@@ -5,13 +5,15 @@ import string
 import time
 
 from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
-from assemblyline.odm.models.config import Config as SystemConfig
-from assemblyline.odm.models.service import Service as ServiceConfig
+from assemblyline.odm.models.config import Config as SystemConfig, ServiceRegistry
+from assemblyline.odm.models.service import Service as ServiceConfig, DockerConfig
 
 from base64 import b64encode
 from collections import defaultdict
 from logging import Logger
+from typing import Dict, List
 from packaging.version import parse, Version
+from urllib.parse import urlencode
 
 DEFAULT_DOCKER_REGISTRY = "hub.docker.com"
 
@@ -130,9 +132,42 @@ REGISTRY_TYPE_MAPPING = {
     'harbor': HarborRegistry()
 }
 
-
-def get_latest_tag_for_service(
-        service_config: ServiceConfig, system_config: SystemConfig, logger: Logger, prefix: str = ""):
+def get_registry_config(docker_config: DockerConfig, system_config: SystemConfig) -> Dict[str, str]:
+    # Fix service image for calling Docker API
+    image_variables = defaultdict(str)
+    image_variables.update(system_config.services.image_variables)
+    image_variables.update(system_config.services.update_image_variables)
+    searchable_image = string.Template(docker_config.image).safe_substitute(image_variables)
+
+    server = searchable_image.split("/", 1)[0]
+
+    # Prioritize authentication given as a system configuration
+    registries: List[ServiceRegistry] = system_config.services.registries or []
+    for registry in registries:
+        if server.startswith(registry.name):
+            # Return authentication credentials and the type of registry
+            return dict(username=registry.username, password=registry.password, type=registry.type)
+
+    # Otherwise return what's configured for the service
+    return dict(username=docker_config.registry_username, password=docker_config.registry_password,
+                type=docker_config.registry_type)
+
+def get_latest_tag_for_service(service_config: ServiceConfig, system_config: SystemConfig, logger: Logger,
+                               prefix: str = ""):
+    """
+    Retrieves the latest compatible tag for a given service from its respective Docker registry.
+    This function processes the image name to determine the correct server, authenticates
+    with the registry, and then fetches the latest tag that matches the service's framework
+    and system version constraints. It handles different registry configurations including
+    private and public ones.
+    Args:
+        service_config (ServiceConfig): The configuration object for the service which includes Docker configuration.
+        system_config (SystemConfig): System-wide configuration which includes service updates and registry configurations.
+        logger (Logger): Logger object for logging messages.
+        prefix (str, optional): A prefix string to prepend to log messages for clarity in logs.
+    Returns:
+        tuple: Returns a tuple of the final image name, the latest tag, and authentication config used for the Docker registry.
+    """
     def process_image(image):
         # Find which server to search in
         server = image.split("/")[0]
@@ -152,10 +187,9 @@ def get_latest_tag_for_service(
 
         # Split repo name without the tag
         image_name = image_name.rsplit(":", 1)[0]
-
         return server, image_name
 
-    # Extract info
+    # Extract and process service information
     service_name = service_config.name
     image = service_config.docker_config.image
     update_channel = service_config.update_channel
@@ -168,36 +202,23 @@ def get_latest_tag_for_service(
 
     # Get authentication
     auth = None
-    auth_config = None
     server, image_name = process_image(searchable_image)
 
-    if not (service_config.docker_config.registry_username and service_config.docker_config.registry_password):
-        # If the passed in service configuration is missing registry credentials, check against system configuration
-        for registry in system_config.services.registries:
-            if server.startswith(registry['name']):
-                # Apply the credentials that the system is configured to use with the registry
-                service_config.docker_config.registry_username = registry['username']
-                service_config.docker_config.registry_password = registry['password']
-                service_config.docker_config.registry_type = registry['type']
-                break
-
-    if service_config.docker_config.registry_username and service_config.docker_config.registry_password:
-        # We're authenticating using Basic Auth
-        auth_config = {
-            'username': service_config.docker_config.registry_username,
-            'password': service_config.docker_config.registry_password
-        }
-        upass = f"{service_config.docker_config.registry_username}:{service_config.docker_config.registry_password}"
+    # Generate 'Authenication' header value for pulling tag list from registry
+    auth_config = get_registry_config(service_config.docker_config, system_config)
+    registry_type = auth_config.pop('type')
+    if auth_config['username'] and auth_config['password']:
+        upass = f"{auth_config['username']}:{auth_config['password']}"
         auth = f"Basic {b64encode(upass.encode()).decode()}"
-    elif service_config.docker_config.registry_password:
+    elif auth_config['password']:
         # We're assuming that if only a password is given, then this is a token
-        auth = f"Bearer {service_config.docker_config.registry_password}"
+        auth = f"Bearer {auth_config['password']}"
 
     if server.endswith(".azurecr.io"):
         # This is an Azure Container Registry based on the server name
         registry = AzureContainerRegistry()
     else:
-        registry = REGISTRY_TYPE_MAPPING[service_config.docker_config.registry_type]
+        registry = REGISTRY_TYPE_MAPPING[registry_type]
     token_server = None
     proxies = None
     for reg_conf in system_config.core.updater.registry_configs:
@@ -207,32 +228,27 @@ def get_latest_tag_for_service(
             break
 
     if server == DEFAULT_DOCKER_REGISTRY:
-        tags = _get_dockerhub_tags(image_name, update_channel, proxies)
+        tags = _get_dockerhub_tags(image_name, update_channel, prefix, proxies, logger=logger)
     else:
         tags = registry._get_proprietary_registry_tags(server, image_name, auth,
                                                        not system_config.services.allow_insecure_registry,
                                                        proxies, token_server)
-
-    tag_name = None
-
     # Pre-filter tags to only consider 'compatible' tags relative to the running system
-    tags = [t for t in tags
-            if re.match(f"({FRAMEWORK_VERSION})[.]({SYSTEM_VERSION})[.]\\d+[.]({update_channel})\\d+", t)]
-
+    tags = [tag for tag in tags
+            if re.match(f"({FRAMEWORK_VERSION})\.({SYSTEM_VERSION})\.\\d+\.({update_channel})\\d+", tag)]
     if not tags:
-        logger.warning(f"{prefix}Cannot fetch latest tag for service {service_name} - {image_name}"
+        logger.warning(f"{prefix} Cannot fetch latest tag for service {service_name} - {image_name}" \
                        f" => [server: {server}, repo_name: {image_name}, channel: {update_channel}]")
-    else:
-        for t in tags:
-            t_version = Version(t.replace(update_channel, ""))
-            # Tag name gets assigned to the first viable option then relies on comparison to get the latest
-            if not tag_name:
-                tag_name = t
-            elif t_version.major == FRAMEWORK_VERSION and t_version.minor == SYSTEM_VERSION and \
-                    t_version > parse(tag_name.replace(update_channel, "")):
-                tag_name = t
-
-        logger.info(f"{prefix}Latest {service_name} tag on {update_channel.upper()} channel is: {tag_name}")
+    tag_name = None
+    for tag in tags:
+        t_version = Version(tag.replace(update_channel, ""))
+        # Tag name gets assigned to the first viable option then relies on comparison to get the latest
+        if not tag_name:
+            tag_name = tag
+        elif t_version.major == FRAMEWORK_VERSION and t_version.minor == SYSTEM_VERSION and \
+                t_version > parse(tag_name.replace(update_channel, "")):
+            tag_name = tag
+    logger.info(f"{prefix}Latest {service_name} tag on {update_channel.upper()} channel is: {tag_name}")
 
     # Fix service image for use in Kubernetes
     image_variables = defaultdict(str)
@@ -246,29 +262,73 @@ def get_latest_tag_for_service(
 
     return image_name, tag_name, auth_config
 
-
-# Default for obtaining tags from DockerHub
-def _get_dockerhub_tags(image_name, update_channel, proxies=None):
-    # Find latest tag for each types
-    rv = []
+def _get_dockerhub_tags(image_name, update_channel, prefix, proxies=None, docker_registry=DEFAULT_DOCKER_REGISTRY,
+                        logger: Logger=None):
+    """
+    Retrieve DockerHub tags for a specific image and update channel.
+    Args:
+    image_name (str): Full name of the image (including namespace).
+    update_channel (str): Specific channel to filter tags (e.g., stable, beta).
+    proxies (dict, optional): Proxy configuration.
+    docker_registry (str, optional): Docker registry URL. Defaults to DEFAULT_DOCKER_REGISTRY.
+    Returns:
+    list: A list of tag names that match the given criteria, or None if an error occurs.
+    """
     namespace, repository = image_name.split('/', 1)
-    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/namespaces/{namespace}/repositories/{repository}/tags"
-    f"?page_size=50&page=1&name={update_channel}"
-
+    base_url = f"https://{docker_registry}/v2/namespaces/{namespace}/repositories/{repository}/tags"
+    query_params = {
+        'page_size': 50,
+        'page': 1,
+        'name': update_channel,
+        'ordering': 'last_updated'
+    }
+    url = f"{base_url}?{urlencode(query_params)}"
     while True:
-        resp = requests.get(url, proxies=proxies)
-        if resp.ok:
-            resp_data = resp.json()
-            rv.extend([x['name'] for x in resp_data['results']])
-            # Page until there are no results left
-            url = resp_data.get('next', None)
-            if url is None:
+        try:
+            response = requests.get(url, proxies=proxies)
+            response.raise_for_status()  # Raises an HTTPError for bad responses
+            response_data = response.json()
+            tags = []
+            if namespace == "cccs":
+                # The tags are coming from a repository managed by the Assemblyline team
+                for tag_info in response_data['results']:
+                    tag_name = tag_info.get('name')
+                    if tag_name and tag_name == f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.{update_channel}":
+                        # Ignore tag aliases containing the update_channel
+                        continue
+                    if tag_name and tag_name.startswith(f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}"):
+                        # Because the order of paging is based on `last_updated`,
+                        # we can return the first valid candidate
+                        logger.info(f"{prefix}Valid tag found for {repository}: {tag_name}")
+                        return [tag_name]
+            else:
+                # The tags are coming from another repository, so we're don't know the ordering of tags
+                tags.extend(tag_info.get('name') for tag_info in response_data['results'] if tag_info.get('name'))
+            url = response_data.get('next')
+            if not url:
+                logger.info(f"{prefix}No more pages left to fetch.")
                 break
-        elif resp.status_code == 429:
-            # Based on https://docs.docker.com/docker-hub/api/latest/#tag/rate-limiting
-            # We've hit the rate limit so we have to wait and try again later
-            time.sleep(int(resp.headers['retry-after']) - int(time.time()))
-        else:
+        except requests.exceptions.HTTPError as e:
+            if response.status_code == 429:
+                # Based on https://docs.docker.com/docker-hub/api/latest/#tag/rate-limiting
+                # We've hit the rate limit so we have to wait and try again later
+                logger.warning(f"{prefix}Rate limit reached for DockerHub. Retrying after sleep..")
+                time.sleep(int(response.headers['retry-after']) - int(time.time()))
+            else:
+                logger.error(f"{prefix}HTTP error occurred: {e}")
             break
-
-    return rv
+        except requests.exceptions.ConnectionError as e:
+            logger.error(f"{prefix}Connection error occurred: {e}")
+            break
+        except requests.exceptions.Timeout as e:
+            logger.error(f"{prefix}Request timed out.")
+            break
+        except requests.exceptions.RequestException as e:
+            logger.error(f"{prefix}An error occurred during requests: {e}")
+            break
+        except ValueError as e:
+            logger.error(f"{prefix}JSON decode error: {e}")
+            break
+    if tags:
+        logger.info(f"{prefix}Tags retrieved successfully: {tags}")
+    return tags

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core/workflow/run_workflow.py

@@ -175,7 +175,7 @@ class WorkflowManager(ServerBase):
 
                     except SearchException:
                         self.log.warning(f"Invalid query '{safe_str(workflow.query or '')}' in workflow "
-                                         f"'{workflow.name or 'unknown'}' by '{workflow.created_by or 'unknown'}'")
+                                         f"'{workflow.name or 'unknown'}' by '{workflow.creator or 'unknown'}'")
 
                         # End of transaction
                         if self.apm_client:

{assemblyline-core-4.5.0.33 → assemblyline-core-4.5.0.35}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.0.33
+Version: 4.5.0.35
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.0.33/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.0.33