assemblyline-core 4.7.4.dev9__tar.gz → 4.7.4.1__tar.gz

{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.4.dev9
+Version: 4.7.4.1
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.4.1/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.7.4.1

{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.4.1}/assemblyline_core/ingester/ingester.py

@@ -9,51 +9,16 @@ score received, possibly sending a message to indicate that an alert should
 be created.
 """
 
-import logging
-import threading
-import time
 from os import environ
 from random import random
-from typing import Any, Iterable, List, Optional, Tuple
-
-import elasticapm
 
 from assemblyline import odm
-from assemblyline.common import exceptions, forge, isotime
-from assemblyline.common.constants import DROP_PRIORITY
-from assemblyline.common.exceptions import get_stacktrace_info
-from assemblyline.common.importing import load_module_by_path
-from assemblyline.common.isotime import now, now_as_iso
-from assemblyline.common.metrics import MetricsFactory
-from assemblyline.common.str_utils import dotdump, safe_str
-from assemblyline.datastore.exceptions import DataStoreException
-from assemblyline.filestore import CorruptedFileStoreException, FileStoreException
-from assemblyline.odm.messages.ingest_heartbeat import Metrics
 from assemblyline.odm.messages.submission import Submission as MessageSubmission
-from assemblyline.odm.messages.submission import SubmissionMessage
 from assemblyline.odm.models.alert import EXTENDED_SCAN_VALUES
-from assemblyline.odm.models.filescore import FileScore
-from assemblyline.odm.models.submission import Submission as DatabaseSubmission
 from assemblyline.odm.models.submission import SubmissionParams
-from assemblyline.odm.models.user import User
-from assemblyline.remote.datatypes.events import EventWatcher
-from assemblyline.remote.datatypes.hash import Hash
-from assemblyline.remote.datatypes.queues.comms import CommsQueue
-from assemblyline.remote.datatypes.queues.multi import MultiQueue
-from assemblyline.remote.datatypes.queues.named import NamedQueue
-from assemblyline.remote.datatypes.queues.priority import PriorityQueue
-from assemblyline.remote.datatypes.user_quota_tracker import UserQuotaTracker
-from assemblyline_core.dispatching.dispatcher import Dispatcher
-from assemblyline_core.server_base import ThreadedCoreBase
-from assemblyline_core.submission_client import SubmissionClient
-
-from .constants import COMPLETE_QUEUE_NAME, INGEST_QUEUE_NAME, drop_chance
-
-_dup_prefix = 'w-m-'
-_notification_queue_prefix = 'nq-'
-_max_retries = 10
-_retry_delay = 60 * 4  # Wait 4 minutes to retry
-_max_time = 2 * 24 * 60 * 60  # Wait 2 days for responses.
+
+from .constants import drop_chance
+
 HOUR_IN_SECONDS = 60 * 60
 COMPLETE_THREADS = int(environ.get('INGESTER_COMPLETE_THREADS', 4))
 INGEST_THREADS = int(environ.get('INGESTER_INGEST_THREADS', 1))

{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.4.1}/assemblyline_core/updater/helper.py

@@ -163,32 +163,9 @@ class HarborRegistry(ContainerRegistry):
         return []
 
 
-class JfrogRegistry(ContainerRegistry):
-    def _get_proprietary_registry_tags(self, server, image_name, auth, verify, proxies=None, token_server=None):
-        # Find latest tag for each types
-        # JFrog Artifactory Docker registry API: https://docs.jfrog.com/artifactory/reference/listDockerTags
-        # The repo key is derived from the first segment of the image name
-        repo_key, repo_path = image_name.split('/', 1)
-        url = f"https://{server}/artifactory/api/docker/{repo_key}/v2/{repo_path}/tags/list"
-
-        # Get tag list
-        headers = {}
-        if auth:
-            headers["Authorization"] = auth
-
-        resp = self._perform_request(url, headers, verify, proxies)
-
-        # Test for valid response
-        if resp and resp.ok:
-            resp_data = resp.json()
-            return resp_data['tags'] or []
-        return []
-
-
 REGISTRY_TYPE_MAPPING = {
     'docker': DockerRegistry(),
-    'harbor': HarborRegistry(),
-    'jfrog': JfrogRegistry()
+    'harbor': HarborRegistry()
 }
 
 def get_registry_config(docker_config: DockerConfig, system_config: SystemConfig) -> Dict[str, str]:

{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.4.1}/assemblyline_core/updater/run_updater.py

@@ -489,12 +489,18 @@ class ServiceUpdater(ThreadedCoreBase):
         # Add any mounts defined in the service defaults to the updater interface so they get applied to all updates/installs
         self.mounts = self.config.core.scaler.service_defaults.mounts
 
+        # If there are no services installed push the auto install services to the install table
+        installed_service_names = list(self.datastore.service_delta.keys())
+        if len(installed_service_names) == 0:
+            for service in self.config.core.updater.auto_install:
+                self.container_install.add(service.name, {'image': service.image})
+
     def _handle_service_change_event(self, data: Optional[ServiceChange]):
         if data is not None:
             if data.operation == Operation.Incompatible:
                 self.incompatible_services.add(data.name)
 
-    def container_installs(self):
+    def container_installs(self) -> None:
         """Go through the list of services and check what are the latest tags for it"""
         while self.running:
             self.log.info("[CI] Installing all services marked for install...")
@@ -561,8 +567,8 @@ class ServiceUpdater(ThreadedCoreBase):
                     )
 
                 except Exception as e:
-                    self.log.error(
-                        f"[CI] Service {service_name} has failed to install. Install procedure cancelled... [{str(e)}]")
+                    self.log.error("[CI] Service %s has failed to install. Install procedure cancelled... [%s]",
+                                   service_name, e)
                 finally:
                     self.container_keys.pop(api_key)
 
@@ -574,17 +580,18 @@ class ServiceUpdater(ThreadedCoreBase):
                 for service_name, install_data in self.container_install.items().items():
                     install_threads.append(service_installs_exec.submit(install_service, service_name, install_data))
 
-            # Check the status of the container installs
-            while install_threads:
-                # Delay each check by 5 seconds to give the container adequate time to perform version registration
-                time.sleep(5)
-                pending_threads = []
-                for thread in install_threads:
-                    if not thread.done():
-                        # Add thead to the list of still pending jobs
-                        pending_threads.append(thread)
-                    else:
-                        service_key = thread.result()
+                # Check the status of the container installs
+                while install_threads:
+                    # Delay each check by 5 seconds to give the container adequate time to perform version registration
+                    time.sleep(5)
+                    pending_threads = []
+                    for thread in install_threads:
+                        if not thread.done():
+                            # Add thead to the list of still pending jobs
+                            pending_threads.append(thread)
+                            continue
+
+                        service_key: str = thread.result()
                         service_name, latest_tag = service_key.split("_")
 
                         if self.datastore.service.get_if_exists(service_key):
@@ -612,13 +619,13 @@ class ServiceUpdater(ThreadedCoreBase):
                                 f"service key ({service_key}) does not exist. Install procedure cancelled...")
                         self.container_install.pop(service_name)
 
-                # Update with still pending and loop until all are completed/failed
-                install_threads = pending_threads
+                    # Update with still pending and loop until all are completed/failed
+                    install_threads = pending_threads
 
             # Clear out any old dead containers
             self.controller.cleanup_stale()
 
-            self.log.info(f"[CI] Done installing services, waiting {UPDATE_CHECK_INTERVAL} seconds for next install...")
+            self.log.info("[CI] Done installing services, waiting {UPDATE_CHECK_INTERVAL} seconds for next install...")
             time.sleep(UPDATE_CHECK_INTERVAL)
 
     def container_updates(self):

{assemblyline_core-4.7.4.dev9 → assemblyline_core-4.7.4.1}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.4.dev9
+Version: 4.7.4.1
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.4.dev9/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.7.4.dev9