assemblyline-core 4.5.1.dev167__tar.gz → 4.5.1.dev169__tar.gz

{assemblyline-core-4.5.1.dev167 → assemblyline-core-4.5.1.dev169}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev167
+Version: 4.5.1.dev169
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev169/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.5.1.dev169

{assemblyline-core-4.5.1.dev167 → assemblyline-core-4.5.1.dev169}/assemblyline_core/tasking_client.py

@@ -177,6 +177,10 @@ class TaskingClient:
 
                         heuristic = Heuristic(heuristic)
                         heuristic_id = heuristic.heur_id
+                        existing_heuristic_obj = self.datastore.heuristic.get_if_exists(heuristic_id)
+                        if existing_heuristic_obj:
+                            # Ensure statistics of heuristic are preserved
+                            heuristic.stats = existing_heuristic_obj.stats
                         plan.add_upsert_operation(heuristic_id, heuristic)
                     except Exception as e:
                         msg = f"{service.name} has an invalid heuristic ({heuristic_id}): {str(e)}"

{assemblyline-core-4.5.1.dev167 → assemblyline-core-4.5.1.dev169}/assemblyline_core/updater/helper.py

@@ -1,8 +1,8 @@
-import json
 import requests
 import re
 import socket
 import string
+import time
 
 from assemblyline.common.version import FRAMEWORK_VERSION, SYSTEM_VERSION
 from assemblyline.odm.models.config import Config as SystemConfig
@@ -13,7 +13,7 @@ from collections import defaultdict
 from logging import Logger
 from packaging.version import parse, Version
 
-DEFAULT_DOCKER_REGISTRY = "registry.hub.docker.com"
+DEFAULT_DOCKER_REGISTRY = "hub.docker.com"
 
 
 class ContainerRegistry():
@@ -201,7 +201,7 @@ def get_latest_tag_for_service(
     server, image_name = process_image(image)
 
     # Append server to image if not the default server
-    if server != "registry.hub.docker.com":
+    if server != DEFAULT_DOCKER_REGISTRY:
         image_name = "/".join([server, image_name])
 
     return image_name, tag_name, auth_config
@@ -211,8 +211,10 @@ def get_latest_tag_for_service(
 def _get_dockerhub_tags(image_name, update_channel, proxies=None):
     # Find latest tag for each types
     rv = []
-    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/repositories/{image_name}/tags" \
-          f"?page_size=50&page=1&name={update_channel}"
+    namespace, repository = image_name.split('/', 1)
+    url = f"https://{DEFAULT_DOCKER_REGISTRY}/v2/namespaces/{namespace}/repositories/{repository}/tags"
+    f"?page_size=50&page=1&name={update_channel}"
+
     while True:
         resp = requests.get(url, proxies=proxies)
         if resp.ok:
@@ -222,6 +224,10 @@ def _get_dockerhub_tags(image_name, update_channel, proxies=None):
             url = resp_data.get('next', None)
             if url is None:
                 break
+        elif resp.status_code == 429:
+            # Based on https://docs.docker.com/docker-hub/api/latest/#tag/rate-limiting
+            # We've hit the rate limit so we have to wait and try again later
+            time.sleep(int(time.now()) - int(resp.headers['retry-after']))
         else:
             break
 

{assemblyline-core-4.5.1.dev167 → assemblyline-core-4.5.1.dev169}/assemblyline_core/updater/run_updater.py

@@ -359,11 +359,17 @@ class KubernetesUpdateInterface:
 
         if blocking:
             try:
-                # Obtain the name of the pod that spawned from the Job
-                pod_name = self.api.list_namespaced_pod(namespace=self.namespace,
-                                                        label_selector=",".join([f"{k}={v}"
-                                                                                 for k, v in labels.items()]),
-                                                        limit=1).items[0].metadata.name
+                pod_name = None
+                while not pod_name:
+                    time.sleep(3)
+                    # Obtain the name of the pod that spawned from the Job
+                    pod = self.api.list_namespaced_pod(namespace=self.namespace,
+                                                       label_selector=",".join([f"{k}={v}"
+                                                                                for k, v in labels.items()]),
+                                                                                limit=1)
+                    if pod.items:
+                        pod_name = pod.items[0].metadata.name
+
                 while not (status.failed or status.succeeded):
                     time.sleep(3)
                     status = self.batch_api.read_namespaced_job(namespace=self.namespace, name=name,

{assemblyline-core-4.5.1.dev167 → assemblyline-core-4.5.1.dev169}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: assemblyline-core
-Version: 4.5.1.dev167
+Version: 4.5.1.dev169
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline-core-4.5.1.dev167/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.5.1.dev167