assemblyline-core 4.7.2.dev2__tar.gz → 4.7.2.dev5__tar.gz

{assemblyline_core-4.7.2.dev2 → assemblyline_core-4.7.2.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.2.dev2
+Version: 4.7.2.dev5
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.2.dev5/assemblyline_core/VERSION

@@ -0,0 +1 @@
+4.7.2.dev5

{assemblyline_core-4.7.2.dev2 → assemblyline_core-4.7.2.dev5}/assemblyline_core/metrics/metrics_server.py

@@ -200,55 +200,58 @@ class MetricsServer(ServerBase):
                     self.apm_client.end_transaction('process_message', 'success')
 
     def _create_aggregated_metrics(self):
-        self.log.info("Copying counters ...")
-        # APM Transaction start
-        if self.apm_client:
-            self.apm_client.begin_transaction('metrics')
-
-        with self.counters_lock:
-            counter_copy, self.counters = self.counters, {}
+        try:
+            self.log.info("Copying counters ...")
+            # APM Transaction start
+            if self.apm_client:
+                self.apm_client.begin_transaction('metrics')
 
-        self.log.info("Aggregating metrics ...")
-        timestamp = now_as_iso()
-        for component, counts in counter_copy.items():
-            component_name, component_type = component
-            output_metrics = {'name': component_name, 'type': component_type}
+            with self.counters_lock:
+                counter_copy, self.counters = self.counters, {}
 
-            for key, value in counts.items():
-                # Skip counts, they will be paired with a time entry and we only want to count it once
-                if key.endswith('.c'):
-                    continue
-                # We have an entry that is a timer, should also have a .c count
-                elif key.endswith('.t'):
-                    name = key.rstrip('.t')
-                    output_metrics[name] = counts[key] / counts.get(name + ".c", 1)
-                    output_metrics[name + "_count"] = counts.get(name + ".c", 0)
-                # Plain old metric, no modifications needed
-                else:
-                    output_metrics[key] = value
+            self.log.info("Aggregating metrics ...")
+            timestamp = now_as_iso()
+            for component, counts in counter_copy.items():
+                component_name, component_type = component
+                output_metrics = {'name': component_name, 'type': component_type}
+
+                for key, value in counts.items():
+                    # Skip counts, they will be paired with a time entry and we only want to count it once
+                    if key.endswith('.c'):
+                        continue
+                    # We have an entry that is a timer, should also have a .c count
+                    elif key.endswith('.t'):
+                        name = key.rstrip('.t')
+                        output_metrics[name] = counts[key] / max(counts.get(name + ".c", 1), 1)
+                        output_metrics[name + "_count"] = counts.get(name + ".c", 0)
+                    # Plain old metric, no modifications needed
+                    else:
+                        output_metrics[key] = value
 
-            ensure_indexes(self.log, self.es, self.config.core.metrics.elasticsearch, [component_type],
-                           datastream_enabled=self.is_datastream)
+                ensure_indexes(self.log, self.es, self.config.core.metrics.elasticsearch, [component_type],
+                               datastream_enabled=self.is_datastream)
 
-            index = f"al_metrics_{component_type}"
-            # Were data streams created for the index specified?
-            try:
-                if self.es.indices.get_index_template(name=f"{index}_ds"):
-                    output_metrics['@timestamp'] = timestamp
-                    index = f"{index}_ds"
-            except elasticsearch.exceptions.TransportError:
-                pass
-            output_metrics['timestamp'] = timestamp
-            output_metrics = cleanup_metrics(output_metrics)
+                index = f"al_metrics_{component_type}"
+                # Were data streams created for the index specified?
+                try:
+                    if self.es.indices.get_index_template(name=f"{index}_ds"):
+                        output_metrics['@timestamp'] = timestamp
+                        index = f"{index}_ds"
+                except elasticsearch.exceptions.TransportError:
+                    pass
+                output_metrics['timestamp'] = timestamp
+                output_metrics = cleanup_metrics(output_metrics)
 
-            self.log.info(output_metrics)
-            with_retries(self.log, self.es.index, index=index, body=output_metrics)
+                self.log.info(output_metrics)
+                with_retries(self.log, self.es.index, index=index, body=output_metrics)
 
-        self.log.info("Metrics aggregated. Waiting for next run...")
+            self.log.info("Metrics aggregated. Waiting for next run...")
 
-        # APM Transaction end
-        if self.apm_client:
-            self.apm_client.end_transaction('aggregate_metrics', 'success')
+            # APM Transaction end
+            if self.apm_client:
+                self.apm_client.end_transaction('aggregate_metrics', 'success')
+        except Exception:
+            self.log.exception("Unknown exception occurred during metrics aggregation:")
 
 
 # noinspection PyBroadException

{assemblyline_core-4.7.2.dev2 → assemblyline_core-4.7.2.dev5}/assemblyline_core/updater/helper.py

@@ -380,7 +380,19 @@ def _get_dockerhub_tags(image_name, update_channel, prefix, proxies=None, docker
                 # Based on https://docs.docker.com/docker-hub/api/latest/#tag/rate-limiting
                 # We've hit the rate limit so we have to wait and try again later
                 logger.warning(f"{prefix}Rate limit reached for DockerHub. Retrying after sleep..")
-                time.sleep(int(response.headers['retry-after']) - int(time.time()))
+                # Fix for crash when retry-after is seconds vs timestamp
+                retry_after = int(response.headers.get('retry-after', 60))
+                # Check if it looks like a timestamp (huge number) or duration (small number)
+                # Docker Hub API documentation says duration in seconds.
+                if retry_after > 1700000000: # It's a timestamp
+                     sleep_time = retry_after - int(time.time())
+                else: # It's a duration
+                     sleep_time = retry_after
+
+                if sleep_time < 0:
+                     sleep_time = 60 # Default to 60 seconds if calculation is wrong
+
+                time.sleep(sleep_time)
             else:
                 logger.error(f"{prefix}HTTP error occurred: {e}")
             break

{assemblyline_core-4.7.2.dev2 → assemblyline_core-4.7.2.dev5}/assemblyline_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: assemblyline-core
-Version: 4.7.2.dev2
+Version: 4.7.2.dev5
 Summary: Assemblyline 4 - Core components
 Home-page: https://github.com/CybercentreCanada/assemblyline-core/
 Author: CCCS Assemblyline development team

assemblyline_core-4.7.2.dev2/assemblyline_core/VERSION

@@ -1 +0,0 @@
-4.7.2.dev2