assemblyline-core 4.5.0.69__tar.gz → 4.5.0.71__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/PKG-INFO +1 -1
  2. assemblyline_core-4.5.0.71/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/updater/helper.py +5 -2
  4. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/updater/run_updater.py +23 -2
  5. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core.egg-info/PKG-INFO +1 -1
  6. assemblyline_core-4.5.0.69/assemblyline_core/VERSION +0 -1
  7. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/LICENCE.md +0 -0
  8. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/README.md +0 -0
  9. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/dispatching/__main__.py +0 -0
  18. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/dispatching/dispatcher.py +0 -0
  20. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/dispatching/schedules.py +0 -0
  21. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/dispatching/timeout.py +0 -0
  22. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/expiry/__init__.py +0 -0
  23. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/expiry/run_expiry.py +0 -0
  24. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/ingester/__init__.py +0 -0
  25. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/ingester/__main__.py +0 -0
  26. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/ingester/constants.py +0 -0
  27. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/ingester/ingester.py +0 -0
  28. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/__init__.py +0 -0
  29. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/es_metrics.py +0 -0
  30. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  31. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/helper.py +0 -0
  32. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/metrics_server.py +0 -0
  33. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  34. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  35. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  36. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/plumber/__init__.py +0 -0
  37. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/plumber/run_plumber.py +0 -0
  38. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/__init__.py +0 -0
  39. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/client.py +0 -0
  40. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/creator/__init__.py +0 -0
  41. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/creator/run.py +0 -0
  42. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/creator/run_worker.py +0 -0
  43. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/loader/__init__.py +0 -0
  44. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/loader/run.py +0 -0
  45. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/loader/run_worker.py +0 -0
  46. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/replay/replay.py +0 -0
  47. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/safelist_client.py +0 -0
  48. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/__init__.py +0 -0
  49. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/collection.py +0 -0
  50. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  51. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  52. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/controllers/interface.py +0 -0
  53. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  54. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/run_scaler.py +0 -0
  55. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/scaler/scaler_server.py +0 -0
  56. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/server_base.py +0 -0
  57. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/signature_client.py +0 -0
  58. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/submission_client.py +0 -0
  59. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/tasking_client.py +0 -0
  60. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/updater/__init__.py +0 -0
  61. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/setup.cfg +0 -0
  74. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/setup.py +0 -0
  75. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_alerter.py +0 -0
  76. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_badlist_client.py +0 -0
  77. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_dispatcher.py +0 -0
  78. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_expiry.py +0 -0
  79. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_plumber.py +0 -0
  80. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_replay.py +0 -0
  81. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_safelist_client.py +0 -0
  82. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_scaler.py +0 -0
  83. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_scheduler.py +0 -0
  84. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_signature_client.py +0 -0
  85. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_simulation.py +0 -0
  86. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_vacuum.py +0 -0
  87. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_worker_ingest.py +0 -0
  88. {assemblyline_core-4.5.0.69 → assemblyline_core-4.5.0.71}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.69
3
+ Version: 4.5.0.71
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.0.71
@@ -171,7 +171,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
171
171
  logger (Logger): Logger object for logging messages.
172
172
  prefix (str, optional): A prefix string to prepend to log messages for clarity in logs.
173
173
  Returns:
174
- tuple: Returns a tuple of the final image name, the latest tag, and authentication config used for the Docker registry.
174
+ tuple: Returns a tuple of the final image name, the latest tag, authentication and auto_update config used for the Docker registry.
175
175
  """
176
176
  def process_image(image):
177
177
  # Find which server to search in
@@ -198,6 +198,9 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
198
198
  service_name = service_config.name
199
199
  image = service_config.docker_config.image
200
200
  update_channel = service_config.update_channel
201
+ auto_update = service_config.auto_update
202
+ if auto_update is None:
203
+ auto_update = system_config.services.default_auto_update
201
204
 
202
205
  # Fix service image for calling Docker API
203
206
  image_variables = defaultdict(str)
@@ -290,7 +293,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
290
293
  if server != DEFAULT_DOCKER_REGISTRY:
291
294
  image_name = "/".join([server, image_name])
292
295
 
293
- return image_name, tag_name, auth_config
296
+ return image_name, tag_name, auth_config, auto_update
294
297
 
295
298
  def _get_dockerhub_tags(image_name, update_channel, prefix, proxies=None, docker_registry=DEFAULT_DOCKER_REGISTRY,
296
299
  logger: Logger=None):
@@ -508,7 +508,7 @@ class ServiceUpdater(ThreadedCoreBase):
508
508
  'version': tag,
509
509
  'docker_config': {'image': install_data.get('image')}})
510
510
 
511
- image_name, tag_name, auth = get_latest_tag_for_service(
511
+ image_name, tag_name, auth, _ = get_latest_tag_for_service(
512
512
  service, self.config, self.log, prefix="[CI] ")
513
513
 
514
514
  if not tag_name:
@@ -712,10 +712,31 @@ class ServiceUpdater(ThreadedCoreBase):
712
712
 
713
713
  for service in self.datastore.list_all_services(full=True):
714
714
  discovered_services.append(service.name)
715
- image_name, tag_name, auth = get_latest_tag_for_service(service, self.config, self.log, prefix="[CV] ")
715
+ image_name, tag_name, auth, auto_update = get_latest_tag_for_service(
716
+ service, self.config, self.log, prefix="[CV] ")
716
717
  self.latest_service_tags.set(service.name,
717
718
  {'auth': auth, 'image': image_name, service.update_channel: tag_name})
718
719
 
720
+ # Check if the service is marked for auto-update and we found a version for it
721
+ if auto_update and tag_name:
722
+ self.log.info(f"[CV] {service.name} auto-update is enabled...")
723
+ srv_update_available = tag_name is not None and tag_name.replace('stable', '') != service.version
724
+ updating = self.container_update.exists(service.name)
725
+
726
+ # If the service is not already updating, and there is an update available, mark it for update
727
+ if srv_update_available and not updating:
728
+ self.log.info(f"[CV] Requesting update for service {service.name}...")
729
+ self.container_update.set(service.name, {
730
+ "auth": auth,
731
+ "image": f"{image_name}:{tag_name}",
732
+ "latest_tag": tag_name,
733
+ "update_available": srv_update_available,
734
+ "updating": updating
735
+ })
736
+ else:
737
+ self.log.info(f"[CV] {service.name} already marked for update...")
738
+
739
+
719
740
  # Remove services we have locally or in redis that have been deleted from the database
720
741
  for stray_service in existing_services - set(discovered_services):
721
742
  self.log.info(f"[CV] Service updates disabled for {stray_service}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.69
3
+ Version: 4.5.0.71
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.0.69