assemblyline-core 4.5.1.dev415__tar.gz → 4.5.1.dev417__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/PKG-INFO +1 -1
  2. assemblyline_core-4.5.1.dev417/assemblyline_core/VERSION +1 -0
  3. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/updater/helper.py +5 -2
  4. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/updater/run_updater.py +23 -2
  5. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core.egg-info/PKG-INFO +1 -1
  6. assemblyline_core-4.5.1.dev415/assemblyline_core/VERSION +0 -1
  7. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/LICENCE.md +0 -0
  8. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/README.md +0 -0
  9. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/__init__.py +0 -0
  10. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/alerter/__init__.py +0 -0
  11. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/alerter/processing.py +0 -0
  12. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/alerter/run_alerter.py +0 -0
  13. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/archiver/__init__.py +0 -0
  14. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/archiver/run_archiver.py +0 -0
  15. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/badlist_client.py +0 -0
  16. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/dispatching/__init__.py +0 -0
  17. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/dispatching/__main__.py +0 -0
  18. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/dispatching/client.py +0 -0
  19. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/dispatching/dispatcher.py +0 -0
  20. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/dispatching/schedules.py +0 -0
  21. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/dispatching/timeout.py +0 -0
  22. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/expiry/__init__.py +0 -0
  23. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/expiry/run_expiry.py +0 -0
  24. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/ingester/__init__.py +0 -0
  25. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/ingester/__main__.py +0 -0
  26. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/ingester/constants.py +0 -0
  27. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/ingester/ingester.py +0 -0
  28. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/__init__.py +0 -0
  29. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/es_metrics.py +0 -0
  30. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  31. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/helper.py +0 -0
  32. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/metrics_server.py +0 -0
  33. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  34. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  35. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  36. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/plumber/__init__.py +0 -0
  37. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/plumber/run_plumber.py +0 -0
  38. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/__init__.py +0 -0
  39. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/client.py +0 -0
  40. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/creator/__init__.py +0 -0
  41. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/creator/run.py +0 -0
  42. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/creator/run_worker.py +0 -0
  43. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/loader/__init__.py +0 -0
  44. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/loader/run.py +0 -0
  45. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/loader/run_worker.py +0 -0
  46. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/replay/replay.py +0 -0
  47. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/safelist_client.py +0 -0
  48. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/__init__.py +0 -0
  49. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/collection.py +0 -0
  50. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  51. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  52. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/controllers/interface.py +0 -0
  53. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +0 -0
  54. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/run_scaler.py +0 -0
  55. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/scaler/scaler_server.py +0 -0
  56. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/server_base.py +0 -0
  57. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/signature_client.py +0 -0
  58. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/submission_client.py +0 -0
  59. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/tasking_client.py +0 -0
  60. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/updater/__init__.py +0 -0
  61. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/setup.cfg +0 -0
  74. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/setup.py +0 -0
  75. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_alerter.py +0 -0
  76. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_badlist_client.py +0 -0
  77. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_dispatcher.py +0 -0
  78. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_expiry.py +0 -0
  79. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_plumber.py +0 -0
  80. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_replay.py +0 -0
  81. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_safelist_client.py +0 -0
  82. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_scaler.py +0 -0
  83. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_scheduler.py +0 -0
  84. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_signature_client.py +0 -0
  85. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_simulation.py +0 -0
  86. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_vacuum.py +0 -0
  87. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_worker_ingest.py +0 -0
  88. {assemblyline_core-4.5.1.dev415 → assemblyline_core-4.5.1.dev417}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev415
3
+ Version: 4.5.1.dev417
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.1.dev417
@@ -170,7 +170,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
170
170
  logger (Logger): Logger object for logging messages.
171
171
  prefix (str, optional): A prefix string to prepend to log messages for clarity in logs.
172
172
  Returns:
173
- tuple: Returns a tuple of the final image name, the latest tag, and authentication config used for the Docker registry.
173
+ tuple: Returns a tuple of the final image name, the latest tag, authentication and auto_update config used for the Docker registry.
174
174
  """
175
175
  def process_image(image):
176
176
  # Find which server to search in
@@ -197,6 +197,9 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
197
197
  service_name = service_config.name
198
198
  image = service_config.docker_config.image
199
199
  update_channel = service_config.update_channel
200
+ auto_update = service_config.auto_update
201
+ if auto_update is None:
202
+ auto_update = system_config.services.default_auto_update
200
203
 
201
204
  # Fix service image for calling Docker API
202
205
  image_variables = defaultdict(str)
@@ -290,7 +293,7 @@ def get_latest_tag_for_service(service_config: ServiceConfig, system_config: Sys
290
293
  if server != DEFAULT_DOCKER_REGISTRY:
291
294
  image_name = "/".join([server, image_name])
292
295
 
293
- return image_name, tag_name, auth_config
296
+ return image_name, tag_name, auth_config, auto_update
294
297
 
295
298
  def _get_dockerhub_tags(image_name, update_channel, prefix, proxies=None, docker_registry=DEFAULT_DOCKER_REGISTRY,
296
299
  logger: Logger=None):
@@ -508,7 +508,7 @@ class ServiceUpdater(ThreadedCoreBase):
508
508
  'version': tag,
509
509
  'docker_config': {'image': install_data.get('image')}})
510
510
 
511
- image_name, tag_name, auth = get_latest_tag_for_service(
511
+ image_name, tag_name, auth, _ = get_latest_tag_for_service(
512
512
  service, self.config, self.log, prefix="[CI] ")
513
513
 
514
514
  if not tag_name:
@@ -712,10 +712,31 @@ class ServiceUpdater(ThreadedCoreBase):
712
712
 
713
713
  for service in self.datastore.list_all_services(full=True):
714
714
  discovered_services.append(service.name)
715
- image_name, tag_name, auth = get_latest_tag_for_service(service, self.config, self.log, prefix="[CV] ")
715
+ image_name, tag_name, auth, auto_update = get_latest_tag_for_service(
716
+ service, self.config, self.log, prefix="[CV] ")
716
717
  self.latest_service_tags.set(service.name,
717
718
  {'auth': auth, 'image': image_name, service.update_channel: tag_name})
718
719
 
720
+ # Check if the service is marked for auto-update and we found a version for it
721
+ if auto_update and tag_name:
722
+ self.log.info(f"[CV] {service.name} auto-update is enabled...")
723
+ srv_update_available = tag_name is not None and tag_name.replace('stable', '') != service.version
724
+ updating = self.container_update.exists(service.name)
725
+
726
+ # If the service is not already updating, and there is an update available, mark it for update
727
+ if srv_update_available and not updating:
728
+ self.log.info(f"[CV] Requesting update for service {service.name}...")
729
+ self.container_update.set(service.name, {
730
+ "auth": auth,
731
+ "image": f"{image_name}:{tag_name}",
732
+ "latest_tag": tag_name,
733
+ "update_available": srv_update_available,
734
+ "updating": updating
735
+ })
736
+ else:
737
+ self.log.info(f"[CV] {service.name} already marked for update...")
738
+
739
+
719
740
  # Remove services we have locally or in redis that have been deleted from the database
720
741
  for stray_service in existing_services - set(discovered_services):
721
742
  self.log.info(f"[CV] Service updates disabled for {stray_service}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: assemblyline-core
3
- Version: 4.5.1.dev415
3
+ Version: 4.5.1.dev417
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.1.dev415