assemblyline-core 4.5.0.54__tar.gz → 4.5.0.55__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of assemblyline-core might be problematic. Click here for more details.

Files changed (88) hide show
  1. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/PKG-INFO +1 -1
  2. assemblyline-core-4.5.0.55/assemblyline_core/VERSION +1 -0
  3. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/controllers/kubernetes_ctl.py +4 -1
  4. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/scaler_server.py +6 -1
  5. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/updater/run_updater.py +4 -1
  6. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core.egg-info/PKG-INFO +1 -1
  7. assemblyline-core-4.5.0.54/assemblyline_core/VERSION +0 -1
  8. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/LICENCE.md +0 -0
  9. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/README.md +0 -0
  10. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/__init__.py +0 -0
  11. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/alerter/__init__.py +0 -0
  12. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/alerter/processing.py +0 -0
  13. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/alerter/run_alerter.py +0 -0
  14. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/archiver/__init__.py +0 -0
  15. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/archiver/run_archiver.py +0 -0
  16. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/badlist_client.py +0 -0
  17. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/dispatching/__init__.py +0 -0
  18. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/dispatching/__main__.py +0 -0
  19. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/dispatching/client.py +0 -0
  20. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/dispatching/dispatcher.py +0 -0
  21. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/dispatching/schedules.py +0 -0
  22. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/dispatching/timeout.py +0 -0
  23. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/expiry/__init__.py +0 -0
  24. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/expiry/run_expiry.py +0 -0
  25. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/ingester/__init__.py +0 -0
  26. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/ingester/__main__.py +0 -0
  27. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/ingester/constants.py +0 -0
  28. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/ingester/ingester.py +0 -0
  29. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/__init__.py +0 -0
  30. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/es_metrics.py +0 -0
  31. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/heartbeat_formatter.py +0 -0
  32. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/helper.py +0 -0
  33. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/metrics_server.py +0 -0
  34. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/run_heartbeat_manager.py +0 -0
  35. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/run_metrics_aggregator.py +0 -0
  36. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/metrics/run_statistics_aggregator.py +0 -0
  37. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/plumber/__init__.py +0 -0
  38. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/plumber/run_plumber.py +0 -0
  39. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/__init__.py +0 -0
  40. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/client.py +0 -0
  41. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/creator/__init__.py +0 -0
  42. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/creator/run.py +0 -0
  43. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/creator/run_worker.py +0 -0
  44. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/loader/__init__.py +0 -0
  45. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/loader/run.py +0 -0
  46. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/loader/run_worker.py +0 -0
  47. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/replay/replay.py +0 -0
  48. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/safelist_client.py +0 -0
  49. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/__init__.py +0 -0
  50. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/collection.py +0 -0
  51. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/controllers/__init__.py +0 -0
  52. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/controllers/docker_ctl.py +0 -0
  53. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/controllers/interface.py +0 -0
  54. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/scaler/run_scaler.py +0 -0
  55. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/server_base.py +0 -0
  56. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/signature_client.py +0 -0
  57. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/submission_client.py +0 -0
  58. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/tasking_client.py +0 -0
  59. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/updater/__init__.py +0 -0
  60. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/updater/helper.py +0 -0
  61. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/vacuum/__init__.py +0 -0
  62. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/vacuum/crawler.py +0 -0
  63. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/vacuum/department_map.py +0 -0
  64. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/vacuum/safelist.py +0 -0
  65. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/vacuum/stream_map.py +0 -0
  66. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/vacuum/worker.py +0 -0
  67. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/workflow/__init__.py +0 -0
  68. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core/workflow/run_workflow.py +0 -0
  69. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core.egg-info/SOURCES.txt +0 -0
  70. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core.egg-info/dependency_links.txt +0 -0
  71. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core.egg-info/requires.txt +0 -0
  72. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/assemblyline_core.egg-info/top_level.txt +0 -0
  73. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/setup.cfg +0 -0
  74. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/setup.py +0 -0
  75. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_alerter.py +0 -0
  76. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_badlist_client.py +0 -0
  77. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_dispatcher.py +0 -0
  78. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_expiry.py +0 -0
  79. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_plumber.py +0 -0
  80. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_replay.py +0 -0
  81. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_safelist_client.py +0 -0
  82. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_scaler.py +0 -0
  83. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_scheduler.py +0 -0
  84. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_signature_client.py +0 -0
  85. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_simulation.py +0 -0
  86. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_vacuum.py +0 -0
  87. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_worker_ingest.py +0 -0
  88. {assemblyline-core-4.5.0.54 → assemblyline-core-4.5.0.55}/test/test_worker_submit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.54
3
+ Version: 4.5.0.55
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -0,0 +1 @@
1
+ 4.5.0.55
@@ -241,7 +241,8 @@ def parse_cpu(string: str) -> float:
241
241
  class KubernetesController(ControllerInterface):
242
242
  def __init__(self, logger, namespace: str, prefix: str, priority: str, dependency_priority: str,
243
243
  cpu_reservation: float, linux_node_selector: Selector, labels=None, log_level="INFO", core_env={},
244
- default_service_account=None, cluster_pod_list=True, default_service_tolerations = []):
244
+ default_service_account=None, cluster_pod_list=True, default_service_tolerations = [],
245
+ priv_labels=None):
245
246
  # Try loading a kubernetes connection from either the fact that we are running
246
247
  # inside of a cluster, or have a config file that tells us how
247
248
  try:
@@ -270,6 +271,7 @@ class KubernetesController(ControllerInterface):
270
271
  self.logger = logger
271
272
  self.log_level: str = log_level
272
273
  self._labels: dict[str, str] = labels or {}
274
+ self._priv_labels: dict[str, str] = priv_labels or {}
273
275
  self.linux_node_selector = linux_node_selector
274
276
  self.apps_api = client.AppsV1Api()
275
277
  self.api = client.CoreV1Api()
@@ -800,6 +802,7 @@ class KubernetesController(ControllerInterface):
800
802
  all_labels['component'] = service_name
801
803
  if core_mounts:
802
804
  all_labels['privilege'] = 'core'
805
+ all_labels.update(self._priv_labels)
803
806
  all_labels.update(labels or {})
804
807
 
805
808
  # Build set of volumes, first the global mounts, then the core specific ones,
@@ -293,6 +293,7 @@ class ScalerServer(ThreadedCoreBase):
293
293
  'section': 'service',
294
294
  'privilege': 'service'
295
295
  }
296
+ priv_labels = {}
296
297
 
297
298
  service_defaults_config = self.config.core.scaler.service_defaults
298
299
 
@@ -305,6 +306,9 @@ class ScalerServer(ThreadedCoreBase):
305
306
  if self.config.core.scaler.additional_labels:
306
307
  labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)})
307
308
 
309
+ if self.config.core.scaler.privileged_services_additional_labels:
310
+ priv_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.privileged_services_additional_labels)})
311
+
308
312
  if KUBERNETES_AL_CONFIG:
309
313
  self.log.info(f"Loading Kubernetes cluster interface on namespace: {NAMESPACE}")
310
314
  self.controller = KubernetesController(logger=self.log, prefix='alsvc_', labels=labels,
@@ -316,7 +320,8 @@ class ScalerServer(ThreadedCoreBase):
316
320
  core_env=core_env,
317
321
  cluster_pod_list=self.config.core.scaler.cluster_pod_list,
318
322
  default_service_account=self.config.services.service_account,
319
- default_service_tolerations=service_defaults_config.tolerations
323
+ default_service_tolerations=service_defaults_config.tolerations,
324
+ priv_labels=priv_labels
320
325
  )
321
326
 
322
327
  # Add global configuration for privileged services
@@ -457,7 +457,10 @@ class ServiceUpdater(ThreadedCoreBase):
457
457
  if 'KUBERNETES_SERVICE_HOST' in os.environ and NAMESPACE:
458
458
  extra_labels = {}
459
459
  if self.config.core.scaler.additional_labels:
460
- extra_labels = {k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)}
460
+ extra_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.additional_labels)})
461
+
462
+ if self.config.core.scaler.privileged_services_additional_labels:
463
+ extra_labels.update({k: v for k, v in (_l.split("=") for _l in self.config.core.scaler.privileged_services_additional_labels)})
461
464
 
462
465
  # If Updater has envs that set the service-server to use HTTPS, then assume a Root CA needs to be mounted
463
466
  if SERVICE_API_HOST and SERVICE_API_HOST.startswith('https'):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: assemblyline-core
3
- Version: 4.5.0.54
3
+ Version: 4.5.0.55
4
4
  Summary: Assemblyline 4 - Core components
5
5
  Home-page: https://github.com/CybercentreCanada/assemblyline-core/
6
6
  Author: CCCS Assemblyline development team
@@ -1 +0,0 @@
1
- 4.5.0.54